diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index a324d20abc9..7cdf11fec0f 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -452,7 +452,8 @@ jobs: - name: Check docker clickhouse/clickhouse-server building run: | cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type head --no-push + python3 docker_server.py --release-type head --no-push \ + --image-repo clickhouse/clickhouse-server --image-path docker/server python3 docker_server.py --release-type head --no-push --no-ubuntu \ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - name: Cleanup diff --git a/.github/workflows/cherry_pick.yml b/.github/workflows/cherry_pick.yml index 065e584182b..8d1e2055978 100644 --- a/.github/workflows/cherry_pick.yml +++ b/.github/workflows/cherry_pick.yml @@ -35,7 +35,6 @@ jobs: fetch-depth: 0 - name: Cherry pick run: | - sudo pip install GitPython cd "$GITHUB_WORKSPACE/tests/ci" python3 cherry_pick.py - name: Cleanup diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index b70fe256833..7c5e477ab60 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -860,7 +860,8 @@ jobs: - name: Check docker clickhouse/clickhouse-server building run: | cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type head + python3 docker_server.py --release-type head \ + --image-repo clickhouse/clickhouse-server --image-path docker/server python3 docker_server.py --release-type head --no-ubuntu \ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - name: Cleanup diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ff98739db00..2f2c263df37 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -37,7 +37,6 @@ jobs: cd "$GITHUB_WORKSPACE/tests/ci" python3 run_check.py PythonUnitTests: - needs: CheckLabels runs-on: [self-hosted, style-checker] steps: - name: Check out repository code @@ -917,7 +916,8 @@ jobs: - name: Check docker clickhouse/clickhouse-server building run: | cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type head --no-push + python3 docker_server.py --release-type head --no-push \ + --image-repo clickhouse/clickhouse-server --image-path docker/server python3 docker_server.py --release-type head --no-push --no-ubuntu \ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - name: Cleanup diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2ef05fe989b..73246af6dfc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,15 +7,28 @@ on: # yamllint disable-line rule:truthy release: types: - published + workflow_dispatch: + inputs: + tag: + description: 'Release tag' + required: true + type: string jobs: ReleasePublish: runs-on: [self-hosted, style-checker] steps: + - name: Set tag from input + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_TAG=${{ github.event.inputs.tag }}" >> "$GITHUB_ENV" + - name: Set tag from REF + if: github.event_name == 'release' + run: | + echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV" - name: Deploy packages and assets run: | - GITHUB_TAG="${GITHUB_REF#refs/tags/}" - curl --silent --data '' \ + curl --silent --data '' --no-buffer \ '${{ secrets.PACKAGES_RELEASE_URL }}/release/'"${GITHUB_TAG}"'?binary=binary_darwin&binary=binary_darwin_aarch64&sync=true' ############################################################################################ ##################################### Docker images ####################################### @@ -23,16 +36,26 @@ jobs: DockerServerImages: runs-on: [self-hosted, style-checker] steps: + - name: Set tag from input + if: github.event_name == 'workflow_dispatch' + run: | + echo "GITHUB_TAG=${{ github.event.inputs.tag }}" >> "$GITHUB_ENV" + - name: Set tag from REF + if: github.event_name == 'release' + run: | + echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV" - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true fetch-depth: 0 # otherwise we will have no version info + ref: ${{ env.GITHUB_TAG }} - name: Check docker clickhouse/clickhouse-server building run: | cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type auto --version "${{ github.ref }}" - python3 docker_server.py --release-type auto --version "${{ github.ref }}" --no-ubuntu \ + python3 docker_server.py --release-type auto --version "$GITHUB_TAG" \ + --image-repo clickhouse/clickhouse-server --image-path docker/server + python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --no-ubuntu \ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - name: Cleanup if: always() diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 74ec1163cc9..e56a1fb58fc 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -525,7 +525,8 @@ jobs: - name: Check docker clickhouse/clickhouse-server building run: | cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type head --no-push + python3 docker_server.py --release-type head --no-push \ + --image-repo clickhouse/clickhouse-server --image-path docker/server python3 docker_server.py --release-type head --no-push --no-ubuntu \ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - name: Cleanup diff --git a/base/base/hex.h b/base/base/hex.h new file mode 100644 index 00000000000..e0c57f9dd42 --- /dev/null +++ b/base/base/hex.h @@ -0,0 +1,214 @@ +#pragma once + +#include +#include "types.h" + +/// Maps 0..15 to 0..9A..F or 0..9a..f correspondingly. + +constexpr inline std::string_view hex_digit_to_char_uppercase_table = "0123456789ABCDEF"; +constexpr inline std::string_view hex_digit_to_char_lowercase_table = "0123456789abcdef"; + +constexpr char hexDigitUppercase(unsigned char c) +{ + return hex_digit_to_char_uppercase_table[c]; +} +constexpr char hexDigitLowercase(unsigned char c) +{ + return hex_digit_to_char_lowercase_table[c]; +} + +/// Maps 0..255 to 00..FF or 00..ff correspondingly + +constexpr inline std::string_view hex_byte_to_char_uppercase_table = // + "000102030405060708090A0B0C0D0E0F" + "101112131415161718191A1B1C1D1E1F" + "202122232425262728292A2B2C2D2E2F" + "303132333435363738393A3B3C3D3E3F" + "404142434445464748494A4B4C4D4E4F" + "505152535455565758595A5B5C5D5E5F" + "606162636465666768696A6B6C6D6E6F" + "707172737475767778797A7B7C7D7E7F" + "808182838485868788898A8B8C8D8E8F" + "909192939495969798999A9B9C9D9E9F" + "A0A1A2A3A4A5A6A7A8A9AAABACADAEAF" + "B0B1B2B3B4B5B6B7B8B9BABBBCBDBEBF" + "C0C1C2C3C4C5C6C7C8C9CACBCCCDCECF" + "D0D1D2D3D4D5D6D7D8D9DADBDCDDDEDF" + "E0E1E2E3E4E5E6E7E8E9EAEBECEDEEEF" + "F0F1F2F3F4F5F6F7F8F9FAFBFCFDFEFF"; + +constexpr inline std::string_view hex_byte_to_char_lowercase_table = // + "000102030405060708090a0b0c0d0e0f" + "101112131415161718191a1b1c1d1e1f" + "202122232425262728292a2b2c2d2e2f" + "303132333435363738393a3b3c3d3e3f" + "404142434445464748494a4b4c4d4e4f" + "505152535455565758595a5b5c5d5e5f" + "606162636465666768696a6b6c6d6e6f" + "707172737475767778797a7b7c7d7e7f" + "808182838485868788898a8b8c8d8e8f" + "909192939495969798999a9b9c9d9e9f" + "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf" + "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf" + "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf" + "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf" + "e0e1e2e3e4e5e6e7e8e9eaebecedeeef" + "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"; + +inline void writeHexByteUppercase(UInt8 byte, void * out) +{ + memcpy(out, &hex_byte_to_char_uppercase_table[static_cast(byte) * 2], 2); +} + +inline void writeHexByteLowercase(UInt8 byte, void * out) +{ + memcpy(out, &hex_byte_to_char_lowercase_table[static_cast(byte) * 2], 2); +} + +constexpr inline std::string_view bin_byte_to_char_table = // + "0000000000000001000000100000001100000100000001010000011000000111" + "0000100000001001000010100000101100001100000011010000111000001111" + "0001000000010001000100100001001100010100000101010001011000010111" + "0001100000011001000110100001101100011100000111010001111000011111" + "0010000000100001001000100010001100100100001001010010011000100111" + "0010100000101001001010100010101100101100001011010010111000101111" + "0011000000110001001100100011001100110100001101010011011000110111" + "0011100000111001001110100011101100111100001111010011111000111111" + "0100000001000001010000100100001101000100010001010100011001000111" + "0100100001001001010010100100101101001100010011010100111001001111" + "0101000001010001010100100101001101010100010101010101011001010111" + "0101100001011001010110100101101101011100010111010101111001011111" + "0110000001100001011000100110001101100100011001010110011001100111" + "0110100001101001011010100110101101101100011011010110111001101111" + "0111000001110001011100100111001101110100011101010111011001110111" + "0111100001111001011110100111101101111100011111010111111001111111" + "1000000010000001100000101000001110000100100001011000011010000111" + "1000100010001001100010101000101110001100100011011000111010001111" + "1001000010010001100100101001001110010100100101011001011010010111" + "1001100010011001100110101001101110011100100111011001111010011111" + "1010000010100001101000101010001110100100101001011010011010100111" + "1010100010101001101010101010101110101100101011011010111010101111" + "1011000010110001101100101011001110110100101101011011011010110111" + "1011100010111001101110101011101110111100101111011011111010111111" + "1100000011000001110000101100001111000100110001011100011011000111" + "1100100011001001110010101100101111001100110011011100111011001111" + "1101000011010001110100101101001111010100110101011101011011010111" + "1101100011011001110110101101101111011100110111011101111011011111" + "1110000011100001111000101110001111100100111001011110011011100111" + "1110100011101001111010101110101111101100111011011110111011101111" + "1111000011110001111100101111001111110100111101011111011011110111" + "1111100011111001111110101111101111111100111111011111111011111111"; + +inline void writeBinByte(UInt8 byte, void * out) +{ + memcpy(out, &bin_byte_to_char_table[static_cast(byte) * 8], 8); +} + +/// Produces hex representation of an unsigned int with leading zeros (for checksums) +template +inline void writeHexUIntImpl(TUInt uint_, char * out, std::string_view table) +{ + union + { + TUInt value; + UInt8 uint8[sizeof(TUInt)]; + }; + + value = uint_; + + for (size_t i = 0; i < sizeof(TUInt); ++i) + { + if constexpr (std::endian::native == std::endian::little) + memcpy(out + i * 2, &table[static_cast(uint8[sizeof(TUInt) - 1 - i]) * 2], 2); + else + memcpy(out + i * 2, &table[static_cast(uint8[i]) * 2], 2); + } +} + +template +inline void writeHexUIntUppercase(TUInt uint_, char * out) +{ + writeHexUIntImpl(uint_, out, hex_byte_to_char_uppercase_table); +} + +template +inline void writeHexUIntLowercase(TUInt uint_, char * out) +{ + writeHexUIntImpl(uint_, out, hex_byte_to_char_lowercase_table); +} + +template +std::string getHexUIntUppercase(TUInt uint_) +{ + std::string res(sizeof(TUInt) * 2, '\0'); + writeHexUIntUppercase(uint_, res.data()); + return res; +} + +template +std::string getHexUIntLowercase(TUInt uint_) +{ + std::string res(sizeof(TUInt) * 2, '\0'); + writeHexUIntLowercase(uint_, res.data()); + return res; +} + +/// Maps 0..9, A..F, a..f to 0..15. Other chars are mapped to implementation specific value. + +constexpr inline std::string_view hex_char_to_digit_table + = {"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\xff\xff\xff\xff\xff\xff" //0-9 + "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //A-Z + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //a-z + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", + 256}; + +constexpr UInt8 unhex(char c) +{ + return hex_char_to_digit_table[static_cast(c)]; +} + +constexpr UInt8 unhex2(const char * data) +{ + return static_cast(unhex(data[0])) * 0x10 + static_cast(unhex(data[1])); +} + +constexpr UInt16 unhex4(const char * data) +{ + return static_cast(unhex(data[0])) * 0x1000 + static_cast(unhex(data[1])) * 0x100 + + static_cast(unhex(data[2])) * 0x10 + static_cast(unhex(data[3])); +} + +template +constexpr TUInt unhexUInt(const char * data) +{ + TUInt res = 0; + if constexpr ((sizeof(TUInt) <= 8) || ((sizeof(TUInt) % 8) != 0)) + { + for (size_t i = 0; i < sizeof(TUInt) * 2; ++i, ++data) + { + res <<= 4; + res += unhex(*data); + } + } + else + { + for (size_t i = 0; i < sizeof(TUInt) / 8; ++i, data += 16) + { + res <<= 64; + res += unhexUInt(data); + } + } + return res; +} diff --git a/base/base/interpolate.h b/base/base/interpolate.h new file mode 100644 index 00000000000..1d4fc0b6257 --- /dev/null +++ b/base/base/interpolate.h @@ -0,0 +1,13 @@ +#pragma once +#include +#include + +/** Linear interpolation in logarithmic coordinates. + * Exponential interpolation is related to linear interpolation + * exactly in same way as geometric mean is related to arithmetic mean. + */ +constexpr double interpolateExponential(double min, double max, double ratio) +{ + assert(min > 0 && ratio >= 0 && ratio <= 1); + return min * std::pow(max / min, ratio); +} diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index ae6f270a768..4181f916d63 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -115,6 +115,13 @@ configure_file("${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in" "${ORC_BUILD_INCLUDE_DIR}/A # ARROW_ORC + adapters/orc/CMakefiles set(ORC_SRCS + "${CMAKE_CURRENT_BINARY_DIR}/orc_proto.pb.h" + "${ORC_SOURCE_SRC_DIR}/sargs/ExpressionTree.cc" + "${ORC_SOURCE_SRC_DIR}/sargs/Literal.cc" + "${ORC_SOURCE_SRC_DIR}/sargs/PredicateLeaf.cc" + "${ORC_SOURCE_SRC_DIR}/sargs/SargsApplier.cc" + "${ORC_SOURCE_SRC_DIR}/sargs/SearchArgument.cc" + "${ORC_SOURCE_SRC_DIR}/sargs/TruthValue.cc" "${ORC_SOURCE_SRC_DIR}/Exceptions.cc" "${ORC_SOURCE_SRC_DIR}/OrcFile.cc" "${ORC_SOURCE_SRC_DIR}/Reader.cc" @@ -129,13 +136,20 @@ set(ORC_SRCS "${ORC_SOURCE_SRC_DIR}/MemoryPool.cc" "${ORC_SOURCE_SRC_DIR}/RLE.cc" "${ORC_SOURCE_SRC_DIR}/RLEv1.cc" - "${ORC_SOURCE_SRC_DIR}/RLEv2.cc" + "${ORC_SOURCE_SRC_DIR}/RleDecoderV2.cc" + "${ORC_SOURCE_SRC_DIR}/RleEncoderV2.cc" + "${ORC_SOURCE_SRC_DIR}/RLEV2Util.cc" "${ORC_SOURCE_SRC_DIR}/Statistics.cc" "${ORC_SOURCE_SRC_DIR}/StripeStream.cc" "${ORC_SOURCE_SRC_DIR}/Timezone.cc" "${ORC_SOURCE_SRC_DIR}/TypeImpl.cc" "${ORC_SOURCE_SRC_DIR}/Vector.cc" "${ORC_SOURCE_SRC_DIR}/Writer.cc" + "${ORC_SOURCE_SRC_DIR}/Adaptor.cc" + "${ORC_SOURCE_SRC_DIR}/BloomFilter.cc" + "${ORC_SOURCE_SRC_DIR}/Murmur3.cc" + "${ORC_SOURCE_SRC_DIR}/BlockBuffer.cc" + "${ORC_SOURCE_SRC_DIR}/wrap/orc-proto-wrapper.cc" "${ORC_SOURCE_SRC_DIR}/io/InputStream.cc" "${ORC_SOURCE_SRC_DIR}/io/OutputStream.cc" "${ORC_ADDITION_SOURCE_DIR}/orc_proto.pb.cc" @@ -358,6 +372,9 @@ SET(ARROW_SRCS "${LIBRARY_DIR}/util/compression_zlib.cc" ${ARROW_SRCS}) add_definitions(-DARROW_WITH_ZSTD) SET(ARROW_SRCS "${LIBRARY_DIR}/util/compression_zstd.cc" ${ARROW_SRCS}) +add_definitions(-DARROW_WITH_BROTLI) +SET(ARROW_SRCS "${LIBRARY_DIR}/util/compression_brotli.cc" ${ARROW_SRCS}) + add_library(_arrow ${ARROW_SRCS}) @@ -372,6 +389,7 @@ target_link_libraries(_arrow PRIVATE ch_contrib::snappy ch_contrib::zlib ch_contrib::zstd + ch_contrib::brotli ) target_link_libraries(_arrow PUBLIC _orc) diff --git a/contrib/orc b/contrib/orc index f9a393ed243..c5d7755ba0b 160000 --- a/contrib/orc +++ b/contrib/orc @@ -1 +1 @@ -Subproject commit f9a393ed2433a60034795284f82d093b348f2102 +Subproject commit c5d7755ba0b9a95631c8daea4d094101f26ec761 diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md index b46cc10f99d..ace5ab79bb4 100644 --- a/docs/en/development/developer-instruction.md +++ b/docs/en/development/developer-instruction.md @@ -67,7 +67,7 @@ It generally means that the SSH keys for connecting to GitHub are missing. These You can also clone the repository via https protocol: - git clone --recursive--shallow-submodules https://github.com/ClickHouse/ClickHouse.git + git clone --recursive --shallow-submodules https://github.com/ClickHouse/ClickHouse.git This, however, will not let you send your changes to the server. You can still use it temporarily and add the SSH keys later replacing the remote address of the repository with `git remote` command. diff --git a/docs/en/engines/table-engines/integrations/kafka.md b/docs/en/engines/table-engines/integrations/kafka.md index ef422632d3e..255ba06f056 100644 --- a/docs/en/engines/table-engines/integrations/kafka.md +++ b/docs/en/engines/table-engines/integrations/kafka.md @@ -19,8 +19,8 @@ Kafka lets you: ``` sql CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] ( - name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], - name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], + name1 [type1], + name2 [type2], ... ) ENGINE = Kafka() SETTINGS @@ -113,6 +113,10 @@ Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format +:::info +The Kafka table engine doesn't support columns with [default value](../../../sql-reference/statements/create/table.md#default_value). If you need columns with default value, you can add them at materialized view level (see below). +::: + ## Description {#description} The delivered messages are tracked automatically, so each message in a group is only counted once. If you want to get the data twice, then create a copy of the table with another group name. diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index ae2089f6365..db2e773a685 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1981,6 +1981,7 @@ To exchange data with Hadoop, you can use [HDFS table engine](/docs/en/engines/t - [input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference](/docs/en/operations/settings/settings-formats.md/#input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference) - allow skipping columns with unsupported types while schema inference for Parquet format. Default value - `false`. - [output_format_parquet_fixed_string_as_fixed_byte_array](/docs/en/operations/settings/settings-formats.md/#output_format_parquet_fixed_string_as_fixed_byte_array) - use Parquet FIXED_LENGTH_BYTE_ARRAY type instead of Binary/String for FixedString columns. Default value - `true`. - [output_format_parquet_version](/docs/en/operations/settings/settings-formats.md/#output_format_parquet_version) - The version of Parquet format used in output format. Default value - `2.latest`. +- [output_format_parquet_compression_method](/docs/en/operations/settings/settings-formats.md/#output_format_parquet_compression_method) - compression method used in output Parquet format. Default value - `snappy`. ## Arrow {#data-format-arrow} @@ -2051,6 +2052,7 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Arrow" > {filenam - [input_format_arrow_allow_missing_columns](/docs/en/operations/settings/settings-formats.md/#input_format_arrow_allow_missing_columns) - allow missing columns while reading Arrow data. Default value - `false`. - [input_format_arrow_skip_columns_with_unsupported_types_in_schema_inference](/docs/en/operations/settings/settings-formats.md/#input_format_arrow_skip_columns_with_unsupported_types_in_schema_inference) - allow skipping columns with unsupported types while schema inference for Arrow format. Default value - `false`. - [output_format_arrow_fixed_string_as_fixed_byte_array](/docs/en/operations/settings/settings-formats.md/#output_format_arrow_fixed_string_as_fixed_byte_array) - use Arrow FIXED_SIZE_BINARY type instead of Binary/String for FixedString columns. Default value - `true`. +- [output_format_arrow_compression_method](/docs/en/operations/settings/settings-formats.md/#output_format_arrow_compression_method) - compression method used in output Arrow format. Default value - `none`. ## ArrowStream {#data-format-arrow-stream} @@ -2107,6 +2109,7 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT ORC" > {filename. ### Arrow format settings {#parquet-format-settings} - [output_format_arrow_string_as_string](/docs/en/operations/settings/settings-formats.md/#output_format_arrow_string_as_string) - use Arrow String type instead of Binary for String columns. Default value - `false`. +- [output_format_orc_compression_method](/docs/en/operations/settings/settings-formats.md/#output_format_orc_compression_method) - compression method used in output ORC format. Default value - `none`. - [input_format_arrow_import_nested](/docs/en/operations/settings/settings-formats.md/#input_format_arrow_import_nested) - allow inserting array of structs into Nested table in Arrow input format. Default value - `false`. - [input_format_arrow_case_insensitive_column_matching](/docs/en/operations/settings/settings-formats.md/#input_format_arrow_case_insensitive_column_matching) - ignore case when matching Arrow columns with ClickHouse columns. Default value - `false`. - [input_format_arrow_allow_missing_columns](/docs/en/operations/settings/settings-formats.md/#input_format_arrow_allow_missing_columns) - allow missing columns while reading Arrow data. Default value - `false`. diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 17d03dfa4ec..3fe815bc79a 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -967,6 +967,7 @@ The maximum number of jobs that can be scheduled on the Global Thread pool. Incr Possible values: - Positive integer. +- 0 — No limit. Default value: `10000`. @@ -976,6 +977,69 @@ Default value: `10000`. 12000 ``` +## max_io_thread_pool_size {#max-io-thread-pool-size} + +ClickHouse uses threads from the IO Thread pool to do some IO operations (e.g. to interact with S3). `max_io_thread_pool_size` limits the maximum number of threads in the pool. + +Possible values: + +- Positive integer. + +Default value: `100`. + +## max_io_thread_pool_free_size {#max-io-thread-pool-free-size} + +If the number of **idle** threads in the IO Thread pool exceeds `max_io_thread_pool_free_size`, ClickHouse will release resources occupied by idling threads and decrease the pool size. Threads can be created again if necessary. + +Possible values: + +- Positive integer. + +Default value: `0`. + +## io_thread_pool_queue_size {#io-thread-pool-queue-size} + +The maximum number of jobs that can be scheduled on the IO Thread pool. + +Possible values: + +- Positive integer. +- 0 — No limit. + +Default value: `10000`. + +## max_backups_io_thread_pool_size {#max-backups-io-thread-pool-size} + +ClickHouse uses threads from the Backups IO Thread pool to do S3 backup IO operations. `max_backups_io_thread_pool_size` limits the maximum number of threads in the pool. + +Possible values: + +- Positive integer. + +Default value: `1000`. + +## max_backups_io_thread_pool_free_size {#max-backups-io-thread-pool-free-size} + +If the number of **idle** threads in the Backups IO Thread pool exceeds `max_backup_io_thread_pool_free_size`, ClickHouse will release resources occupied by idling threads and decrease the pool size. Threads can be created again if necessary. + +Possible values: + +- Positive integer. +- Zero. + +Default value: `0`. + +## backups_io_thread_pool_queue_size {#backups-io-thread-pool-queue-size} + +The maximum number of jobs that can be scheduled on the Backups IO Thread pool. It is recommended to keep this queue unlimited due to the current S3 backup logic. + +Possible values: + +- Positive integer. +- 0 — No limit. + +Default value: `0`. + ## background_pool_size {#background_pool_size} Sets the number of threads performing background merges and mutations for tables with MergeTree engines. This setting is also could be applied at server startup from the `default` profile configuration for backward compatibility at the ClickHouse server start. You can only increase the number of threads at runtime. To lower the number of threads you have to restart the server. By adjusting this setting, you manage CPU and disk load. Smaller pool size utilizes less CPU and disk resources, but background processes advance slower which might eventually impact query performance. diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md index 3fbfd340455..172627c7c3e 100644 --- a/docs/en/operations/settings/settings-formats.md +++ b/docs/en/operations/settings/settings-formats.md @@ -1014,6 +1014,12 @@ Use Arrow FIXED_SIZE_BINARY type instead of Binary/String for FixedString column Enabled by default. +### output_format_arrow_compression_method {#output_format_arrow_compression_method} + +Compression method used in output Arrow format. Supported codecs: `lz4_frame`, `zstd`, `none` (uncompressed) + +Default value: `none`. + ## ORC format settings {#orc-format-settings} ### input_format_orc_import_nested {#input_format_orc_import_nested} @@ -1057,6 +1063,12 @@ Use ORC String type instead of Binary for String columns. Disabled by default. +### output_format_orc_compression_method {#output_format_orc_compression_method} + +Compression method used in output ORC format. Supported codecs: `lz4`, `snappy`, `zlib`, `zstd`, `none` (uncompressed) + +Default value: `none`. + ## Parquet format settings {#parquet-format-settings} ### input_format_parquet_import_nested {#input_format_parquet_import_nested} @@ -1112,6 +1124,12 @@ The version of Parquet format used in output format. Supported versions: `1.0`, Default value: `2.latest`. +### output_format_parquet_compression_method {#output_format_parquet_compression_method} + +Compression method used in output Parquet format. Supported codecs: `snappy`, `lz4`, `brotli`, `zstd`, `gzip`, `none` (uncompressed) + +Default value: `snappy`. + ## Hive format settings {#hive-format-settings} ### input_format_hive_text_fields_delimiter {#input_format_hive_text_fields_delimiter} diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index f960d2df98e..94dcf159ca9 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1248,7 +1248,9 @@ Possible values: Default value: 1. :::warning -Disable this setting if you use [max_parallel_replicas](#settings-max_parallel_replicas). +Disable this setting if you use [max_parallel_replicas](#settings-max_parallel_replicas) without [parallel_replicas_custom_key](#settings-parallel_replicas_custom_key). +If [parallel_replicas_custom_key](#settings-parallel_replicas_custom_key) is set, disable this setting only if it's used on a cluster with multiple shards containing multiple replicas. +If it's used on a cluster with a single shard and multiple replicas, disabling this setting will have negative effects. ::: ## totals_mode {#totals-mode} @@ -1273,16 +1275,47 @@ Default value: `1`. **Additional Info** -This setting is useful for replicated tables with a sampling key. A query may be processed faster if it is executed on several servers in parallel. But the query performance may degrade in the following cases: +This options will produce different results depending on the settings used. + +:::warning +This setting will produce incorrect results when joins or subqueries are involved, and all tables don't meet certain requirements. See [Distributed Subqueries and max_parallel_replicas](../../sql-reference/operators/in.md/#max_parallel_replica-subqueries) for more details. +::: + +### Parallel processing using `SAMPLE` key + +A query may be processed faster if it is executed on several servers in parallel. But the query performance may degrade in the following cases: - The position of the sampling key in the partitioning key does not allow efficient range scans. - Adding a sampling key to the table makes filtering by other columns less efficient. - The sampling key is an expression that is expensive to calculate. - The cluster latency distribution has a long tail, so that querying more servers increases the query overall latency. -:::warning -This setting will produce incorrect results when joins or subqueries are involved, and all tables don't meet certain requirements. See [Distributed Subqueries and max_parallel_replicas](../../sql-reference/operators/in.md/#max_parallel_replica-subqueries) for more details. -::: +### Parallel processing using [parallel_replicas_custom_key](#settings-parallel_replicas_custom_key) + +This setting is useful for any replicated table. + +## parallel_replicas_custom_key {#settings-parallel_replicas_custom_key} + +An arbitrary integer expression that can be used to split work between replicas for a specific table. +The value can be any integer expression. +A query may be processed faster if it is executed on several servers in parallel but it depends on the used [parallel_replicas_custom_key](#settings-parallel_replicas_custom_key) +and [parallel_replicas_custom_key_filter_type](#settings-parallel_replicas_custom_key_filter_type). + +Simple expressions using primary keys are preferred. + +If the setting is used on a cluster that consists of a single shard with multiple replicas, those replicas will be converted into virtual shards. +Otherwise, it will behave same as for `SAMPLE` key, it will use multiple replicas of each shard. + +## parallel_replicas_custom_key_filter_type {#settings-parallel_replicas_custom_key_filter_type} + +How to use `parallel_replicas_custom_key` expression for splitting work between replicas. + +Possible values: + +- `default` — Use the default implementation using modulo operation on the `parallel_replicas_custom_key`. +- `range` — Split the entire value space of the expression in the ranges. This type of filtering is useful if values of `parallel_replicas_custom_key` are uniformly spread across the entire integer space, e.g. hash values. + +Default value: `default`. ## compile_expressions {#compile-expressions} diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index 845be6e04c7..f3c5b20f886 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -330,7 +330,7 @@ repeat(s, n) **Arguments** - `s` — The string to repeat. [String](../../sql-reference/data-types/string.md). -- `n` — The number of times to repeat the string. [UInt](../../sql-reference/data-types/int-uint.md). +- `n` — The number of times to repeat the string. [UInt or Int](../../sql-reference/data-types/int-uint.md). **Returned value** diff --git a/docs/en/sql-reference/operators/in.md b/docs/en/sql-reference/operators/in.md index 58119cfc4f5..0599a50c0a4 100644 --- a/docs/en/sql-reference/operators/in.md +++ b/docs/en/sql-reference/operators/in.md @@ -233,8 +233,9 @@ If `some_predicate` is not selective enough, it will return large amount of data ### Distributed Subqueries and max_parallel_replicas -When max_parallel_replicas is greater than 1, distributed queries are further transformed. For example, the following: +When [max_parallel_replicas](#settings-max_parallel_replicas) is greater than 1, distributed queries are further transformed. +For example, the following: ```sql SELECT CounterID, count() FROM distributed_table_1 WHERE UserID IN (SELECT UserID FROM local_table_2 WHERE CounterID < 100) SETTINGS max_parallel_replicas=3 @@ -247,8 +248,12 @@ SELECT CounterID, count() FROM local_table_1 WHERE UserID IN (SELECT UserID FROM SETTINGS parallel_replicas_count=3, parallel_replicas_offset=M ``` -where M is between 1 and 3 depending on which replica the local query is executing on. These settings affect every MergeTree-family table in the query and have the same effect as applying `SAMPLE 1/3 OFFSET (M-1)/3` on each table. +where M is between 1 and 3 depending on which replica the local query is executing on. -Therefore adding the max_parallel_replicas setting will only produce correct results if both tables have the same replication scheme and are sampled by UserID or a subkey of it. In particular, if local_table_2 does not have a sampling key, incorrect results will be produced. The same rule applies to JOIN. +These settings affect every MergeTree-family table in the query and have the same effect as applying `SAMPLE 1/3 OFFSET (M-1)/3` on each table. + +Therefore adding the [max_parallel_replicas](#settings-max_parallel_replicas) setting will only produce correct results if both tables have the same replication scheme and are sampled by UserID or a subkey of it. In particular, if local_table_2 does not have a sampling key, incorrect results will be produced. The same rule applies to JOIN. One workaround if local_table_2 does not meet the requirements, is to use `GLOBAL IN` or `GLOBAL JOIN`. + +If a table doesn't have a sampling key, more flexible options for [parallel_replicas_custom_key](#settings-parallel_replicas_custom_key) can be used that can produce different and more optimal behaviour. diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 54977e1b0ab..50e74920e4b 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -110,7 +110,7 @@ If the type is not `Nullable` and if `NULL` is specified, it will be treated as See also [data_type_default_nullable](../../../operations/settings/settings.md#data_type_default_nullable) setting. -## Default Values +## Default Values {#default_values} The column description can specify an expression for a default value, in one of the following ways: `DEFAULT expr`, `MATERIALIZED expr`, `ALIAS expr`. @@ -576,7 +576,7 @@ SELECT * FROM base.t1; You can add a comment to the table when you creating it. :::note -The comment is supported for all table engines except [Kafka](../../../engines/table-engines/integrations/kafka.md), [RabbitMQ](../../../engines/table-engines/integrations/rabbitmq.md) and [EmbeddedRocksDB](../../../engines/table-engines/integrations/embedded-rocksdb.md). +The comment clause is supported by all table engines except [Kafka](../../../engines/table-engines/integrations/kafka.md), [RabbitMQ](../../../engines/table-engines/integrations/rabbitmq.md) and [EmbeddedRocksDB](../../../engines/table-engines/integrations/embedded-rocksdb.md). ::: diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md index acdede3c673..0def42259ab 100644 --- a/docs/en/sql-reference/statements/create/view.md +++ b/docs/en/sql-reference/statements/create/view.md @@ -70,6 +70,12 @@ A materialized view is implemented as follows: when inserting data to the table Materialized views in ClickHouse use **column names** instead of column order during insertion into destination table. If some column names are not present in the `SELECT` query result, ClickHouse uses a default value, even if the column is not [Nullable](../../data-types/nullable.md). A safe practice would be to add aliases for every column when using Materialized views. Materialized views in ClickHouse are implemented more like insert triggers. If there’s some aggregation in the view query, it’s applied only to the batch of freshly inserted data. Any changes to existing data of source table (like update, delete, drop partition, etc.) does not change the materialized view. + +Materialized views in ClickHouse do not have deterministic behaviour in case of errors. This means that blocks that had been already written will be preserved in the destination table, but all blocks after error will not. + +By default if pushing to one of views fails, then the INSERT query will fail too, and some blocks may not be written to the destination table. This can be changed using `materialized_views_ignore_errors` setting (you should set it for `INSERT` query), if you will set `materialized_views_ignore_errors=true`, then any errors while pushing to views will be ignored and all blocks will be written to the destination table. + +Also note, that `materialized_views_ignore_errors` set to `true` by default for `system.*_log` tables. ::: If you specify `POPULATE`, the existing table data is inserted into the view when creating it, as if making a `CREATE TABLE ... AS SELECT ...` . Otherwise, the query contains only the data inserted in the table after creating the view. We **do not recommend** using `POPULATE`, since data inserted in the table during the view creation will not be inserted in it. diff --git a/programs/install/Install.cpp b/programs/install/Install.cpp index d568012bb26..80f3b0bbc63 100644 --- a/programs/install/Install.cpp +++ b/programs/install/Install.cpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Access/EnabledRoles.h b/src/Access/EnabledRoles.h index e0d773db343..5de76abe409 100644 --- a/src/Access/EnabledRoles.h +++ b/src/Access/EnabledRoles.h @@ -44,10 +44,11 @@ private: friend class RoleCache; explicit EnabledRoles(const Params & params_); - void setRolesInfo(const std::shared_ptr & info_, scope_guard * notifications); - const Params params; + /// Called by RoleCache to store `EnabledRolesInfo` in this `EnabledRoles` after the calculation is done. + void setRolesInfo(const std::shared_ptr & info_, scope_guard * notifications); + std::shared_ptr info; mutable std::mutex info_mutex; diff --git a/src/Access/RoleCache.cpp b/src/Access/RoleCache.cpp index 308b771243e..bfc6200929d 100644 --- a/src/Access/RoleCache.cpp +++ b/src/Access/RoleCache.cpp @@ -57,7 +57,9 @@ namespace RoleCache::RoleCache(const AccessControl & access_control_) - : access_control(access_control_), cache(600000 /* 10 minutes */) {} + : access_control(access_control_), cache(600000 /* 10 minutes */) +{ +} RoleCache::~RoleCache() = default; @@ -70,18 +72,18 @@ RoleCache::getEnabledRoles(const std::vector & roles, const std::vectorsecond.lock(); - if (from_cache) - return from_cache; - enabled_roles.erase(it); + if (auto enabled_roles = it->second.enabled_roles.lock()) + return enabled_roles; + enabled_roles_by_params.erase(it); } auto res = std::shared_ptr(new EnabledRoles(params)); - collectEnabledRoles(*res, nullptr); - enabled_roles.emplace(std::move(params), res); + SubscriptionsOnRoles subscriptions_on_roles; + collectEnabledRoles(*res, subscriptions_on_roles, nullptr); + enabled_roles_by_params.emplace(std::move(params), EnabledRolesWithSubscriptions{res, std::move(subscriptions_on_roles)}); return res; } @@ -90,21 +92,23 @@ void RoleCache::collectEnabledRoles(scope_guard * notifications) { /// `mutex` is already locked. - for (auto i = enabled_roles.begin(), e = enabled_roles.end(); i != e;) + for (auto i = enabled_roles_by_params.begin(), e = enabled_roles_by_params.end(); i != e;) { - auto elem = i->second.lock(); - if (!elem) - i = enabled_roles.erase(i); + auto & item = i->second; + if (auto enabled_roles = item.enabled_roles.lock()) + { + collectEnabledRoles(*enabled_roles, item.subscriptions_on_roles, notifications); + ++i; + } else { - collectEnabledRoles(*elem, notifications); - ++i; + i = enabled_roles_by_params.erase(i); } } } -void RoleCache::collectEnabledRoles(EnabledRoles & enabled, scope_guard * notifications) +void RoleCache::collectEnabledRoles(EnabledRoles & enabled_roles, SubscriptionsOnRoles & subscriptions_on_roles, scope_guard * notifications) { /// `mutex` is already locked. @@ -112,43 +116,57 @@ void RoleCache::collectEnabledRoles(EnabledRoles & enabled, scope_guard * notifi auto new_info = std::make_shared(); boost::container::flat_set skip_ids; - auto get_role_function = [this](const UUID & id) { return getRole(id); }; + /// We need to collect and keep not only enabled roles but also subscriptions for them to be able to recalculate EnabledRolesInfo when some of the roles change. + SubscriptionsOnRoles new_subscriptions_on_roles; + new_subscriptions_on_roles.reserve(subscriptions_on_roles.size()); - for (const auto & current_role : enabled.params.current_roles) + auto get_role_function = [this, &subscriptions_on_roles](const UUID & id) TSA_NO_THREAD_SAFETY_ANALYSIS { return getRole(id, subscriptions_on_roles); }; + + for (const auto & current_role : enabled_roles.params.current_roles) collectRoles(*new_info, skip_ids, get_role_function, current_role, true, false); - for (const auto & current_role : enabled.params.current_roles_with_admin_option) + for (const auto & current_role : enabled_roles.params.current_roles_with_admin_option) collectRoles(*new_info, skip_ids, get_role_function, current_role, true, true); + /// Remove duplicates from `subscriptions_on_roles`. + std::sort(new_subscriptions_on_roles.begin(), new_subscriptions_on_roles.end()); + new_subscriptions_on_roles.erase(std::unique(new_subscriptions_on_roles.begin(), new_subscriptions_on_roles.end()), new_subscriptions_on_roles.end()); + subscriptions_on_roles = std::move(new_subscriptions_on_roles); + /// Collect data from the collected roles. - enabled.setRolesInfo(new_info, notifications); + enabled_roles.setRolesInfo(new_info, notifications); } -RolePtr RoleCache::getRole(const UUID & role_id) +RolePtr RoleCache::getRole(const UUID & role_id, SubscriptionsOnRoles & subscriptions_on_roles) { /// `mutex` is already locked. auto role_from_cache = cache.get(role_id); if (role_from_cache) + { + subscriptions_on_roles.emplace_back(role_from_cache->second); return role_from_cache->first; + } - auto subscription = access_control.subscribeForChanges(role_id, - [this, role_id](const UUID &, const AccessEntityPtr & entity) + auto on_role_changed_or_removed = [this, role_id](const UUID &, const AccessEntityPtr & entity) { auto changed_role = entity ? typeid_cast(entity) : nullptr; if (changed_role) roleChanged(role_id, changed_role); else roleRemoved(role_id); - }); + }; + + auto subscription_on_role = std::make_shared(access_control.subscribeForChanges(role_id, on_role_changed_or_removed)); auto role = access_control.tryRead(role_id); if (role) { - auto cache_value = Poco::SharedPtr>( - new std::pair{role, std::move(subscription)}); + auto cache_value = Poco::SharedPtr>>( + new std::pair>{role, subscription_on_role}); cache.add(role_id, cache_value); + subscriptions_on_roles.emplace_back(subscription_on_role); return role; } @@ -162,12 +180,17 @@ void RoleCache::roleChanged(const UUID & role_id, const RolePtr & changed_role) scope_guard notifications; std::lock_guard lock{mutex}; + auto role_from_cache = cache.get(role_id); - if (!role_from_cache) - return; - role_from_cache->first = changed_role; - cache.update(role_id, role_from_cache); - collectEnabledRoles(¬ifications); + if (role_from_cache) + { + /// We update the role stored in a cache entry only if that entry has not expired yet. + role_from_cache->first = changed_role; + cache.update(role_id, role_from_cache); + } + + /// An enabled role for some users has been changed, we need to recalculate the access rights. + collectEnabledRoles(¬ifications); /// collectEnabledRoles() must be called with the `mutex` locked. } @@ -177,8 +200,12 @@ void RoleCache::roleRemoved(const UUID & role_id) scope_guard notifications; std::lock_guard lock{mutex}; + + /// If a cache entry with the role has expired already, that remove() will do nothing. cache.remove(role_id); - collectEnabledRoles(¬ifications); + + /// An enabled role for some users has been removed, we need to recalculate the access rights. + collectEnabledRoles(¬ifications); /// collectEnabledRoles() must be called with the `mutex` locked. } } diff --git a/src/Access/RoleCache.h b/src/Access/RoleCache.h index 51c415d4d1d..24f19cb9d94 100644 --- a/src/Access/RoleCache.h +++ b/src/Access/RoleCache.h @@ -24,15 +24,29 @@ public: const std::vector & current_roles_with_admin_option); private: - void collectEnabledRoles(scope_guard * notifications); - void collectEnabledRoles(EnabledRoles & enabled, scope_guard * notifications); - RolePtr getRole(const UUID & role_id); + using SubscriptionsOnRoles = std::vector>; + + void collectEnabledRoles(scope_guard * notifications) TSA_REQUIRES(mutex); + void collectEnabledRoles(EnabledRoles & enabled_roles, SubscriptionsOnRoles & subscriptions_on_roles, scope_guard * notifications) TSA_REQUIRES(mutex); + RolePtr getRole(const UUID & role_id, SubscriptionsOnRoles & subscriptions_on_roles) TSA_REQUIRES(mutex); void roleChanged(const UUID & role_id, const RolePtr & changed_role); void roleRemoved(const UUID & role_id); const AccessControl & access_control; - Poco::AccessExpireCache> cache; - std::map> enabled_roles; + + Poco::AccessExpireCache>> TSA_GUARDED_BY(mutex) cache; + + struct EnabledRolesWithSubscriptions + { + std::weak_ptr enabled_roles; + + /// We need to keep subscriptions for all enabled roles to be able to recalculate EnabledRolesInfo when some of the roles change. + /// `cache` also keeps subscriptions but that's not enough because values can be purged from the `cache` anytime. + SubscriptionsOnRoles subscriptions_on_roles; + }; + + std::map TSA_GUARDED_BY(mutex) enabled_roles_by_params; + mutable std::mutex mutex; }; diff --git a/src/Analyzer/ColumnNode.h b/src/Analyzer/ColumnNode.h index e378bc5f3d0..79c0e23c86f 100644 --- a/src/Analyzer/ColumnNode.h +++ b/src/Analyzer/ColumnNode.h @@ -3,6 +3,7 @@ #include #include +#include namespace DB { @@ -117,6 +118,11 @@ public: return column.type; } + void convertToNullable() override + { + column.type = makeNullableSafe(column.type); + } + void dumpTreeImpl(WriteBuffer & buffer, FormatState & state, size_t indent) const override; protected: diff --git a/src/Analyzer/FunctionNode.cpp b/src/Analyzer/FunctionNode.cpp index 7961bfbae31..718dcf4bb58 100644 --- a/src/Analyzer/FunctionNode.cpp +++ b/src/Analyzer/FunctionNode.cpp @@ -99,7 +99,7 @@ void FunctionNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state buffer << ", function_type: " << function_type; if (function) - buffer << ", result_type: " + function->getResultType()->getName(); + buffer << ", result_type: " + getResultType()->getName(); const auto & parameters = getParameters(); if (!parameters.getNodes().empty()) @@ -177,6 +177,7 @@ QueryTreeNodePtr FunctionNode::cloneImpl() const */ result_function->function = function; result_function->kind = kind; + result_function->wrap_with_nullable = wrap_with_nullable; return result_function; } diff --git a/src/Analyzer/FunctionNode.h b/src/Analyzer/FunctionNode.h index 6819232b4be..89a684c1d0f 100644 --- a/src/Analyzer/FunctionNode.h +++ b/src/Analyzer/FunctionNode.h @@ -8,6 +8,7 @@ #include #include #include +#include #include namespace DB @@ -187,7 +188,16 @@ public: throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Function node with name '{}' is not resolved", function_name); - return function->getResultType(); + auto type = function->getResultType(); + if (wrap_with_nullable) + return makeNullableSafe(type); + return type; + } + + void convertToNullable() override + { + chassert(kind == FunctionKind::ORDINARY); + wrap_with_nullable = true; } void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; @@ -205,6 +215,7 @@ private: String function_name; FunctionKind kind = FunctionKind::UNKNOWN; IResolvedFunctionPtr function; + bool wrap_with_nullable = false; static constexpr size_t parameters_child_index = 0; static constexpr size_t arguments_child_index = 1; diff --git a/src/Analyzer/IQueryTreeNode.h b/src/Analyzer/IQueryTreeNode.h index 157bbd1b951..e344dd66fbc 100644 --- a/src/Analyzer/IQueryTreeNode.h +++ b/src/Analyzer/IQueryTreeNode.h @@ -90,6 +90,11 @@ public: throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Method getResultType is not supported for {} query node", getNodeTypeName()); } + virtual void convertToNullable() + { + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Method convertToNullable is not supported for {} query node", getNodeTypeName()); + } + struct CompareOptions { bool compare_aliases = true; diff --git a/src/Analyzer/Passes/ArrayExistsToHasPass.cpp b/src/Analyzer/Passes/ArrayExistsToHasPass.cpp index b4b8b5b4579..c0f958588f1 100644 --- a/src/Analyzer/Passes/ArrayExistsToHasPass.cpp +++ b/src/Analyzer/Passes/ArrayExistsToHasPass.cpp @@ -1,3 +1,5 @@ +#include + #include #include @@ -8,71 +10,85 @@ #include #include -#include "ArrayExistsToHasPass.h" - namespace DB { + namespace { - class RewriteArrayExistsToHasVisitor : public InDepthQueryTreeVisitorWithContext + +class RewriteArrayExistsToHasVisitor : public InDepthQueryTreeVisitorWithContext +{ +public: + using Base = InDepthQueryTreeVisitorWithContext; + using Base::Base; + + void visitImpl(QueryTreeNodePtr & node) { - public: - using Base = InDepthQueryTreeVisitorWithContext; - using Base::Base; + if (!getSettings().optimize_rewrite_array_exists_to_has) + return; - void visitImpl(QueryTreeNodePtr & node) + auto * array_exists_function_node = node->as(); + if (!array_exists_function_node || array_exists_function_node->getFunctionName() != "arrayExists") + return; + + auto & array_exists_function_arguments_nodes = array_exists_function_node->getArguments().getNodes(); + if (array_exists_function_arguments_nodes.size() != 2) + return; + + /// lambda function must be like: x -> x = elem + auto * lambda_node = array_exists_function_arguments_nodes[0]->as(); + if (!lambda_node) + return; + + auto & lambda_arguments_nodes = lambda_node->getArguments().getNodes(); + if (lambda_arguments_nodes.size() != 1) + return; + + const auto & lambda_argument_column_node = lambda_arguments_nodes[0]; + if (lambda_argument_column_node->getNodeType() != QueryTreeNodeType::COLUMN) + return; + + auto * filter_node = lambda_node->getExpression()->as(); + if (!filter_node || filter_node->getFunctionName() != "equals") + return; + + const auto & filter_arguments_nodes = filter_node->getArguments().getNodes(); + if (filter_arguments_nodes.size() != 2) + return; + + const auto & filter_lhs_argument_node = filter_arguments_nodes[0]; + auto filter_lhs_argument_node_type = filter_lhs_argument_node->getNodeType(); + + const auto & filter_rhs_argument_node = filter_arguments_nodes[1]; + auto filter_rhs_argument_node_type = filter_rhs_argument_node->getNodeType(); + + QueryTreeNodePtr has_constant_element_argument; + + if (filter_lhs_argument_node_type == QueryTreeNodeType::COLUMN && + filter_rhs_argument_node_type == QueryTreeNodeType::CONSTANT && + filter_lhs_argument_node->isEqual(*lambda_argument_column_node)) { - if (!getSettings().optimize_rewrite_array_exists_to_has) - return; - - auto * function_node = node->as(); - if (!function_node || function_node->getFunctionName() != "arrayExists") - return; - - auto & function_arguments_nodes = function_node->getArguments().getNodes(); - if (function_arguments_nodes.size() != 2) - return; - - /// lambda function must be like: x -> x = elem - auto * lambda_node = function_arguments_nodes[0]->as(); - if (!lambda_node) - return; - - auto & lambda_arguments_nodes = lambda_node->getArguments().getNodes(); - if (lambda_arguments_nodes.size() != 1) - return; - auto * column_node = lambda_arguments_nodes[0]->as(); - - auto * filter_node = lambda_node->getExpression()->as(); - if (!filter_node || filter_node->getFunctionName() != "equals") - return; - - auto filter_arguments_nodes = filter_node->getArguments().getNodes(); - if (filter_arguments_nodes.size() != 2) - return; - - ColumnNode * filter_column_node = nullptr; - if (filter_arguments_nodes[1]->as() && (filter_column_node = filter_arguments_nodes[0]->as()) - && filter_column_node->getColumnName() == column_node->getColumnName()) - { - /// Rewrite arrayExists(x -> x = elem, arr) -> has(arr, elem) - function_arguments_nodes[0] = std::move(function_arguments_nodes[1]); - function_arguments_nodes[1] = std::move(filter_arguments_nodes[1]); - function_node->resolveAsFunction( - FunctionFactory::instance().get("has", getContext())->build(function_node->getArgumentColumns())); - } - else if ( - filter_arguments_nodes[0]->as() && (filter_column_node = filter_arguments_nodes[1]->as()) - && filter_column_node->getColumnName() == column_node->getColumnName()) - { - /// Rewrite arrayExists(x -> elem = x, arr) -> has(arr, elem) - function_arguments_nodes[0] = std::move(function_arguments_nodes[1]); - function_arguments_nodes[1] = std::move(filter_arguments_nodes[0]); - function_node->resolveAsFunction( - FunctionFactory::instance().get("has", getContext())->build(function_node->getArgumentColumns())); - } + /// Rewrite arrayExists(x -> x = elem, arr) -> has(arr, elem) + has_constant_element_argument = filter_rhs_argument_node; } - }; + else if (filter_lhs_argument_node_type == QueryTreeNodeType::CONSTANT && + filter_rhs_argument_node_type == QueryTreeNodeType::COLUMN && + filter_rhs_argument_node->isEqual(*lambda_argument_column_node)) + { + /// Rewrite arrayExists(x -> elem = x, arr) -> has(arr, elem) + has_constant_element_argument = filter_lhs_argument_node; + } + else + { + return; + } + + auto has_function = FunctionFactory::instance().get("has", getContext()); + array_exists_function_arguments_nodes[0] = std::move(array_exists_function_arguments_nodes[1]); + array_exists_function_arguments_nodes[1] = std::move(has_constant_element_argument); + array_exists_function_node->resolveAsFunction(has_function->build(array_exists_function_node->getArgumentColumns())); + } +}; } diff --git a/src/Analyzer/Passes/ArrayExistsToHasPass.h b/src/Analyzer/Passes/ArrayExistsToHasPass.h index 7d9d1cf3d68..8f4623116e3 100644 --- a/src/Analyzer/Passes/ArrayExistsToHasPass.h +++ b/src/Analyzer/Passes/ArrayExistsToHasPass.h @@ -4,8 +4,15 @@ namespace DB { -/// Rewrite possible 'arrayExists(func, arr)' to 'has(arr, elem)' to improve performance -/// arrayExists(x -> x = 1, arr) -> has(arr, 1) + +/** Rewrite possible 'arrayExists(func, arr)' to 'has(arr, elem)' to improve performance. + * + * Example: SELECT arrayExists(x -> x = 1, arr); + * Result: SELECT has(arr, 1); + * + * Example: SELECT arrayExists(x -> 1 = x, arr); + * Result: SELECT has(arr, 1); + */ class RewriteArrayExistsToHasPass final : public IQueryTreePass { public: @@ -15,4 +22,5 @@ public: void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; }; + } diff --git a/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp b/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp index fdf818681d7..fa5fc0e75a8 100644 --- a/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp +++ b/src/Analyzer/Passes/AutoFinalOnQueryPass.cpp @@ -22,8 +22,7 @@ public: void visitImpl(QueryTreeNodePtr & node) { - const auto & context = getContext(); - if (!context->getSettingsRef().final) + if (!getSettings().final) return; const auto * query_node = node->as(); diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizer.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizer.cpp new file mode 100644 index 00000000000..73585a4cd23 --- /dev/null +++ b/src/Analyzer/Passes/LogicalExpressionOptimizer.cpp @@ -0,0 +1,237 @@ +#include + +#include + +#include +#include +#include +#include + +#include + +namespace DB +{ + +class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithContext +{ +public: + using Base = InDepthQueryTreeVisitorWithContext; + + explicit LogicalExpressionOptimizerVisitor(ContextPtr context) + : Base(std::move(context)) + {} + + void visitImpl(QueryTreeNodePtr & node) + { + auto * function_node = node->as(); + + if (!function_node) + return; + + if (function_node->getFunctionName() == "or") + { + tryReplaceOrEqualsChainWithIn(node); + return; + } + + if (function_node->getFunctionName() == "and") + { + tryReplaceAndEqualsChainsWithConstant(node); + return; + } + } +private: + void tryReplaceAndEqualsChainsWithConstant(QueryTreeNodePtr & node) + { + auto & function_node = node->as(); + assert(function_node.getFunctionName() == "and"); + + if (function_node.getResultType()->isNullable()) + return; + + QueryTreeNodes and_operands; + + QueryTreeNodePtrWithHashMap node_to_constants; + + for (const auto & argument : function_node.getArguments()) + { + auto * argument_function = argument->as(); + if (!argument_function || argument_function->getFunctionName() != "equals") + { + and_operands.push_back(argument); + continue; + } + + const auto & equals_arguments = argument_function->getArguments().getNodes(); + const auto & lhs = equals_arguments[0]; + const auto & rhs = equals_arguments[1]; + + const auto has_and_with_different_constant = [&](const QueryTreeNodePtr & expression, const ConstantNode * constant) + { + if (auto it = node_to_constants.find(expression); it != node_to_constants.end()) + { + if (!it->second->isEqual(*constant)) + return true; + } + else + { + node_to_constants.emplace(expression, constant); + and_operands.push_back(argument); + } + + return false; + }; + + bool collapse_to_false = false; + + if (const auto * lhs_literal = lhs->as()) + collapse_to_false = has_and_with_different_constant(rhs, lhs_literal); + else if (const auto * rhs_literal = rhs->as()) + collapse_to_false = has_and_with_different_constant(lhs, rhs_literal); + else + and_operands.push_back(argument); + + if (collapse_to_false) + { + auto false_value = std::make_shared(0u, function_node.getResultType()); + auto false_node = std::make_shared(std::move(false_value)); + node = std::move(false_node); + return; + } + } + + if (and_operands.size() == 1) + { + /// AND operator can have UInt8 or bool as its type. + /// bool is used if a bool constant is at least one operand. + /// Because we reduce the number of operands here by eliminating the same equality checks, + /// the only situation we can end up here is we had AND check where all the equality checks are the same so we know the type is UInt8. + /// Otherwise, we will have > 1 operands and we don't have to do anything. + assert(!function_node.getResultType()->isNullable() && and_operands[0]->getResultType()->equals(*function_node.getResultType())); + node = std::move(and_operands[0]); + return; + } + + auto and_function_resolver = FunctionFactory::instance().get("and", getContext()); + function_node.getArguments().getNodes() = std::move(and_operands); + function_node.resolveAsFunction(and_function_resolver); + } + + void tryReplaceOrEqualsChainWithIn(QueryTreeNodePtr & node) + { + auto & function_node = node->as(); + assert(function_node.getFunctionName() == "or"); + + QueryTreeNodes or_operands; + + QueryTreeNodePtrWithHashMap node_to_equals_functions; + QueryTreeNodePtrWithHashMap node_to_constants; + + for (const auto & argument : function_node.getArguments()) + { + auto * argument_function = argument->as(); + if (!argument_function || argument_function->getFunctionName() != "equals") + { + or_operands.push_back(argument); + continue; + } + + /// collect all equality checks (x = value) + + const auto & equals_arguments = argument_function->getArguments().getNodes(); + const auto & lhs = equals_arguments[0]; + const auto & rhs = equals_arguments[1]; + + const auto add_equals_function_if_not_present = [&](const auto & expression_node, const ConstantNode * constant) + { + auto & constant_set = node_to_constants[expression_node]; + if (!constant_set.contains(constant)) + { + constant_set.insert(constant); + node_to_equals_functions[expression_node].push_back(argument); + } + }; + + if (const auto * lhs_literal = lhs->as()) + add_equals_function_if_not_present(rhs, lhs_literal); + else if (const auto * rhs_literal = rhs->as()) + add_equals_function_if_not_present(lhs, rhs_literal); + else + or_operands.push_back(argument); + } + + auto in_function_resolver = FunctionFactory::instance().get("in", getContext()); + + for (auto & [expression, equals_functions] : node_to_equals_functions) + { + const auto & settings = getSettings(); + if (equals_functions.size() < settings.optimize_min_equality_disjunction_chain_length && !expression.node->getResultType()->lowCardinality()) + { + std::move(equals_functions.begin(), equals_functions.end(), std::back_inserter(or_operands)); + continue; + } + + Tuple args; + args.reserve(equals_functions.size()); + /// first we create tuple from RHS of equals functions + for (const auto & equals : equals_functions) + { + const auto * equals_function = equals->as(); + assert(equals_function && equals_function->getFunctionName() == "equals"); + + const auto & equals_arguments = equals_function->getArguments().getNodes(); + if (const auto * rhs_literal = equals_arguments[1]->as()) + { + args.push_back(rhs_literal->getValue()); + } + else + { + const auto * lhs_literal = equals_arguments[0]->as(); + assert(lhs_literal); + args.push_back(lhs_literal->getValue()); + } + } + + auto rhs_node = std::make_shared(std::move(args)); + + auto in_function = std::make_shared("in"); + + QueryTreeNodes in_arguments; + in_arguments.reserve(2); + in_arguments.push_back(expression.node); + in_arguments.push_back(std::move(rhs_node)); + + in_function->getArguments().getNodes() = std::move(in_arguments); + in_function->resolveAsFunction(in_function_resolver); + + or_operands.push_back(std::move(in_function)); + } + + if (or_operands.size() == 1) + { + /// if the result type of operand is the same as the result type of OR + /// we can replace OR with the operand + if (or_operands[0]->getResultType()->equals(*function_node.getResultType())) + { + assert(!function_node.getResultType()->isNullable()); + node = std::move(or_operands[0]); + return; + } + + /// otherwise add a stub 0 to make OR correct + or_operands.push_back(std::make_shared(static_cast(0))); + } + + auto or_function_resolver = FunctionFactory::instance().get("or", getContext()); + function_node.getArguments().getNodes() = std::move(or_operands); + function_node.resolveAsFunction(or_function_resolver); + } +}; + +void LogicalExpressionOptimizerPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) +{ + LogicalExpressionOptimizerVisitor visitor(std::move(context)); + visitor.visit(query_tree_node); +} + +} diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h new file mode 100644 index 00000000000..05c10ddc685 --- /dev/null +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h @@ -0,0 +1,82 @@ +#pragma once + +#include + +namespace DB +{ + +/** + * This pass tries to do optimizations on logical expression: + * + * 1. Replaces chains of equality functions inside an OR with a single IN operator. + * The replacement is done if: + * - one of the operands of the equality function is a constant + * - length of chain is at least 'optimize_min_equality_disjunction_chain_length' long OR the expression has type of LowCardinality + * + * E.g. (optimize_min_equality_disjunction_chain_length = 2) + * ------------------------------- + * SELECT * + * FROM table + * WHERE a = 1 OR b = 'test' OR a = 2; + * + * will be transformed into + * + * SELECT * + * FROM TABLE + * WHERE b = 'test' OR a IN (1, 2); + * ------------------------------- + * + * 2. Removes duplicate OR checks + * ------------------------------- + * SELECT * + * FROM table + * WHERE a = 1 OR b = 'test' OR a = 1; + * + * will be transformed into + * + * SELECT * + * FROM TABLE + * WHERE a = 1 OR b = 'test'; + * ------------------------------- + * + * 3. Replaces AND chains with a single constant. + * The replacement is done if: + * - one of the operands of the equality function is a constant + * - constants are different for same expression + * ------------------------------- + * SELECT * + * FROM table + * WHERE a = 1 AND b = 'test' AND a = 2; + * + * will be transformed into + * + * SELECT * + * FROM TABLE + * WHERE 0; + * ------------------------------- + * + * 4. Removes duplicate AND checks + * ------------------------------- + * SELECT * + * FROM table + * WHERE a = 1 AND b = 'test' AND a = 1; + * + * will be transformed into + * + * SELECT * + * FROM TABLE + * WHERE a = 1 AND b = 'test'; + * ------------------------------- + */ + +class LogicalExpressionOptimizerPass final : public IQueryTreePass +{ +public: + String getName() override { return "LogicalExpressionOptimizer"; } + + String getDescription() override { return "Transform equality chain to a single IN function or a constant if possible"; } + + void run(QueryTreeNodePtr query_tree_node, ContextPtr context) override; +}; + +} diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index f8645f2b756..34c03a9ffb6 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -199,7 +199,6 @@ namespace ErrorCodes * TODO: SELECT (compound_expression).*, (compound_expression).COLUMNS are not supported on parser level. * TODO: SELECT a.b.c.*, a.b.c.COLUMNS. Qualified matcher where identifier size is greater than 2 are not supported on parser level. * TODO: Support function identifier resolve from parent query scope, if lambda in parent scope does not capture any columns. - * TODO: Support group_by_use_nulls. * TODO: Scalar subqueries cache. */ @@ -472,6 +471,12 @@ public: alias_name_to_expressions[node_alias].push_back(node); } + if (const auto * function = node->as()) + { + if (AggregateFunctionFactory::instance().isAggregateFunctionName(function->getFunctionName())) + ++aggregate_functions_counter; + } + expressions.emplace_back(node); } @@ -490,6 +495,12 @@ public: alias_name_to_expressions.erase(it); } + if (const auto * function = top_expression->as()) + { + if (AggregateFunctionFactory::instance().isAggregateFunctionName(function->getFunctionName())) + --aggregate_functions_counter; + } + expressions.pop_back(); } @@ -508,6 +519,11 @@ public: return alias_name_to_expressions.contains(alias); } + bool hasAggregateFunction() const + { + return aggregate_functions_counter > 0; + } + QueryTreeNodePtr getExpressionWithAlias(const std::string & alias) const { auto expression_it = alias_name_to_expressions.find(alias); @@ -554,6 +570,7 @@ public: private: QueryTreeNodes expressions; + size_t aggregate_functions_counter = 0; std::unordered_map alias_name_to_expressions; }; @@ -686,7 +703,11 @@ struct IdentifierResolveScope if (auto * union_node = scope_node->as()) context = union_node->getContext(); else if (auto * query_node = scope_node->as()) + { context = query_node->getContext(); + group_by_use_nulls = context->getSettingsRef().group_by_use_nulls && + (query_node->isGroupByWithGroupingSets() || query_node->isGroupByWithRollup() || query_node->isGroupByWithCube()); + } } QueryTreeNodePtr scope_node; @@ -734,9 +755,14 @@ struct IdentifierResolveScope /// Table expression node to data std::unordered_map table_expression_node_to_data; + QueryTreeNodePtrWithHashSet nullable_group_by_keys; + /// Use identifier lookup to result cache bool use_identifier_lookup_to_result_cache = true; + /// Apply nullability to aggregation keys + bool group_by_use_nulls = false; + /// JOINs count size_t joins_count = 0; @@ -5407,10 +5433,18 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id } } + if (node + && scope.nullable_group_by_keys.contains(node) + && !scope.expressions_in_resolve_process_stack.hasAggregateFunction()) + { + node = node->clone(); + node->convertToNullable(); + } + /** Update aliases after expression node was resolved. * Do not update node in alias table if we resolve it for duplicate alias. */ - if (!node_alias.empty() && use_alias_table) + if (!node_alias.empty() && use_alias_table && !scope.group_by_use_nulls) { auto it = scope.alias_name_to_expression_node.find(node_alias); if (it != scope.alias_name_to_expression_node.end()) @@ -6418,9 +6452,6 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier auto & query_node_typed = query_node->as(); const auto & settings = scope.context->getSettingsRef(); - if (settings.group_by_use_nulls) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "GROUP BY use nulls is not supported"); - bool is_rollup_or_cube = query_node_typed.isGroupByWithRollup() || query_node_typed.isGroupByWithCube(); if (query_node_typed.isGroupByWithGroupingSets() && query_node_typed.isGroupByWithTotals()) @@ -6556,16 +6587,11 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier resolveQueryJoinTreeNode(query_node_typed.getJoinTree(), scope, visitor); } - scope.use_identifier_lookup_to_result_cache = true; + if (!scope.group_by_use_nulls) + scope.use_identifier_lookup_to_result_cache = true; /// Resolve query node sections. - auto projection_columns = resolveProjectionExpressionNodeList(query_node_typed.getProjectionNode(), scope); - if (query_node_typed.getProjection().getNodes().empty()) - throw Exception(ErrorCodes::EMPTY_LIST_OF_COLUMNS_QUERIED, - "Empty list of columns in projection. In scope {}", - scope.scope_node->formatASTForErrorMessage()); - if (query_node_typed.hasWith()) resolveExpressionNodeList(query_node_typed.getWithNode(), scope, true /*allow_lambda_expression*/, false /*allow_table_expression*/); @@ -6586,6 +6612,15 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier resolveExpressionNodeList(grouping_sets_keys_list_node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); } + + if (scope.group_by_use_nulls) + { + for (const auto & grouping_set : query_node_typed.getGroupBy().getNodes()) + { + for (const auto & group_by_elem : grouping_set->as()->getNodes()) + scope.nullable_group_by_keys.insert(group_by_elem); + } + } } else { @@ -6593,6 +6628,12 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier replaceNodesWithPositionalArguments(query_node_typed.getGroupByNode(), query_node_typed.getProjection().getNodes(), scope); resolveExpressionNodeList(query_node_typed.getGroupByNode(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + + if (scope.group_by_use_nulls) + { + for (const auto & group_by_elem : query_node_typed.getGroupBy().getNodes()) + scope.nullable_group_by_keys.insert(group_by_elem); + } } } @@ -6645,6 +6686,12 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier convertLimitOffsetExpression(query_node_typed.getOffset(), "OFFSET", scope); } + auto projection_columns = resolveProjectionExpressionNodeList(query_node_typed.getProjectionNode(), scope); + if (query_node_typed.getProjection().getNodes().empty()) + throw Exception(ErrorCodes::EMPTY_LIST_OF_COLUMNS_QUERIED, + "Empty list of columns in projection. In scope {}", + scope.scope_node->formatASTForErrorMessage()); + /** Resolve nodes with duplicate aliases. * Table expressions cannot have duplicate aliases. * @@ -6708,7 +6755,7 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier "ARRAY JOIN", "in PREWHERE"); - validateAggregates(query_node); + validateAggregates(query_node, { .group_by_use_nulls = scope.group_by_use_nulls }); /** WITH section can be safely removed, because WITH section only can provide aliases to query expressions * and CTE for other sections to use. diff --git a/src/Analyzer/Passes/ShardNumColumnToFunctionPass.h b/src/Analyzer/Passes/ShardNumColumnToFunctionPass.h index 83b974954fa..71a038bcf39 100644 --- a/src/Analyzer/Passes/ShardNumColumnToFunctionPass.h +++ b/src/Analyzer/Passes/ShardNumColumnToFunctionPass.h @@ -6,6 +6,9 @@ namespace DB { /** Rewrite _shard_num column into shardNum() function. + * + * Example: SELECT _shard_num FROM distributed_table; + * Result: SELECT shardNum() FROM distributed_table; */ class ShardNumColumnToFunctionPass final : public IQueryTreePass { diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp index 2cecf4f81a2..9c0f2381c31 100644 --- a/src/Analyzer/QueryTreePassManager.cpp +++ b/src/Analyzer/QueryTreePassManager.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -147,7 +148,6 @@ private: /** ClickHouse query tree pass manager. * - * TODO: Support logical expressions optimizer. * TODO: Support setting convert_query_to_cnf. * TODO: Support setting optimize_using_constraints. * TODO: Support setting optimize_substitute_columns. @@ -262,6 +262,8 @@ void addQueryTreePasses(QueryTreePassManager & manager) manager.addPass(std::make_unique()); + manager.addPass(std::make_unique()); + manager.addPass(std::make_unique()); manager.addPass(std::make_unique()); manager.addPass(std::make_unique()); diff --git a/src/Analyzer/ValidationUtils.cpp b/src/Analyzer/ValidationUtils.cpp index feb2f8a890b..8ccecc9769c 100644 --- a/src/Analyzer/ValidationUtils.cpp +++ b/src/Analyzer/ValidationUtils.cpp @@ -105,7 +105,7 @@ private: const QueryTreeNodePtr & query_node; }; -void validateAggregates(const QueryTreeNodePtr & query_node) +void validateAggregates(const QueryTreeNodePtr & query_node, ValidationParams params) { const auto & query_node_typed = query_node->as(); auto join_tree_node_type = query_node_typed.getJoinTree()->getNodeType(); @@ -182,7 +182,9 @@ void validateAggregates(const QueryTreeNodePtr & query_node) if (grouping_set_key->as()) continue; - group_by_keys_nodes.push_back(grouping_set_key); + group_by_keys_nodes.push_back(grouping_set_key->clone()); + if (params.group_by_use_nulls) + group_by_keys_nodes.back()->convertToNullable(); } } else @@ -190,7 +192,9 @@ void validateAggregates(const QueryTreeNodePtr & query_node) if (node->as()) continue; - group_by_keys_nodes.push_back(node); + group_by_keys_nodes.push_back(node->clone()); + if (params.group_by_use_nulls) + group_by_keys_nodes.back()->convertToNullable(); } } diff --git a/src/Analyzer/ValidationUtils.h b/src/Analyzer/ValidationUtils.h index b511dc9514b..b8ba6b8cc10 100644 --- a/src/Analyzer/ValidationUtils.h +++ b/src/Analyzer/ValidationUtils.h @@ -5,6 +5,11 @@ namespace DB { +struct ValidationParams +{ + bool group_by_use_nulls; +}; + /** Validate aggregates in query node. * * 1. Check that there are no aggregate functions and GROUPING function in JOIN TREE, WHERE, PREWHERE, in another aggregate functions. @@ -15,7 +20,7 @@ namespace DB * PROJECTION. * 5. Throws exception if there is GROUPING SETS or ROLLUP or CUBE or WITH TOTALS without aggregation. */ -void validateAggregates(const QueryTreeNodePtr & query_node); +void validateAggregates(const QueryTreeNodePtr & query_node, ValidationParams params); /** Assert that there are no function nodes with specified function name in node children. * Do not visit subqueries. diff --git a/src/Backups/BackupCoordinationRemote.cpp b/src/Backups/BackupCoordinationRemote.cpp index c0fb4d5e066..8e43676f59c 100644 --- a/src/Backups/BackupCoordinationRemote.cpp +++ b/src/Backups/BackupCoordinationRemote.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index b5f48a1a277..fb8abee814a 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Backups/IBackupCoordination.h b/src/Backups/IBackupCoordination.h index b75d856b50f..588a20d9eeb 100644 --- a/src/Backups/IBackupCoordination.h +++ b/src/Backups/IBackupCoordination.h @@ -2,7 +2,7 @@ #include #include -#include +#include #include diff --git a/src/Columns/tests/gtest_weak_hash_32.cpp b/src/Columns/tests/gtest_weak_hash_32.cpp index 5755cc3af72..cbf47790b9f 100644 --- a/src/Columns/tests/gtest_weak_hash_32.cpp +++ b/src/Columns/tests/gtest_weak_hash_32.cpp @@ -14,7 +14,7 @@ #include #include -#include +#include #include #include diff --git a/src/Common/BinStringDecodeHelper.h b/src/Common/BinStringDecodeHelper.h index 513a4196b6f..df3e014cfad 100644 --- a/src/Common/BinStringDecodeHelper.h +++ b/src/Common/BinStringDecodeHelper.h @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace DB { diff --git a/src/Common/OpenTelemetryTraceContext.cpp b/src/Common/OpenTelemetryTraceContext.cpp index b62822ceda2..df4ee6a34bf 100644 --- a/src/Common/OpenTelemetryTraceContext.cpp +++ b/src/Common/OpenTelemetryTraceContext.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Common/SymbolIndex.cpp b/src/Common/SymbolIndex.cpp index 6f31009b1d2..f1cace5017c 100644 --- a/src/Common/SymbolIndex.cpp +++ b/src/Common/SymbolIndex.cpp @@ -1,7 +1,7 @@ #if defined(__ELF__) && !defined(OS_FREEBSD) #include -#include +#include #include #include diff --git a/src/Common/escapeForFileName.cpp b/src/Common/escapeForFileName.cpp index bcca04706dc..a1f9bff28d0 100644 --- a/src/Common/escapeForFileName.cpp +++ b/src/Common/escapeForFileName.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index 6e1b5573bef..eabc7bdafbb 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -383,6 +383,14 @@ bool isSymlink(const fs::path & path) return fs::is_symlink(path); /// STYLE_CHECK_ALLOW_STD_FS_SYMLINK } +bool isSymlinkNoThrow(const fs::path & path) +{ + std::error_code dummy; + if (path.filename().empty()) + return fs::is_symlink(path.parent_path(), dummy); /// STYLE_CHECK_ALLOW_STD_FS_SYMLINK + return fs::is_symlink(path, dummy); /// STYLE_CHECK_ALLOW_STD_FS_SYMLINK +} + fs::path readSymlink(const fs::path & path) { /// See the comment for isSymlink diff --git a/src/Common/filesystemHelpers.h b/src/Common/filesystemHelpers.h index 14ee5f54322..8591cd6cf92 100644 --- a/src/Common/filesystemHelpers.h +++ b/src/Common/filesystemHelpers.h @@ -95,6 +95,7 @@ void setModificationTime(const std::string & path, time_t time); time_t getChangeTime(const std::string & path); bool isSymlink(const fs::path & path); +bool isSymlinkNoThrow(const fs::path & path); fs::path readSymlink(const fs::path & path); } diff --git a/src/Common/formatIPv6.cpp b/src/Common/formatIPv6.cpp index 7c027a23b4d..86e33beb7c3 100644 --- a/src/Common/formatIPv6.cpp +++ b/src/Common/formatIPv6.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include diff --git a/src/Common/formatIPv6.h b/src/Common/formatIPv6.h index 7b88f93750b..be4dfc7391e 100644 --- a/src/Common/formatIPv6.h +++ b/src/Common/formatIPv6.h @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include constexpr size_t IPV4_BINARY_LENGTH = 4; diff --git a/src/Common/getHashOfLoadedBinary.cpp b/src/Common/getHashOfLoadedBinary.cpp index da053750036..cc0ad0d2143 100644 --- a/src/Common/getHashOfLoadedBinary.cpp +++ b/src/Common/getHashOfLoadedBinary.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include static int callback(dl_phdr_info * info, size_t, void * data) diff --git a/src/Common/getMappedArea.cpp b/src/Common/getMappedArea.cpp index 573d3194f3d..4f40c604c6a 100644 --- a/src/Common/getMappedArea.cpp +++ b/src/Common/getMappedArea.cpp @@ -4,7 +4,7 @@ #if defined(OS_LINUX) #include -#include +#include #include #include diff --git a/src/Common/hex.cpp b/src/Common/hex.cpp deleted file mode 100644 index e8f9b981062..00000000000 --- a/src/Common/hex.cpp +++ /dev/null @@ -1,92 +0,0 @@ -#include - -const char * const hex_digit_to_char_uppercase_table = "0123456789ABCDEF"; -const char * const hex_digit_to_char_lowercase_table = "0123456789abcdef"; - -const char * const hex_byte_to_char_uppercase_table = - "000102030405060708090A0B0C0D0E0F" - "101112131415161718191A1B1C1D1E1F" - "202122232425262728292A2B2C2D2E2F" - "303132333435363738393A3B3C3D3E3F" - "404142434445464748494A4B4C4D4E4F" - "505152535455565758595A5B5C5D5E5F" - "606162636465666768696A6B6C6D6E6F" - "707172737475767778797A7B7C7D7E7F" - "808182838485868788898A8B8C8D8E8F" - "909192939495969798999A9B9C9D9E9F" - "A0A1A2A3A4A5A6A7A8A9AAABACADAEAF" - "B0B1B2B3B4B5B6B7B8B9BABBBCBDBEBF" - "C0C1C2C3C4C5C6C7C8C9CACBCCCDCECF" - "D0D1D2D3D4D5D6D7D8D9DADBDCDDDEDF" - "E0E1E2E3E4E5E6E7E8E9EAEBECEDEEEF" - "F0F1F2F3F4F5F6F7F8F9FAFBFCFDFEFF"; - -const char * const hex_byte_to_char_lowercase_table = - "000102030405060708090a0b0c0d0e0f" - "101112131415161718191a1b1c1d1e1f" - "202122232425262728292a2b2c2d2e2f" - "303132333435363738393a3b3c3d3e3f" - "404142434445464748494a4b4c4d4e4f" - "505152535455565758595a5b5c5d5e5f" - "606162636465666768696a6b6c6d6e6f" - "707172737475767778797a7b7c7d7e7f" - "808182838485868788898a8b8c8d8e8f" - "909192939495969798999a9b9c9d9e9f" - "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf" - "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf" - "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf" - "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf" - "e0e1e2e3e4e5e6e7e8e9eaebecedeeef" - "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"; - -const char * const hex_char_to_digit_table = - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\xff\xff\xff\xff\xff\xff" //0-9 - "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //A-Z - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff" //a-z - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"; - -const char * const bin_byte_to_char_table = - "0000000000000001000000100000001100000100000001010000011000000111" - "0000100000001001000010100000101100001100000011010000111000001111" - "0001000000010001000100100001001100010100000101010001011000010111" - "0001100000011001000110100001101100011100000111010001111000011111" - "0010000000100001001000100010001100100100001001010010011000100111" - "0010100000101001001010100010101100101100001011010010111000101111" - "0011000000110001001100100011001100110100001101010011011000110111" - "0011100000111001001110100011101100111100001111010011111000111111" - "0100000001000001010000100100001101000100010001010100011001000111" - "0100100001001001010010100100101101001100010011010100111001001111" - "0101000001010001010100100101001101010100010101010101011001010111" - "0101100001011001010110100101101101011100010111010101111001011111" - "0110000001100001011000100110001101100100011001010110011001100111" - "0110100001101001011010100110101101101100011011010110111001101111" - "0111000001110001011100100111001101110100011101010111011001110111" - "0111100001111001011110100111101101111100011111010111111001111111" - "1000000010000001100000101000001110000100100001011000011010000111" - "1000100010001001100010101000101110001100100011011000111010001111" - "1001000010010001100100101001001110010100100101011001011010010111" - "1001100010011001100110101001101110011100100111011001111010011111" - "1010000010100001101000101010001110100100101001011010011010100111" - "1010100010101001101010101010101110101100101011011010111010101111" - "1011000010110001101100101011001110110100101101011011011010110111" - "1011100010111001101110101011101110111100101111011011111010111111" - "1100000011000001110000101100001111000100110001011100011011000111" - "1100100011001001110010101100101111001100110011011100111011001111" - "1101000011010001110100101101001111010100110101011101011011010111" - "1101100011011001110110101101101111011100110111011101111011011111" - "1110000011100001111000101110001111100100111001011110011011100111" - "1110100011101001111010101110101111101100111011011110111011101111" - "1111000011110001111100101111001111110100111101011111011011110111" - "1111100011111001111110101111101111111100111111011111111011111111"; diff --git a/src/Common/hex.h b/src/Common/hex.h deleted file mode 100644 index 062a6c27f76..00000000000 --- a/src/Common/hex.h +++ /dev/null @@ -1,145 +0,0 @@ -#pragma once -#include - - -/// Maps 0..15 to 0..9A..F or 0..9a..f correspondingly. - -extern const char * const hex_digit_to_char_uppercase_table; -extern const char * const hex_digit_to_char_lowercase_table; - -inline char hexDigitUppercase(unsigned char c) -{ - return hex_digit_to_char_uppercase_table[c]; -} - -inline char hexDigitLowercase(unsigned char c) -{ - return hex_digit_to_char_lowercase_table[c]; -} - - -#include -#include - -#include - - -/// Maps 0..255 to 00..FF or 00..ff correspondingly - -extern const char * const hex_byte_to_char_uppercase_table; -extern const char * const hex_byte_to_char_lowercase_table; - -inline void writeHexByteUppercase(UInt8 byte, void * out) -{ - memcpy(out, &hex_byte_to_char_uppercase_table[static_cast(byte) * 2], 2); -} - -inline void writeHexByteLowercase(UInt8 byte, void * out) -{ - memcpy(out, &hex_byte_to_char_lowercase_table[static_cast(byte) * 2], 2); -} - -extern const char * const bin_byte_to_char_table; - -inline void writeBinByte(UInt8 byte, void * out) -{ - memcpy(out, &bin_byte_to_char_table[static_cast(byte) * 8], 8); -} - -/// Produces hex representation of an unsigned int with leading zeros (for checksums) -template -inline void writeHexUIntImpl(TUInt uint_, char * out, const char * const table) -{ - union - { - TUInt value; - UInt8 uint8[sizeof(TUInt)]; - }; - - value = uint_; - - for (size_t i = 0; i < sizeof(TUInt); ++i) - { - if constexpr (std::endian::native == std::endian::little) - memcpy(out + i * 2, &table[static_cast(uint8[sizeof(TUInt) - 1 - i]) * 2], 2); - else - memcpy(out + i * 2, &table[static_cast(uint8[i]) * 2], 2); - } -} - -template -inline void writeHexUIntUppercase(TUInt uint_, char * out) -{ - writeHexUIntImpl(uint_, out, hex_byte_to_char_uppercase_table); -} - -template -inline void writeHexUIntLowercase(TUInt uint_, char * out) -{ - writeHexUIntImpl(uint_, out, hex_byte_to_char_lowercase_table); -} - -template -std::string getHexUIntUppercase(TUInt uint_) -{ - std::string res(sizeof(TUInt) * 2, '\0'); - writeHexUIntUppercase(uint_, res.data()); - return res; -} - -template -std::string getHexUIntLowercase(TUInt uint_) -{ - std::string res(sizeof(TUInt) * 2, '\0'); - writeHexUIntLowercase(uint_, res.data()); - return res; -} - - -/// Maps 0..9, A..F, a..f to 0..15. Other chars are mapped to implementation specific value. - -extern const char * const hex_char_to_digit_table; - -inline UInt8 unhex(char c) -{ - return hex_char_to_digit_table[static_cast(c)]; -} - -inline UInt8 unhex2(const char * data) -{ - return - static_cast(unhex(data[0])) * 0x10 - + static_cast(unhex(data[1])); -} - -inline UInt16 unhex4(const char * data) -{ - return - static_cast(unhex(data[0])) * 0x1000 - + static_cast(unhex(data[1])) * 0x100 - + static_cast(unhex(data[2])) * 0x10 - + static_cast(unhex(data[3])); -} - -template -TUInt unhexUInt(const char * data) -{ - TUInt res = 0; - if constexpr ((sizeof(TUInt) <= 8) || ((sizeof(TUInt) % 8) != 0)) - { - for (size_t i = 0; i < sizeof(TUInt) * 2; ++i, ++data) - { - res <<= 4; - res += unhex(*data); - } - } - else - { - for (size_t i = 0; i < sizeof(TUInt) / 8; ++i, data += 16) - { - res <<= 64; - res += unhexUInt(data); - } - } - return res; -} diff --git a/src/Common/interpolate.h b/src/Common/interpolate.h deleted file mode 100644 index 05900563b80..00000000000 --- a/src/Common/interpolate.h +++ /dev/null @@ -1,20 +0,0 @@ -#pragma once - -#include - - -inline double interpolateLinear(double min, double max, double ratio) -{ - return min + (max - min) * ratio; -} - - -/** It is linear interpolation in logarithmic coordinates. - * Exponential interpolation is related to linear interpolation - * exactly in same way as geometric mean is related to arithmetic mean. - * 'min' must be greater than zero, 'ratio' must be from 0 to 1. - */ -inline double interpolateExponential(double min, double max, double ratio) -{ - return min * std::pow(max / min, ratio); -} diff --git a/src/Compression/CompressedReadBufferBase.cpp b/src/Compression/CompressedReadBufferBase.cpp index ab856cc9801..3111f649b26 100644 --- a/src/Compression/CompressedReadBufferBase.cpp +++ b/src/Compression/CompressedReadBufferBase.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Compression/CompressionCodecMultiple.cpp b/src/Compression/CompressionCodecMultiple.cpp index 5203e349317..dba67749e4d 100644 --- a/src/Compression/CompressionCodecMultiple.cpp +++ b/src/Compression/CompressionCodecMultiple.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include namespace DB diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index a6d16334924..2aa11dd9eed 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 33b2a91d8bf..41a6af54204 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 2fa2c83b158..f67ce6be9ed 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -147,6 +147,8 @@ class IColumn; M(UInt64, max_parallel_replicas, 1, "The maximum number of replicas of each shard used when the query is executed. For consistency (to get different parts of the same partition), this option only works for the specified sampling key. The lag of the replicas is not controlled.", 0) \ M(UInt64, parallel_replicas_count, 0, "This is internal setting that should not be used directly and represents an implementation detail of the 'parallel replicas' mode. This setting will be automatically set up by the initiator server for distributed queries to the number of parallel replicas participating in query processing.", 0) \ M(UInt64, parallel_replica_offset, 0, "This is internal setting that should not be used directly and represents an implementation detail of the 'parallel replicas' mode. This setting will be automatically set up by the initiator server for distributed queries to the index of the replica participating in query processing among parallel replicas.", 0) \ + M(String, parallel_replicas_custom_key, "", "Custom key assigning work to replicas when parallel replicas are used.", 0) \ + M(ParallelReplicasCustomKeyFilterType, parallel_replicas_custom_key_filter_type, ParallelReplicasCustomKeyFilterType::DEFAULT, "Type of filter to use with custom key for parallel replicas. default - use modulo operation on the custom key, range - use range filter on custom key using all possible values for the value type of custom key.", 0) \ \ M(String, cluster_for_parallel_replicas, "default", "Cluster for a shard in which current server is located", 0) \ M(Bool, allow_experimental_parallel_reading_from_replicas, false, "If true, ClickHouse will send a SELECT query to all replicas of a table. It will work for any kind on MergeTree table.", 0) \ @@ -514,6 +516,7 @@ class IColumn; M(Bool, allow_experimental_alter_materialized_view_structure, false, "Allow atomic alter on Materialized views. Work in progress.", 0) \ M(Bool, enable_early_constant_folding, true, "Enable query optimization where we analyze function and subqueries results and rewrite query if there're constants there", 0) \ M(Bool, deduplicate_blocks_in_dependent_materialized_views, false, "Should deduplicate blocks for materialized views if the block is not a duplicate for the table. Use true to always deduplicate in dependent tables.", 0) \ + M(Bool, materialized_views_ignore_errors, false, "Allows to ignore errors for MATERIALIZED VIEW, and deliver original block to the table regardless of MVs", 0) \ M(Bool, use_compact_format_in_distributed_parts_names, true, "Changes format of directories names for distributed table insert parts.", 0) \ M(Bool, validate_polygons, true, "Throw exception if polygon is invalid in function pointInPolygon (e.g. self-tangent, self-intersecting). If the setting is false, the function will accept invalid polygons but may silently return wrong result.", 0) \ M(UInt64, max_parser_depth, DBMS_DEFAULT_MAX_PARSER_DEPTH, "Maximum parser depth (recursion depth of recursive descend parser).", 0) \ @@ -864,6 +867,7 @@ class IColumn; M(Bool, output_format_parquet_string_as_string, false, "Use Parquet String type instead of Binary for String columns.", 0) \ M(Bool, output_format_parquet_fixed_string_as_fixed_byte_array, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type instead of Binary for FixedString columns.", 0) \ M(ParquetVersion, output_format_parquet_version, "2.latest", "Parquet format version for output format. Supported versions: 1.0, 2.4, 2.6 and 2.latest (default)", 0) \ + M(ParquetCompression, output_format_parquet_compression_method, "lz4", "Compression method for Parquet output format. Supported codecs: snappy, lz4, brotli, zstd, gzip, none (uncompressed)", 0) \ M(String, output_format_avro_codec, "", "Compression codec used for output. Possible values: 'null', 'deflate', 'snappy'.", 0) \ M(UInt64, output_format_avro_sync_interval, 16 * 1024, "Sync interval in bytes.", 0) \ M(String, output_format_avro_string_column_pattern, "", "For Avro format: regexp of String columns to select as AVRO string.", 0) \ @@ -906,8 +910,10 @@ class IColumn; M(Bool, output_format_arrow_low_cardinality_as_dictionary, false, "Enable output LowCardinality type as Dictionary Arrow type", 0) \ M(Bool, output_format_arrow_string_as_string, false, "Use Arrow String type instead of Binary for String columns", 0) \ M(Bool, output_format_arrow_fixed_string_as_fixed_byte_array, true, "Use Arrow FIXED_SIZE_BINARY type instead of Binary for FixedString columns.", 0) \ + M(ArrowCompression, output_format_arrow_compression_method, "lz4_frame", "Compression method for Arrow output format. Supported codecs: lz4_frame, zstd, none (uncompressed)", 0) \ \ M(Bool, output_format_orc_string_as_string, false, "Use ORC String type instead of Binary for String columns", 0) \ + M(ORCCompression, output_format_orc_compression_method, "lz4", "Compression method for ORC output format. Supported codecs: lz4, snappy, zlib, zstd, none (uncompressed)", 0) \ \ M(EnumComparingMode, format_capn_proto_enum_comparising_mode, FormatSettings::EnumComparingMode::BY_VALUES, "How to map ClickHouse Enum and CapnProto Enum", 0) \ \ diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index a4ef60436e6..caf18cf8fb8 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -82,7 +82,10 @@ static std::map sett { {"23.3", {{"output_format_parquet_version", "1.0", "2.latest", "Use latest Parquet format version for output format"}, {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"}, - {"input_format_native_allow_types_conversion", false, true, "Allow types conversion in Native input forma"}}}, + {"input_format_native_allow_types_conversion", false, true, "Allow types conversion in Native input forma"}, + {"output_format_arrow_compression_method", "none", "lz4_frame", "Use lz4 compression in Arrow output format by default"}, + {"output_format_parquet_compression_method", "snappy", "lz4", "Use lz4 compression in Parquet output format by default"}, + {"output_format_orc_compression_method", "none", "lz4_frame", "Use lz4 compression in ORC output format by default"}}}, {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"}, {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"}, {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"}, diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index 9e1ab585bb0..e0f16ea00db 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -158,7 +158,7 @@ IMPLEMENT_SETTING_ENUM(EscapingRule, ErrorCodes::BAD_ARGUMENTS, {"XML", FormatSettings::EscapingRule::XML}, {"Raw", FormatSettings::EscapingRule::Raw}}) -IMPLEMENT_SETTING_ENUM(MsgPackUUIDRepresentation , ErrorCodes::BAD_ARGUMENTS, +IMPLEMENT_SETTING_ENUM(MsgPackUUIDRepresentation, ErrorCodes::BAD_ARGUMENTS, {{"bin", FormatSettings::MsgPackUUIDRepresentation::BIN}, {"str", FormatSettings::MsgPackUUIDRepresentation::STR}, {"ext", FormatSettings::MsgPackUUIDRepresentation::EXT}}) @@ -167,16 +167,39 @@ IMPLEMENT_SETTING_ENUM(Dialect, ErrorCodes::BAD_ARGUMENTS, {{"clickhouse", Dialect::clickhouse}, {"kusto", Dialect::kusto}}) +IMPLEMENT_SETTING_ENUM(ParallelReplicasCustomKeyFilterType, ErrorCodes::BAD_ARGUMENTS, + {{"default", ParallelReplicasCustomKeyFilterType::DEFAULT}, + {"range", ParallelReplicasCustomKeyFilterType::RANGE}}) + IMPLEMENT_SETTING_ENUM(LocalFSReadMethod, ErrorCodes::BAD_ARGUMENTS, {{"mmap", LocalFSReadMethod::mmap}, {"pread", LocalFSReadMethod::pread}, {"read", LocalFSReadMethod::read}}) - IMPLEMENT_SETTING_ENUM_WITH_RENAME(ParquetVersion, ErrorCodes::BAD_ARGUMENTS, {{"1.0", FormatSettings::ParquetVersion::V1_0}, {"2.4", FormatSettings::ParquetVersion::V2_4}, {"2.6", FormatSettings::ParquetVersion::V2_6}, {"2.latest", FormatSettings::ParquetVersion::V2_LATEST}}) +IMPLEMENT_SETTING_ENUM(ParquetCompression, ErrorCodes::BAD_ARGUMENTS, + {{"none", FormatSettings::ParquetCompression::NONE}, + {"snappy", FormatSettings::ParquetCompression::SNAPPY}, + {"zstd", FormatSettings::ParquetCompression::ZSTD}, + {"gzip", FormatSettings::ParquetCompression::GZIP}, + {"lz4", FormatSettings::ParquetCompression::LZ4}, + {"brotli", FormatSettings::ParquetCompression::BROTLI}}) + +IMPLEMENT_SETTING_ENUM(ArrowCompression, ErrorCodes::BAD_ARGUMENTS, + {{"none", FormatSettings::ArrowCompression::NONE}, + {"lz4_frame", FormatSettings::ArrowCompression::LZ4_FRAME}, + {"zstd", FormatSettings::ArrowCompression::ZSTD}}) + +IMPLEMENT_SETTING_ENUM(ORCCompression, ErrorCodes::BAD_ARGUMENTS, + {{"none", FormatSettings::ORCCompression::NONE}, + {"snappy", FormatSettings::ORCCompression::SNAPPY}, + {"zstd", FormatSettings::ORCCompression::ZSTD}, + {"zlib", FormatSettings::ORCCompression::ZLIB}, + {"lz4", FormatSettings::ORCCompression::LZ4}}) + } diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 139a04f3a5a..3ae7bfaa673 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -194,6 +194,12 @@ DECLARE_SETTING_ENUM_WITH_RENAME(EscapingRule, FormatSettings::EscapingRule) DECLARE_SETTING_ENUM_WITH_RENAME(MsgPackUUIDRepresentation, FormatSettings::MsgPackUUIDRepresentation) +DECLARE_SETTING_ENUM_WITH_RENAME(ParquetCompression, FormatSettings::ParquetCompression) + +DECLARE_SETTING_ENUM_WITH_RENAME(ArrowCompression, FormatSettings::ArrowCompression) + +DECLARE_SETTING_ENUM_WITH_RENAME(ORCCompression, FormatSettings::ORCCompression) + enum class Dialect { clickhouse, @@ -203,5 +209,13 @@ enum class Dialect DECLARE_SETTING_ENUM(Dialect) +enum class ParallelReplicasCustomKeyFilterType : uint8_t +{ + DEFAULT, + RANGE, +}; + +DECLARE_SETTING_ENUM(ParallelReplicasCustomKeyFilterType) + DECLARE_SETTING_ENUM(LocalFSReadMethod) } diff --git a/src/Daemon/SentryWriter.cpp b/src/Daemon/SentryWriter.cpp index 9f4f18e64d1..3c62e54b117 100644 --- a/src/Daemon/SentryWriter.cpp +++ b/src/Daemon/SentryWriter.cpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include "config.h" #include "config_version.h" diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index 34c4fd3d5d8..7e20b6f6535 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -273,7 +273,7 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_ else renameNoReplace(old_metadata_path, new_metadata_path); - /// After metadata was successfully moved, the following methods should not throw (if them do, it's a logical error) + /// After metadata was successfully moved, the following methods should not throw (if they do, it's a logical error) table_data_path = detach(*this, table_name, table->storesDataOnDisk()); if (exchange) other_table_data_path = detach(other_db, to_table_name, other_table->storesDataOnDisk()); @@ -509,6 +509,9 @@ void DatabaseAtomic::tryCreateMetadataSymlink() { try { + /// fs::exists could return false for broken symlink + if (FS::isSymlinkNoThrow(metadata_symlink)) + fs::remove(metadata_symlink); fs::create_directory_symlink(metadata_path, path_to_metadata_symlink); } catch (...) diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 44a719c82bb..72346787cfb 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index 43b6544acb0..8450e740ab5 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Disks/getOrCreateDiskFromAST.cpp b/src/Disks/getOrCreateDiskFromAST.cpp index 5deb9ab11b5..637acff7b95 100644 --- a/src/Disks/getOrCreateDiskFromAST.cpp +++ b/src/Disks/getOrCreateDiskFromAST.cpp @@ -6,9 +6,13 @@ #include #include #include +#include +#include #include #include #include +#include +#include namespace DB { @@ -18,43 +22,85 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -std::string getOrCreateDiskFromDiskAST(const ASTFunction & function, ContextPtr context) +namespace { - /// We need a unique name for a created custom disk, but it needs to be the same - /// after table is reattached or server is restarted, so take a hash of the disk - /// configuration serialized ast as a disk name suffix. - auto disk_setting_string = serializeAST(function, true); - auto disk_name = DiskSelector::TMP_INTERNAL_DISK_PREFIX - + toString(sipHash128(disk_setting_string.data(), disk_setting_string.size())); - - auto result_disk = context->getOrCreateDisk(disk_name, [&](const DisksMap & disks_map) -> DiskPtr { - const auto * function_args_expr = assert_cast(function.arguments.get()); - const auto & function_args = function_args_expr->children; - auto config = getDiskConfigurationFromAST(disk_name, function_args, context); - auto disk = DiskFactory::instance().create(disk_name, *config, disk_name, context, disks_map); - /// Mark that disk can be used without storage policy. - disk->markDiskAsCustom(); - return disk; - }); - - if (!result_disk->isRemote()) + std::string getOrCreateDiskFromDiskAST(const ASTFunction & function, ContextPtr context) { - static constexpr auto custom_disks_base_dir_in_config = "custom_local_disks_base_directory"; - auto disk_path_expected_prefix = context->getConfigRef().getString(custom_disks_base_dir_in_config, ""); + /// We need a unique name for a created custom disk, but it needs to be the same + /// after table is reattached or server is restarted, so take a hash of the disk + /// configuration serialized ast as a disk name suffix. + auto disk_setting_string = serializeAST(function, true); + auto disk_name = DiskSelector::TMP_INTERNAL_DISK_PREFIX + + toString(sipHash128(disk_setting_string.data(), disk_setting_string.size())); - if (disk_path_expected_prefix.empty()) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Base path for custom local disks must be defined in config file by `{}`", - custom_disks_base_dir_in_config); + auto result_disk = context->getOrCreateDisk(disk_name, [&](const DisksMap & disks_map) -> DiskPtr { + const auto * function_args_expr = assert_cast(function.arguments.get()); + const auto & function_args = function_args_expr->children; + auto config = getDiskConfigurationFromAST(disk_name, function_args, context); + auto disk = DiskFactory::instance().create(disk_name, *config, disk_name, context, disks_map); + /// Mark that disk can be used without storage policy. + disk->markDiskAsCustom(); + return disk; + }); - if (!pathStartsWith(result_disk->getPath(), disk_path_expected_prefix)) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Path of the custom local disk must be inside `{}` directory", - disk_path_expected_prefix); + if (!result_disk->isRemote()) + { + static constexpr auto custom_disks_base_dir_in_config = "custom_local_disks_base_directory"; + auto disk_path_expected_prefix = context->getConfigRef().getString(custom_disks_base_dir_in_config, ""); + + if (disk_path_expected_prefix.empty()) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Base path for custom local disks must be defined in config file by `{}`", + custom_disks_base_dir_in_config); + + if (!pathStartsWith(result_disk->getPath(), disk_path_expected_prefix)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Path of the custom local disk must be inside `{}` directory", + disk_path_expected_prefix); + } + + return disk_name; } + class DiskConfigurationFlattener + { + public: + struct Data + { + ContextPtr context; + }; + + static bool needChildVisit(const ASTPtr &, const ASTPtr &) { return true; } + + static void visit(ASTPtr & ast, Data & data) + { + if (isDiskFunction(ast)) + { + auto disk_name = getOrCreateDiskFromDiskAST(*ast->as(), data.context); + ast = std::make_shared(disk_name); + } + } + }; + + /// Visits children first. + using FlattenDiskConfigurationVisitor = InDepthNodeVisitor; +} + + +std::string getOrCreateDiskFromDiskAST(const ASTPtr & disk_function, ContextPtr context) +{ + if (!isDiskFunction(disk_function)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected a disk function"); + + auto ast = disk_function->clone(); + + FlattenDiskConfigurationVisitor::Data data{context}; + FlattenDiskConfigurationVisitor{data}.visit(ast); + + auto disk_name = assert_cast(*ast).value.get(); + LOG_TRACE(&Poco::Logger::get("getOrCreateDiskFromDiskAST"), "Result disk name: {}", disk_name); return disk_name; } diff --git a/src/Disks/getOrCreateDiskFromAST.h b/src/Disks/getOrCreateDiskFromAST.h index 7c64707b0bd..0195f575278 100644 --- a/src/Disks/getOrCreateDiskFromAST.h +++ b/src/Disks/getOrCreateDiskFromAST.h @@ -13,6 +13,6 @@ class ASTFunction; * add it to DiskSelector by a unique (but always the same for given configuration) disk name * and return this name. */ -std::string getOrCreateDiskFromDiskAST(const ASTFunction & function, ContextPtr context); +std::string getOrCreateDiskFromDiskAST(const ASTPtr & disk_function, ContextPtr context); } diff --git a/src/Formats/BSONTypes.cpp b/src/Formats/BSONTypes.cpp index 813c155325a..88396fd2aba 100644 --- a/src/Formats/BSONTypes.cpp +++ b/src/Formats/BSONTypes.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include namespace DB { diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 605ef5346ef..a951a7fdd92 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -118,6 +118,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.parquet.output_string_as_string = settings.output_format_parquet_string_as_string; format_settings.parquet.output_fixed_string_as_fixed_byte_array = settings.output_format_parquet_fixed_string_as_fixed_byte_array; format_settings.parquet.max_block_size = settings.input_format_parquet_max_block_size; + format_settings.parquet.output_compression_method = settings.output_format_parquet_compression_method; format_settings.pretty.charset = settings.output_format_pretty_grid_charset.toString() == "ASCII" ? FormatSettings::Pretty::Charset::ASCII : FormatSettings::Pretty::Charset::UTF8; format_settings.pretty.color = settings.output_format_pretty_color; format_settings.pretty.max_column_pad_width = settings.output_format_pretty_max_column_pad_width; @@ -158,6 +159,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.arrow.case_insensitive_column_matching = settings.input_format_arrow_case_insensitive_column_matching; format_settings.arrow.output_string_as_string = settings.output_format_arrow_string_as_string; format_settings.arrow.output_fixed_string_as_fixed_byte_array = settings.output_format_arrow_fixed_string_as_fixed_byte_array; + format_settings.arrow.output_compression_method = settings.output_format_arrow_compression_method; format_settings.orc.import_nested = settings.input_format_orc_import_nested; format_settings.orc.allow_missing_columns = settings.input_format_orc_allow_missing_columns; format_settings.orc.row_batch_size = settings.input_format_orc_row_batch_size; @@ -168,6 +170,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.orc.skip_columns_with_unsupported_types_in_schema_inference = settings.input_format_orc_skip_columns_with_unsupported_types_in_schema_inference; format_settings.orc.case_insensitive_column_matching = settings.input_format_orc_case_insensitive_column_matching; format_settings.orc.output_string_as_string = settings.output_format_orc_string_as_string; + format_settings.orc.output_compression_method = settings.output_format_orc_compression_method; format_settings.defaults_for_omitted_fields = settings.input_format_defaults_for_omitted_fields; format_settings.capn_proto.enum_comparing_mode = settings.format_capn_proto_enum_comparising_mode; format_settings.capn_proto.skip_fields_with_unsupported_types_in_schema_inference = settings.input_format_capn_proto_skip_fields_with_unsupported_types_in_schema_inference; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 60ba170e0a7..7be7b5b98aa 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -86,6 +86,13 @@ struct FormatSettings UInt64 max_parser_depth = DBMS_DEFAULT_MAX_PARSER_DEPTH; + enum class ArrowCompression + { + NONE, + LZ4_FRAME, + ZSTD + }; + struct { UInt64 row_group_size = 1000000; @@ -96,6 +103,7 @@ struct FormatSettings bool case_insensitive_column_matching = false; bool output_string_as_string = false; bool output_fixed_string_as_fixed_byte_array = true; + ArrowCompression output_compression_method = ArrowCompression::NONE; } arrow; struct @@ -183,6 +191,16 @@ struct FormatSettings V2_LATEST, }; + enum class ParquetCompression + { + NONE, + SNAPPY, + ZSTD, + LZ4, + GZIP, + BROTLI, + }; + struct { UInt64 row_group_size = 1000000; @@ -195,6 +213,7 @@ struct FormatSettings bool output_fixed_string_as_fixed_byte_array = true; UInt64 max_block_size = 8192; ParquetVersion output_version; + ParquetCompression output_compression_method = ParquetCompression::SNAPPY; } parquet; struct Pretty @@ -276,6 +295,15 @@ struct FormatSettings bool accurate_types_of_literals = true; } values; + enum class ORCCompression + { + NONE, + LZ4, + SNAPPY, + ZSTD, + ZLIB, + }; + struct { bool import_nested = false; @@ -285,6 +313,7 @@ struct FormatSettings bool case_insensitive_column_matching = false; std::unordered_set skip_stripes = {}; bool output_string_as_string = false; + ORCCompression output_compression_method = ORCCompression::NONE; } orc; /// For capnProto format we should determine how to diff --git a/src/Formats/verbosePrintString.cpp b/src/Formats/verbosePrintString.cpp index 2f3e09ed75f..5c6111c2929 100644 --- a/src/Formats/verbosePrintString.cpp +++ b/src/Formats/verbosePrintString.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include diff --git a/src/Functions/FunctionsCodingIP.cpp b/src/Functions/FunctionsCodingIP.cpp index a941092b7d6..4784368db9b 100644 --- a/src/Functions/FunctionsCodingIP.cpp +++ b/src/Functions/FunctionsCodingIP.cpp @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Functions/FunctionsCodingUUID.cpp b/src/Functions/FunctionsCodingUUID.cpp index dade406c801..dd9170e44ad 100644 --- a/src/Functions/FunctionsCodingUUID.cpp +++ b/src/Functions/FunctionsCodingUUID.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Functions/URL/decodeURLComponent.cpp b/src/Functions/URL/decodeURLComponent.cpp index 9e516e73e3c..7d98ccd63a0 100644 --- a/src/Functions/URL/decodeURLComponent.cpp +++ b/src/Functions/URL/decodeURLComponent.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Functions/bitShiftRight.cpp b/src/Functions/bitShiftRight.cpp index 108847f13ed..21a0f7584aa 100644 --- a/src/Functions/bitShiftRight.cpp +++ b/src/Functions/bitShiftRight.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include namespace DB { diff --git a/src/Functions/decodeXMLComponent.cpp b/src/Functions/decodeXMLComponent.cpp index 8b84bb1194e..a25e67e0e37 100644 --- a/src/Functions/decodeXMLComponent.cpp +++ b/src/Functions/decodeXMLComponent.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include diff --git a/src/Functions/repeat.cpp b/src/Functions/repeat.cpp index dcd05f373fc..0c323c39969 100644 --- a/src/Functions/repeat.cpp +++ b/src/Functions/repeat.cpp @@ -39,13 +39,15 @@ struct RepeatImpl size, max_string_size); } + template static void vectorStrConstRepeat( const ColumnString::Chars & data, const ColumnString::Offsets & offsets, ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets, - UInt64 repeat_time) + T repeat_time) { + repeat_time = repeat_time < 0 ? 0 : repeat_time; checkRepeatTime(repeat_time); UInt64 data_size = 0; @@ -77,7 +79,8 @@ struct RepeatImpl res_offsets.assign(offsets); for (UInt64 i = 0; i < col_num.size(); ++i) { - size_t repeated_size = (offsets[i] - offsets[i - 1] - 1) * col_num[i] + 1; + T repeat_time = col_num[i] < 0 ? 0 : col_num[i]; + size_t repeated_size = (offsets[i] - offsets[i - 1] - 1) * repeat_time + 1; checkStringSize(repeated_size); data_size += repeated_size; res_offsets[i] = data_size; @@ -86,7 +89,7 @@ struct RepeatImpl for (UInt64 i = 0; i < col_num.size(); ++i) { - T repeat_time = col_num[i]; + T repeat_time = col_num[i] < 0 ? 0 : col_num[i]; checkRepeatTime(repeat_time); process(data.data() + offsets[i - 1], res_data.data() + res_offsets[i - 1], offsets[i] - offsets[i - 1], repeat_time); } @@ -105,7 +108,8 @@ struct RepeatImpl UInt64 col_size = col_num.size(); for (UInt64 i = 0; i < col_size; ++i) { - size_t repeated_size = str_size * col_num[i] + 1; + T repeat_time = col_num[i] < 0 ? 0 : col_num[i]; + size_t repeated_size = str_size * repeat_time + 1; checkStringSize(repeated_size); data_size += repeated_size; res_offsets[i] = data_size; @@ -113,7 +117,7 @@ struct RepeatImpl res_data.resize(data_size); for (UInt64 i = 0; i < col_size; ++i) { - T repeat_time = col_num[i]; + T repeat_time = col_num[i] < 0 ? 0 : col_num[i]; checkRepeatTime(repeat_time); process( reinterpret_cast(const_cast(copy_str.data())), @@ -168,7 +172,8 @@ class FunctionRepeat : public IFunction template static bool castType(const IDataType * type, F && f) { - return castTypeToEither(type, std::forward(f)); + return castTypeToEither(type, std::forward(f)); } public: @@ -186,7 +191,7 @@ public: if (!isString(arguments[0])) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[0]->getName(), getName()); - if (!isUnsignedInteger(arguments[1])) + if (!isInteger(arguments[1])) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[1]->getName(), getName()); return arguments[0]; @@ -204,9 +209,15 @@ public: { if (const ColumnConst * scale_column_num = checkAndGetColumn(numcolumn.get())) { - UInt64 repeat_time = scale_column_num->getValue(); auto col_res = ColumnString::create(); - RepeatImpl::vectorStrConstRepeat(col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets(), repeat_time); + castType(arguments[1].type.get(), [&](const auto & type) + { + using DataType = std::decay_t; + using T = typename DataType::FieldType; + T repeat_time = scale_column_num->getValue(); + RepeatImpl::vectorStrConstRepeat(col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets(), repeat_time); + return true; + }); return col_res; } else if (castType(arguments[1].type.get(), [&](const auto & type) diff --git a/src/Functions/runningDifference.h b/src/Functions/runningDifference.h index 154370d4cd9..f1ec4f9e523 100644 --- a/src/Functions/runningDifference.h +++ b/src/Functions/runningDifference.h @@ -70,7 +70,7 @@ private: if (!has_prev_value) { - dst[i] = is_first_line_zero ? 0 : src[i]; + dst[i] = is_first_line_zero ? static_cast(0) : static_cast(src[i]); prev = src[i]; has_prev_value = true; } @@ -102,6 +102,10 @@ private: f(UInt32()); else if (which.isUInt64()) f(UInt64()); + else if (which.isUInt128()) + f(UInt128()); + else if (which.isUInt256()) + f(UInt256()); else if (which.isInt8()) f(Int8()); else if (which.isInt16()) @@ -110,6 +114,10 @@ private: f(Int32()); else if (which.isInt64()) f(Int64()); + else if (which.isInt128()) + f(Int128()); + else if (which.isInt256()) + f(Int256()); else if (which.isFloat32()) f(Float32()); else if (which.isFloat64()) diff --git a/src/IO/HTTPChunkedReadBuffer.cpp b/src/IO/HTTPChunkedReadBuffer.cpp index b9c42088c41..65ccad4aab7 100644 --- a/src/IO/HTTPChunkedReadBuffer.cpp +++ b/src/IO/HTTPChunkedReadBuffer.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index 86a2b9c650e..e14b3ae9129 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include diff --git a/src/IO/WriteHelpers.cpp b/src/IO/WriteHelpers.cpp index caeea0a82a3..a0eceddc6f6 100644 --- a/src/IO/WriteHelpers.cpp +++ b/src/IO/WriteHelpers.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include namespace DB diff --git a/src/IO/tests/gtest_hadoop_snappy_decoder.cpp b/src/IO/tests/gtest_hadoop_snappy_decoder.cpp index 4db0deac08e..2847c730735 100644 --- a/src/IO/tests/gtest_hadoop_snappy_decoder.cpp +++ b/src/IO/tests/gtest_hadoop_snappy_decoder.cpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include using namespace DB; TEST(HadoopSnappyDecoder, repeatNeedMoreInput) { diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index daba4c1608d..46b5a93b28c 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -199,6 +200,23 @@ const ActionsDAG::Node & ActionsDAG::addFunction( std::move(children), std::move(arguments), std::move(result_name), + function_base->getResultType(), + all_const); +} + +const ActionsDAG::Node & ActionsDAG::addFunction( + const FunctionNode & function, + NodeRawConstPtrs children, + std::string result_name) +{ + auto [arguments, all_const] = getFunctionArguments(children); + + return addFunctionImpl( + function.getFunction(), + std::move(children), + std::move(arguments), + std::move(result_name), + function.getResultType(), all_const); } @@ -214,6 +232,7 @@ const ActionsDAG::Node & ActionsDAG::addFunction( std::move(children), std::move(arguments), std::move(result_name), + function_base->getResultType(), all_const); } @@ -238,6 +257,7 @@ const ActionsDAG::Node & ActionsDAG::addFunctionImpl( NodeRawConstPtrs children, ColumnsWithTypeAndName arguments, std::string result_name, + DataTypePtr result_type, bool all_const) { size_t num_arguments = children.size(); @@ -247,7 +267,7 @@ const ActionsDAG::Node & ActionsDAG::addFunctionImpl( node.children = std::move(children); node.function_base = function_base; - node.result_type = node.function_base->getResultType(); + node.result_type = result_type; node.function = node.function_base->prepare(arguments); node.is_deterministic = node.function_base->isDeterministic(); @@ -2264,7 +2284,15 @@ ActionsDAGPtr ActionsDAG::buildFilterActionsDAG( for (const auto & child : node->children) function_children.push_back(node_to_result_node.find(child)->second); - result_node = &result_dag->addFunction(node->function_base, std::move(function_children), {}); + auto [arguments, all_const] = getFunctionArguments(function_children); + + result_node = &result_dag->addFunctionImpl( + node->function_base, + std::move(function_children), + std::move(arguments), + {}, + node->result_type, + all_const); break; } } diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 0182db8e027..5f0005dae37 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -23,6 +23,8 @@ using FunctionBasePtr = std::shared_ptr; class IFunctionOverloadResolver; using FunctionOverloadResolverPtr = std::shared_ptr; +class FunctionNode; + class IDataType; using DataTypePtr = std::shared_ptr; @@ -139,6 +141,10 @@ public: const FunctionOverloadResolverPtr & function, NodeRawConstPtrs children, std::string result_name); + const Node & addFunction( + const FunctionNode & function, + NodeRawConstPtrs children, + std::string result_name); const Node & addFunction( const FunctionBasePtr & function_base, NodeRawConstPtrs children, @@ -358,6 +364,7 @@ private: NodeRawConstPtrs children, ColumnsWithTypeAndName arguments, std::string result_name, + DataTypePtr result_type, bool all_const); #if USE_EMBEDDED_COMPILER diff --git a/src/Interpreters/Cache/FileCacheKey.h b/src/Interpreters/Cache/FileCacheKey.h index fed4c7f47e0..67e1466e2d4 100644 --- a/src/Interpreters/Cache/FileCacheKey.h +++ b/src/Interpreters/Cache/FileCacheKey.h @@ -1,6 +1,6 @@ #pragma once #include -#include +#include #include namespace DB diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index f4d7b2612a5..bd4554c6532 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/Interpreters/Cluster.cpp b/src/Interpreters/Cluster.cpp index bf3a66fed99..0add0e427f9 100644 --- a/src/Interpreters/Cluster.cpp +++ b/src/Interpreters/Cluster.cpp @@ -15,6 +15,7 @@ #include #include +#include namespace DB { @@ -509,7 +510,6 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config, shard_local_addresses.push_back(replica); shard_all_addresses.push_back(replica); } - ConnectionPoolWithFailoverPtr shard_pool = std::make_shared( all_replicas_pools, settings.load_balancing, settings.distributed_replica_error_half_life.totalSeconds(), settings.distributed_replica_error_cap); @@ -653,9 +653,9 @@ void Cluster::initMisc() } } -std::unique_ptr Cluster::getClusterWithReplicasAsShards(const Settings & settings) const +std::unique_ptr Cluster::getClusterWithReplicasAsShards(const Settings & settings, size_t max_replicas_from_shard) const { - return std::unique_ptr{ new Cluster(ReplicasAsShardsTag{}, *this, settings)}; + return std::unique_ptr{ new Cluster(ReplicasAsShardsTag{}, *this, settings, max_replicas_from_shard)}; } std::unique_ptr Cluster::getClusterWithSingleShard(size_t index) const @@ -668,7 +668,44 @@ std::unique_ptr Cluster::getClusterWithMultipleShards(const std::vector return std::unique_ptr{ new Cluster(SubclusterTag{}, *this, indices) }; } -Cluster::Cluster(Cluster::ReplicasAsShardsTag, const Cluster & from, const Settings & settings) +namespace +{ + +void shuffleReplicas(std::vector & replicas, const Settings & settings, size_t replicas_needed) +{ + std::random_device rd; + std::mt19937 gen{rd()}; + + if (settings.prefer_localhost_replica) + { + // force for local replica to always be included + auto first_non_local_replica = std::partition(replicas.begin(), replicas.end(), [](const auto & replica) { return replica.is_local; }); + size_t local_replicas_count = first_non_local_replica - replicas.begin(); + + if (local_replicas_count == replicas_needed) + { + /// we have exact amount of local replicas as needed, no need to do anything + return; + } + + if (local_replicas_count > replicas_needed) + { + /// we can use only local replicas, shuffle them + std::shuffle(replicas.begin(), first_non_local_replica, gen); + return; + } + + /// shuffle just non local replicas + std::shuffle(first_non_local_replica, replicas.end(), gen); + return; + } + + std::shuffle(replicas.begin(), replicas.end(), gen); +} + +} + +Cluster::Cluster(Cluster::ReplicasAsShardsTag, const Cluster & from, const Settings & settings, size_t max_replicas_from_shard) { if (from.addresses_with_failover.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cluster is empty"); @@ -677,40 +714,55 @@ Cluster::Cluster(Cluster::ReplicasAsShardsTag, const Cluster & from, const Setti std::set> unique_hosts; for (size_t shard_index : collections::range(0, from.shards_info.size())) { - const auto & replicas = from.addresses_with_failover[shard_index]; - for (const auto & address : replicas) + auto create_shards_from_replicas = [&](std::span replicas) { - if (!unique_hosts.emplace(address.host_name, address.port).second) - continue; /// Duplicate host, skip. + for (const auto & address : replicas) + { + if (!unique_hosts.emplace(address.host_name, address.port).second) + continue; /// Duplicate host, skip. - ShardInfo info; - info.shard_num = ++shard_num; + ShardInfo info; + info.shard_num = ++shard_num; - if (address.is_local) - info.local_addresses.push_back(address); + if (address.is_local) + info.local_addresses.push_back(address); - info.all_addresses.push_back(address); + info.all_addresses.push_back(address); - auto pool = ConnectionPoolFactory::instance().get( - static_cast(settings.distributed_connections_pool_size), - address.host_name, - address.port, - address.default_database, - address.user, - address.password, - address.quota_key, - address.cluster, - address.cluster_secret, - "server", - address.compression, - address.secure, - address.priority); + auto pool = ConnectionPoolFactory::instance().get( + static_cast(settings.distributed_connections_pool_size), + address.host_name, + address.port, + address.default_database, + address.user, + address.password, + address.quota_key, + address.cluster, + address.cluster_secret, + "server", + address.compression, + address.secure, + address.priority); - info.pool = std::make_shared(ConnectionPoolPtrs{pool}, settings.load_balancing); - info.per_replica_pools = {std::move(pool)}; + info.pool = std::make_shared(ConnectionPoolPtrs{pool}, settings.load_balancing); + info.per_replica_pools = {std::move(pool)}; - addresses_with_failover.emplace_back(Addresses{address}); - shards_info.emplace_back(std::move(info)); + addresses_with_failover.emplace_back(Addresses{address}); + shards_info.emplace_back(std::move(info)); + } + }; + + const auto & replicas = from.addresses_with_failover[shard_index]; + if (!max_replicas_from_shard || replicas.size() <= max_replicas_from_shard) + { + create_shards_from_replicas(replicas); + } + else + { + auto shuffled_replicas = replicas; + // shuffle replicas so we don't always pick the same subset + shuffleReplicas(shuffled_replicas, settings, max_replicas_from_shard); + create_shards_from_replicas(std::span{shuffled_replicas.begin(), max_replicas_from_shard}); } } diff --git a/src/Interpreters/Cluster.h b/src/Interpreters/Cluster.h index a7f5a914974..02c450c3c6b 100644 --- a/src/Interpreters/Cluster.h +++ b/src/Interpreters/Cluster.h @@ -250,7 +250,7 @@ public: std::unique_ptr getClusterWithMultipleShards(const std::vector & indices) const; /// Get a new Cluster that contains all servers (all shards with all replicas) from existing cluster as independent shards. - std::unique_ptr getClusterWithReplicasAsShards(const Settings & settings) const; + std::unique_ptr getClusterWithReplicasAsShards(const Settings & settings, size_t max_replicas_from_shard = 0) const; /// Returns false if cluster configuration doesn't allow to use it for cross-replication. /// NOTE: true does not mean, that it's actually a cross-replication cluster. @@ -271,7 +271,7 @@ private: /// For getClusterWithReplicasAsShards implementation struct ReplicasAsShardsTag {}; - Cluster(ReplicasAsShardsTag, const Cluster & from, const Settings & settings); + Cluster(ReplicasAsShardsTag, const Cluster & from, const Settings & settings, size_t max_replicas_from_shard); /// Inter-server secret String secret; diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 85a012d126f..439033f89f2 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,7 @@ #include #include + namespace DB { @@ -157,7 +159,8 @@ void executeQuery( const ASTPtr & query_ast, ContextPtr context, const SelectQueryInfo & query_info, const ExpressionActionsPtr & sharding_key_expr, const std::string & sharding_key_column_name, - const ClusterPtr & not_optimized_cluster) + const ClusterPtr & not_optimized_cluster, + AdditionalShardFilterGenerator shard_filter_generator) { const Settings & settings = context->getSettingsRef(); @@ -189,7 +192,22 @@ void executeQuery( visitor.visit(query_ast_for_shard); } else - query_ast_for_shard = query_ast; + query_ast_for_shard = query_ast->clone(); + + if (shard_filter_generator) + { + auto shard_filter = shard_filter_generator(shard_info.shard_num); + if (shard_filter) + { + auto & select_query = query_ast_for_shard->as(); + + auto where_expression = select_query.where(); + if (where_expression) + shard_filter = makeASTFunction("and", where_expression, shard_filter); + + select_query.setExpression(ASTSelectQuery::Expression::WHERE, std::move(shard_filter)); + } + } stream_factory.createForShard(shard_info, query_ast_for_shard, main_table, table_func_ptr, diff --git a/src/Interpreters/ClusterProxy/executeQuery.h b/src/Interpreters/ClusterProxy/executeQuery.h index 787e79313cc..41f6da55686 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.h +++ b/src/Interpreters/ClusterProxy/executeQuery.h @@ -37,6 +37,7 @@ class SelectStreamFactory; ContextMutablePtr updateSettingsForCluster( const Cluster & cluster, ContextPtr context, const Settings & settings, const StorageID & main_table, const SelectQueryInfo * query_info = nullptr, Poco::Logger * log = nullptr); +using AdditionalShardFilterGenerator = std::function; /// Execute a distributed query, creating a query plan, from which the query pipeline can be built. /// `stream_factory` object encapsulates the logic of creating plans for a different type of query /// (currently SELECT, DESCRIBE). @@ -50,7 +51,8 @@ void executeQuery( const ASTPtr & query_ast, ContextPtr context, const SelectQueryInfo & query_info, const ExpressionActionsPtr & sharding_key_expr, const std::string & sharding_key_column_name, - const ClusterPtr & not_optimized_cluster); + const ClusterPtr & not_optimized_cluster, + AdditionalShardFilterGenerator shard_filter_generator = {}); void executeQueryWithParallelReplicas( diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index d1b09707bca..cf1d5203bf7 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -4056,21 +4056,34 @@ std::shared_ptr Context::getAsyncReadCounters() const return async_read_counters; } +Context::ParallelReplicasMode Context::getParallelReplicasMode() const +{ + const auto & settings = getSettingsRef(); + + using enum Context::ParallelReplicasMode; + if (!settings.parallel_replicas_custom_key.value.empty()) + return CUSTOM_KEY; + + if (settings.allow_experimental_parallel_reading_from_replicas + && !settings.use_hedged_requests) + return READ_TASKS; + + return SAMPLE_KEY; +} + bool Context::canUseParallelReplicasOnInitiator() const { const auto & settings = getSettingsRef(); - return settings.allow_experimental_parallel_reading_from_replicas + return getParallelReplicasMode() == ParallelReplicasMode::READ_TASKS && settings.max_parallel_replicas > 1 - && !settings.use_hedged_requests && !getClientInfo().collaborate_with_initiator; } bool Context::canUseParallelReplicasOnFollower() const { const auto & settings = getSettingsRef(); - return settings.allow_experimental_parallel_reading_from_replicas + return getParallelReplicasMode() == ParallelReplicasMode::READ_TASKS && settings.max_parallel_replicas > 1 - && !settings.use_hedged_requests && getClientInfo().collaborate_with_initiator; } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 19bb6868331..67594a41459 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1123,6 +1123,15 @@ public: bool canUseParallelReplicasOnInitiator() const; bool canUseParallelReplicasOnFollower() const; + enum class ParallelReplicasMode : uint8_t + { + SAMPLE_KEY, + CUSTOM_KEY, + READ_TASKS, + }; + + ParallelReplicasMode getParallelReplicasMode() const; + private: std::unique_lock getLock() const; diff --git a/src/Interpreters/GraceHashJoin.cpp b/src/Interpreters/GraceHashJoin.cpp index 79a825a752f..7795061072c 100644 --- a/src/Interpreters/GraceHashJoin.cpp +++ b/src/Interpreters/GraceHashJoin.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include diff --git a/src/Interpreters/GraceHashJoin.h b/src/Interpreters/GraceHashJoin.h index 4f7694e2f07..b8d83f4cad0 100644 --- a/src/Interpreters/GraceHashJoin.h +++ b/src/Interpreters/GraceHashJoin.h @@ -139,6 +139,7 @@ private: mutable SharedMutex rehash_mutex; FileBucket * current_bucket = nullptr; + mutable std::mutex current_bucket_mutex; InMemoryJoinPtr hash_join; diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 29b7a4db609..c352280b7ed 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index 40698386ccb..3330159aff5 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -161,6 +161,8 @@ public: if (curr_process.processed) continue; + LOG_DEBUG(&Poco::Logger::get("KillQuery"), "Will kill query {} (synchronously)", curr_process.query_id); + auto code = process_list.sendCancelToQuery(curr_process.query_id, curr_process.user, true); if (code != CancellationCode::QueryIsNotInitializedYet && code != CancellationCode::CancelSent) @@ -226,6 +228,8 @@ BlockIO InterpreterKillQueryQuery::execute() MutableColumns res_columns = header.cloneEmptyColumns(); for (const auto & query_desc : queries_to_stop) { + if (!query.test) + LOG_DEBUG(&Poco::Logger::get("KillQuery"), "Will kill query {} (asynchronously)", query_desc.query_id); auto code = (query.test) ? CancellationCode::Unknown : process_list.sendCancelToQuery(query_desc.query_id, query_desc.user, true); insertResultRow(query_desc.source_num, code, processes_block, header, res_columns); } diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index a4eed606de1..2f579244b9a 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -114,6 +115,7 @@ namespace ErrorCodes extern const int INVALID_WITH_FILL_EXPRESSION; extern const int ACCESS_DENIED; extern const int UNKNOWN_IDENTIFIER; + extern const int BAD_ARGUMENTS; } /// Assumes `storage` is set and the table filter (row-level security) is not empty. @@ -229,10 +231,13 @@ InterpreterSelectQuery::InterpreterSelectQuery( InterpreterSelectQuery::~InterpreterSelectQuery() = default; +namespace +{ + /** There are no limits on the maximum size of the result for the subquery. * Since the result of the query is not the result of the entire query. */ -static ContextPtr getSubqueryContext(const ContextPtr & context) +ContextPtr getSubqueryContext(const ContextPtr & context) { auto subquery_context = Context::createCopy(context); Settings subquery_settings = context->getSettings(); @@ -244,7 +249,7 @@ static ContextPtr getSubqueryContext(const ContextPtr & context) return subquery_context; } -static void rewriteMultipleJoins(ASTPtr & query, const TablesWithColumns & tables, const String & database, const Settings & settings) +void rewriteMultipleJoins(ASTPtr & query, const TablesWithColumns & tables, const String & database, const Settings & settings) { ASTSelectQuery & select = query->as(); @@ -264,7 +269,7 @@ static void rewriteMultipleJoins(ASTPtr & query, const TablesWithColumns & table } /// Checks that the current user has the SELECT privilege. -static void checkAccessRightsForSelect( +void checkAccessRightsForSelect( const ContextPtr & context, const StorageID & table_id, const StorageMetadataPtr & table_metadata, @@ -294,7 +299,7 @@ static void checkAccessRightsForSelect( context->checkAccess(AccessType::SELECT, table_id, syntax_analyzer_result.requiredSourceColumnsForAccessCheck()); } -static ASTPtr parseAdditionalFilterConditionForTable( +ASTPtr parseAdditionalFilterConditionForTable( const Map & setting, const DatabaseAndTableWithAlias & target, const Context & context) @@ -322,7 +327,7 @@ static ASTPtr parseAdditionalFilterConditionForTable( } /// Returns true if we should ignore quotas and limits for a specified table in the system database. -static bool shouldIgnoreQuotaAndLimits(const StorageID & table_id) +bool shouldIgnoreQuotaAndLimits(const StorageID & table_id) { if (table_id.database_name == DatabaseCatalog::SYSTEM_DATABASE) { @@ -333,6 +338,8 @@ static bool shouldIgnoreQuotaAndLimits(const StorageID & table_id) return false; } +} + InterpreterSelectQuery::InterpreterSelectQuery( const ASTPtr & query_ptr_, const ContextPtr & context_, @@ -448,10 +455,11 @@ InterpreterSelectQuery::InterpreterSelectQuery( } } - if (joined_tables.tablesCount() > 1 && settings.allow_experimental_parallel_reading_from_replicas) + if (joined_tables.tablesCount() > 1 && (!settings.parallel_replicas_custom_key.value.empty() || settings.allow_experimental_parallel_reading_from_replicas)) { LOG_WARNING(log, "Joins are not supported with parallel replicas. Query will be executed without using them."); context->setSetting("allow_experimental_parallel_reading_from_replicas", false); + context->setSetting("parallel_replicas_custom_key", String{""}); } /// Rewrite JOINs @@ -509,6 +517,42 @@ InterpreterSelectQuery::InterpreterSelectQuery( query_info.additional_filter_ast = parseAdditionalFilterConditionForTable( settings.additional_table_filters, joined_tables.tablesWithColumns().front().table, *context); + ASTPtr parallel_replicas_custom_filter_ast = nullptr; + if (context->getParallelReplicasMode() == Context::ParallelReplicasMode::CUSTOM_KEY && !joined_tables.tablesWithColumns().empty()) + { + if (settings.parallel_replicas_count > 1) + { + if (auto custom_key_ast = parseCustomKeyForTable(settings.parallel_replicas_custom_key, *context)) + { + LOG_TRACE(log, "Processing query on a replica using custom_key '{}'", settings.parallel_replicas_custom_key.value); + if (!storage) + throw DB::Exception(ErrorCodes::BAD_ARGUMENTS, "Storage is unknown when trying to parse custom key for parallel replica"); + + parallel_replicas_custom_filter_ast = getCustomKeyFilterForParallelReplica( + settings.parallel_replicas_count, + settings.parallel_replica_offset, + std::move(custom_key_ast), + settings.parallel_replicas_custom_key_filter_type, + *storage, + context); + } + else if (settings.parallel_replica_offset > 0) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Parallel replicas processing with custom_key has been requested " + "(setting 'max_parallel_replicas') but the table does not have custom_key defined for it " + "or it's invalid (settings `parallel_replicas_custom_key`)"); + } + } + else if (auto * distributed = dynamic_cast(storage.get()); + distributed && canUseCustomKey(settings, *distributed->getCluster(), *context)) + { + query_info.use_custom_key = true; + context->setSetting("distributed_group_by_no_merge", 2); + } + } + if (autoFinalOnQuery(query)) { query.setFinal(); @@ -693,6 +737,16 @@ InterpreterSelectQuery::InterpreterSelectQuery( query_info.filter_asts.push_back(query_info.additional_filter_ast); } + if (parallel_replicas_custom_filter_ast) + { + parallel_replicas_custom_filter_info = generateFilterActions( + table_id, parallel_replicas_custom_filter_ast, context, storage, storage_snapshot, metadata_snapshot, required_columns, + prepared_sets); + + parallel_replicas_custom_filter_info->do_remove_column = true; + query_info.filter_asts.push_back(parallel_replicas_custom_filter_ast); + } + source_header = storage_snapshot->getSampleBlockForColumns(required_columns, parameter_values); } @@ -1435,17 +1489,23 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

( + auto filter_step = std::make_unique( query_plan.getCurrentDataStream(), - additional_filter_info->actions, - additional_filter_info->column_name, - additional_filter_info->do_remove_column); + new_filter_info->actions, + new_filter_info->column_name, + new_filter_info->do_remove_column); - additional_filter_step->setStepDescription("Additional filter"); - query_plan.addStep(std::move(additional_filter_step)); - } + filter_step->setStepDescription(description); + query_plan.addStep(std::move(filter_step)); + }; + + if (additional_filter_info) + add_filter_step(additional_filter_info, "Additional filter"); + + if (parallel_replicas_custom_filter_info) + add_filter_step(parallel_replicas_custom_filter_info, "Parallel replica custom key filter"); if (expressions.before_array_join) { diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h index 140e10a5f81..58fddb8ffe9 100644 --- a/src/Interpreters/InterpreterSelectQuery.h +++ b/src/Interpreters/InterpreterSelectQuery.h @@ -215,6 +215,9 @@ private: /// For additional_filter setting. FilterDAGInfoPtr additional_filter_info; + /// For "per replica" filter when multiple replicas are used + FilterDAGInfoPtr parallel_replicas_custom_filter_info; + QueryProcessingStage::Enum from_stage = QueryProcessingStage::FetchColumns; /// List of columns to read to execute the query. diff --git a/src/Interpreters/MergeTreeTransaction.cpp b/src/Interpreters/MergeTreeTransaction.cpp index 50ecb061752..bfdda354c9b 100644 --- a/src/Interpreters/MergeTreeTransaction.cpp +++ b/src/Interpreters/MergeTreeTransaction.cpp @@ -168,6 +168,8 @@ void MergeTreeTransaction::addMutation(const StoragePtr & table, const String & bool MergeTreeTransaction::isReadOnly() const { std::lock_guard lock{mutex}; + if (finalized) + return is_read_only; chassert((creating_parts.empty() && removing_parts.empty() && mutations.empty()) == storages.empty()); return storages.empty(); } @@ -318,6 +320,11 @@ bool MergeTreeTransaction::rollback() noexcept void MergeTreeTransaction::afterFinalize() { std::lock_guard lock{mutex}; + chassert((creating_parts.empty() && removing_parts.empty() && mutations.empty()) == storages.empty()); + + /// Remember if it was read-only transaction before we clear storages + is_read_only = storages.empty(); + /// Release shared pointers just in case storages.clear(); mutations.clear(); diff --git a/src/Interpreters/MergeTreeTransaction.h b/src/Interpreters/MergeTreeTransaction.h index e5a80e03e18..4ca36cf64ad 100644 --- a/src/Interpreters/MergeTreeTransaction.h +++ b/src/Interpreters/MergeTreeTransaction.h @@ -78,6 +78,9 @@ private: bool finalized TSA_GUARDED_BY(mutex) = false; + /// Indicates if transaction was read-only before `afterFinalize` + bool is_read_only TSA_GUARDED_BY(mutex) = false; + /// Lists of changes made by transaction std::unordered_set storages TSA_GUARDED_BY(mutex); DataPartsVector creating_parts TSA_GUARDED_BY(mutex); diff --git a/src/Interpreters/OpenTelemetrySpanLog.cpp b/src/Interpreters/OpenTelemetrySpanLog.cpp index 57d5c11ad97..63b8ae406a6 100644 --- a/src/Interpreters/OpenTelemetrySpanLog.cpp +++ b/src/Interpreters/OpenTelemetrySpanLog.cpp @@ -10,7 +10,7 @@ #include #include -#include +#include #include #include diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 49d7989ac5e..b792ea538ae 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -362,7 +362,11 @@ QueryStatus::QueryStatus( QueryStatus::~QueryStatus() { - assert(executors.empty()); +#if !defined(NDEBUG) + /// Check that all executors were invalidated. + for (const auto & e : executors) + assert(!e->executor); +#endif if (auto * memory_tracker = getMemoryTracker()) { @@ -373,6 +377,19 @@ QueryStatus::~QueryStatus() } } +void QueryStatus::ExecutorHolder::cancel() +{ + std::lock_guard lock(mutex); + if (executor) + executor->cancel(); +} + +void QueryStatus::ExecutorHolder::remove() +{ + std::lock_guard lock(mutex); + executor = nullptr; +} + CancellationCode QueryStatus::cancelQuery(bool) { if (is_killed.load()) @@ -380,8 +397,25 @@ CancellationCode QueryStatus::cancelQuery(bool) is_killed.store(true); - std::lock_guard lock(executors_mutex); - for (auto * e : executors) + std::vector executors_snapshot; + + { + /// Create a snapshot of executors under a mutex. + std::lock_guard lock(executors_mutex); + executors_snapshot = executors; + } + + /// We should call cancel() for each executor with unlocked executors_mutex, because + /// cancel() can try to lock some internal mutex that is already locked by query executing + /// thread, and query executing thread can call removePipelineExecutor and lock executors_mutex, + /// which will lead to deadlock. + /// Note that the size and the content of executors cannot be changed while + /// executors_mutex is unlocked, because: + /// 1) We don't allow adding new executors while cancelling query in addPipelineExecutor + /// 2) We don't actually remove executor holder from executors in removePipelineExecutor, + /// just mark that executor is invalid. + /// So, it's ok to use a snapshot created above under a mutex, it won't be any differ from actual executors. + for (const auto & e : executors_snapshot) e->cancel(); return CancellationCode::CancelSent; @@ -396,15 +430,17 @@ void QueryStatus::addPipelineExecutor(PipelineExecutor * e) throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); std::lock_guard lock(executors_mutex); - assert(std::find(executors.begin(), executors.end(), e) == executors.end()); - executors.push_back(e); + assert(std::find_if(executors.begin(), executors.end(), [e](const ExecutorHolderPtr & x){ return x->executor == e; }) == executors.end()); + executors.push_back(std::make_shared(e)); } void QueryStatus::removePipelineExecutor(PipelineExecutor * e) { std::lock_guard lock(executors_mutex); - assert(std::find(executors.begin(), executors.end(), e) != executors.end()); - std::erase_if(executors, [e](PipelineExecutor * x) { return x == e; }); + auto it = std::find_if(executors.begin(), executors.end(), [e](const ExecutorHolderPtr & x){ return x->executor == e; }); + assert(it != executors.end()); + /// Invalidate executor pointer inside holder, but don't remove holder from the executors (to avoid race with cancelQuery) + (*it)->remove(); } bool QueryStatus::checkTimeLimit() diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index d5c136ab62a..30bfde4e218 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -119,8 +119,22 @@ protected: mutable std::mutex executors_mutex; + struct ExecutorHolder + { + ExecutorHolder(PipelineExecutor * e) : executor(e) {} + + void cancel(); + + void remove(); + + PipelineExecutor * executor; + std::mutex mutex; + }; + + using ExecutorHolderPtr = std::shared_ptr; + /// Array of PipelineExecutors to be cancelled when a cancelQuery is received - std::vector executors; + std::vector executors; enum QueryStreamsStatus { diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp index 9d8547abcf2..78513920236 100644 --- a/src/Interpreters/SystemLog.cpp +++ b/src/Interpreters/SystemLog.cpp @@ -426,6 +426,8 @@ void SystemLog::flushImpl(const std::vector & to_flush, // we need query context to do inserts to target table with MV containing subqueries or joins auto insert_context = Context::createCopy(context); insert_context->makeQueryContext(); + /// We always want to deliver the data to the original table regardless of the MVs + insert_context->setSetting("materialized_views_ignore_errors", true); InterpreterInsertQuery interpreter(query_ptr, insert_context); BlockIO io = interpreter.execute(); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index e2aa2c02fc8..076b0c08d93 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -998,7 +998,7 @@ static std::tuple executeQueryImpl( { double elapsed_seconds = static_cast(info.elapsed_microseconds) / 1000000.0; double rows_per_second = static_cast(elem.read_rows) / elapsed_seconds; - LOG_INFO( + LOG_DEBUG( &Poco::Logger::get("executeQuery"), "Read {} rows, {} in {} sec., {} rows/sec., {}/sec.", elem.read_rows, diff --git a/src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp b/src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp new file mode 100644 index 00000000000..2a32d450497 --- /dev/null +++ b/src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp @@ -0,0 +1,134 @@ +#include + +#include +#include +#include +#include + +#include + +#include + +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER; +} + +bool canUseCustomKey(const Settings & settings, const Cluster & cluster, const Context & context) +{ + return settings.max_parallel_replicas > 1 && context.getParallelReplicasMode() == Context::ParallelReplicasMode::CUSTOM_KEY + && cluster.getShardCount() == 1 && cluster.getShardsInfo()[0].getAllNodeCount() > 1; +} + +ASTPtr getCustomKeyFilterForParallelReplica( + size_t replicas_count, + size_t replica_num, + ASTPtr custom_key_ast, + ParallelReplicasCustomKeyFilterType filter_type, + const IStorage & storage, + const ContextPtr & context) +{ + assert(replicas_count > 1); + if (filter_type == ParallelReplicasCustomKeyFilterType::DEFAULT) + { + // first we do modulo with replica count + auto modulo_function = makeASTFunction("positiveModulo", custom_key_ast, std::make_shared(replicas_count)); + + /// then we compare result to the current replica number (offset) + auto equals_function = makeASTFunction("equals", std::move(modulo_function), std::make_shared(replica_num)); + + return equals_function; + } + + assert(filter_type == ParallelReplicasCustomKeyFilterType::RANGE); + + KeyDescription custom_key_description + = KeyDescription::getKeyFromAST(custom_key_ast, storage.getInMemoryMetadataPtr()->columns, context); + + using RelativeSize = boost::rational; + + RelativeSize size_of_universum = 0; + DataTypePtr custom_key_column_type = custom_key_description.data_types[0]; + + size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); + if (custom_key_description.data_types.size() == 1) + { + if (typeid_cast(custom_key_column_type.get())) + size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); + else if (typeid_cast(custom_key_column_type.get())) + size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); + else if (typeid_cast(custom_key_column_type.get())) + size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); + else if (typeid_cast(custom_key_column_type.get())) + size_of_universum = RelativeSize(std::numeric_limits::max()) + RelativeSize(1); + } + + if (size_of_universum == RelativeSize(0)) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER, + "Invalid custom key column type: {}. Must be one unsigned integer type", + custom_key_column_type->getName()); + + RelativeSize relative_range_size = RelativeSize(1) / replicas_count; + RelativeSize relative_range_offset = relative_range_size * RelativeSize(replica_num); + + /// Calculate the half-interval of `[lower, upper)` column values. + bool has_lower_limit = false; + bool has_upper_limit = false; + + RelativeSize lower_limit_rational = relative_range_offset * size_of_universum; + RelativeSize upper_limit_rational = (relative_range_offset + relative_range_size) * size_of_universum; + + UInt64 lower = boost::rational_cast(lower_limit_rational); + UInt64 upper = boost::rational_cast(upper_limit_rational); + + if (lower > 0) + has_lower_limit = true; + + if (upper_limit_rational < size_of_universum) + has_upper_limit = true; + + assert(has_lower_limit || has_upper_limit); + + /// Let's add the conditions to cut off something else when the index is scanned again and when the request is processed. + std::shared_ptr lower_function; + std::shared_ptr upper_function; + + if (has_lower_limit) + { + lower_function = makeASTFunction("greaterOrEquals", custom_key_ast, std::make_shared(lower)); + + if (!has_upper_limit) + return lower_function; + } + + if (has_upper_limit) + { + upper_function = makeASTFunction("less", custom_key_ast, std::make_shared(upper)); + + if (!has_lower_limit) + return upper_function; + } + + assert(upper_function && lower_function); + + return makeASTFunction("and", std::move(lower_function), std::move(upper_function)); +} + +ASTPtr parseCustomKeyForTable(const String & custom_key, const Context & context) +{ + /// Try to parse expression + ParserExpression parser; + const auto & settings = context.getSettingsRef(); + return parseQuery( + parser, custom_key.data(), custom_key.data() + custom_key.size(), + "parallel replicas custom key", settings.max_query_size, settings.max_parser_depth); +} + +} diff --git a/src/Interpreters/getCustomKeyFilterForParallelReplicas.h b/src/Interpreters/getCustomKeyFilterForParallelReplicas.h new file mode 100644 index 00000000000..543f1889b32 --- /dev/null +++ b/src/Interpreters/getCustomKeyFilterForParallelReplicas.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +bool canUseCustomKey(const Settings & settings, const Cluster & cluster, const Context & context); + +/// Get AST for filter created from custom_key +/// replica_num is the number of the replica for which we are generating filter starting from 0 +ASTPtr getCustomKeyFilterForParallelReplica( + size_t replicas_count, + size_t replica_num, + ASTPtr custom_key_ast, + ParallelReplicasCustomKeyFilterType filter_type, + const IStorage & storage, + const ContextPtr & context); + +ASTPtr parseCustomKeyForTable(const String & custom_keys, const Context & context); + +} diff --git a/src/Interpreters/tests/gtest_lru_file_cache.cpp b/src/Interpreters/tests/gtest_lru_file_cache.cpp index 93faafb5cea..62aef2441d6 100644 --- a/src/Interpreters/tests/gtest_lru_file_cache.cpp +++ b/src/Interpreters/tests/gtest_lru_file_cache.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Parsers/Access/ParserCreateUserQuery.cpp b/src/Parsers/Access/ParserCreateUserQuery.cpp index de83c5760c1..c1d0691d305 100644 --- a/src/Parsers/Access/ParserCreateUserQuery.cpp +++ b/src/Parsers/Access/ParserCreateUserQuery.cpp @@ -17,7 +17,7 @@ #include #include #include "config.h" -#include +#include #if USE_SSL # include # include diff --git a/src/Parsers/FieldFromAST.cpp b/src/Parsers/FieldFromAST.cpp index 3cd10c1cf80..a81bf45a8be 100644 --- a/src/Parsers/FieldFromAST.cpp +++ b/src/Parsers/FieldFromAST.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -31,42 +32,64 @@ bool FieldFromASTImpl::isSecret() const return isDiskFunction(ast); } +class DiskConfigurationMasker +{ +public: + struct Data {}; + + static bool needChildVisit(const ASTPtr &, const ASTPtr &) { return true; } + + static void visit(ASTPtr & ast, Data &) + { + if (isDiskFunction(ast)) + { + const auto & disk_function = assert_cast(*ast); + const auto * disk_function_args_expr = assert_cast(disk_function.arguments.get()); + const auto & disk_function_args = disk_function_args_expr->children; + + auto is_secret_arg = [](const std::string & arg_name) + { + /// We allow to not hide type of the disk, e.g. disk(type = s3, ...) + /// and also nested disk, e.g. disk = 'disk_name' + return arg_name != "type" && arg_name != "disk"; + }; + + for (const auto & arg : disk_function_args) + { + auto * setting_function = arg->as(); + if (!setting_function || setting_function->name != "equals") + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected equals function"); + + auto * function_args_expr = assert_cast(setting_function->arguments.get()); + if (!function_args_expr) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected arguments"); + + auto & function_args = function_args_expr->children; + if (function_args.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected non zero number of arguments"); + + auto * key_identifier = function_args[0]->as(); + if (!key_identifier) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected Identifier"); + + const std::string & key = key_identifier->name(); + if (is_secret_arg(key)) + function_args[1] = std::make_shared("[HIDDEN]"); + } + } + } +}; + +/// Visits children first. +using HideDiskConfigurationVisitor = InDepthNodeVisitor; + String FieldFromASTImpl::toString(bool show_secrets) const { if (!show_secrets && isDiskFunction(ast)) { auto hidden = ast->clone(); - const auto & disk_function = assert_cast(*hidden); - const auto * disk_function_args_expr = assert_cast(disk_function.arguments.get()); - const auto & disk_function_args = disk_function_args_expr->children; - - auto is_secret_arg = [](const std::string & arg_name) - { - return arg_name != "type"; - }; - - for (const auto & arg : disk_function_args) - { - auto * setting_function = arg->as(); - if (!setting_function || setting_function->name != "equals") - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected equals function"); - - auto * function_args_expr = assert_cast(setting_function->arguments.get()); - if (!function_args_expr) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected arguments"); - - auto & function_args = function_args_expr->children; - if (function_args.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected non zero number of arguments"); - - auto * key_identifier = function_args[0]->as(); - if (!key_identifier) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad format: expected Identifier"); - - const std::string & key = key_identifier->name(); - if (is_secret_arg(key)) - function_args[1] = std::make_shared("[HIDDEN]"); - } + HideDiskConfigurationVisitor::Data data{}; + HideDiskConfigurationVisitor{data}.visit(hidden); return serializeAST(*hidden); } diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp index 614e5c03bc1..ac524a011a9 100644 --- a/src/Planner/PlannerActionsVisitor.cpp +++ b/src/Planner/PlannerActionsVisitor.cpp @@ -132,7 +132,7 @@ public: } template - const ActionsDAG::Node * addFunctionIfNecessary(const std::string & node_name, ActionsDAG::NodeRawConstPtrs children, FunctionOrOverloadResolver function) + const ActionsDAG::Node * addFunctionIfNecessary(const std::string & node_name, ActionsDAG::NodeRawConstPtrs children, const FunctionOrOverloadResolver & function) { auto it = node_name_to_node.find(node_name); if (it != node_name_to_node.end()) @@ -339,7 +339,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi actions_stack.pop_back(); // TODO: Pass IFunctionBase here not FunctionCaptureOverloadResolver. - actions_stack[level].addFunctionIfNecessary(lambda_node_name, std::move(lambda_children), std::move(function_capture)); + actions_stack[level].addFunctionIfNecessary(lambda_node_name, std::move(lambda_children), function_capture); size_t actions_stack_size = actions_stack.size(); for (size_t i = level + 1; i < actions_stack_size; ++i) @@ -501,7 +501,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi } else { - actions_stack[level].addFunctionIfNecessary(function_node_name, children, function_node.getFunction()); + actions_stack[level].addFunctionIfNecessary(function_node_name, children, function_node); } size_t actions_stack_size = actions_stack.size(); diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp index 0e97e69cd67..9a7340f936c 100644 --- a/src/Planner/PlannerExpressionAnalysis.cpp +++ b/src/Planner/PlannerExpressionAnalysis.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include @@ -33,12 +34,11 @@ namespace * It is client responsibility to update filter analysis result if filter column must be removed after chain is finalized. */ FilterAnalysisResult analyzeFilter(const QueryTreeNodePtr & filter_expression_node, - const ColumnsWithTypeAndName & join_tree_input_columns, + const ColumnsWithTypeAndName & current_output_columns, const PlannerContextPtr & planner_context, ActionsChain & actions_chain) { - const auto * chain_available_output_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); - const auto & filter_input = chain_available_output_columns ? *chain_available_output_columns : join_tree_input_columns; + const auto & filter_input = current_output_columns; FilterAnalysisResult result; @@ -52,8 +52,8 @@ FilterAnalysisResult analyzeFilter(const QueryTreeNodePtr & filter_expression_no /** Construct aggregation analysis result if query tree has GROUP BY or aggregates. * Actions before aggregation are added into actions chain, if result is not null optional. */ -std::optional analyzeAggregation(const QueryTreeNodePtr & query_tree, - const ColumnsWithTypeAndName & join_tree_input_columns, +std::pair, std::optional> analyzeAggregation(const QueryTreeNodePtr & query_tree, + const ColumnsWithTypeAndName & current_output_columns, const PlannerContextPtr & planner_context, ActionsChain & actions_chain) { @@ -69,8 +69,7 @@ std::optional analyzeAggregation(const QueryTreeNodeP Names aggregation_keys; - const auto * chain_available_output_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); - const auto & group_by_input = chain_available_output_columns ? *chain_available_output_columns : join_tree_input_columns; + const auto & group_by_input = current_output_columns; ActionsDAGPtr before_aggregation_actions = std::make_shared(group_by_input); before_aggregation_actions->getOutputs().clear(); @@ -83,6 +82,8 @@ std::optional analyzeAggregation(const QueryTreeNodeP PlannerActionsVisitor actions_visitor(planner_context); /// Add expressions from GROUP BY + bool group_by_use_nulls = planner_context->getQueryContext()->getSettingsRef().group_by_use_nulls && + (query_node.isGroupByWithGroupingSets() || query_node.isGroupByWithRollup() || query_node.isGroupByWithCube()); if (query_node.hasGroupBy()) { @@ -107,6 +108,8 @@ std::optional analyzeAggregation(const QueryTreeNodeP if (before_aggregation_actions_output_node_names.contains(expression_dag_node->result_name)) continue; + auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type; + available_columns_after_aggregation.emplace_back(nullptr, expression_type_after_aggregation, expression_dag_node->result_name); aggregation_keys.push_back(expression_dag_node->result_name); before_aggregation_actions->getOutputs().push_back(expression_dag_node); before_aggregation_actions_output_node_names.insert(expression_dag_node->result_name); @@ -150,6 +153,8 @@ std::optional analyzeAggregation(const QueryTreeNodeP if (before_aggregation_actions_output_node_names.contains(expression_dag_node->result_name)) continue; + auto expression_type_after_aggregation = group_by_use_nulls ? makeNullableSafe(expression_dag_node->result_type) : expression_dag_node->result_type; + available_columns_after_aggregation.emplace_back(nullptr, expression_type_after_aggregation, expression_dag_node->result_name); aggregation_keys.push_back(expression_dag_node->result_name); before_aggregation_actions->getOutputs().push_back(expression_dag_node); before_aggregation_actions_output_node_names.insert(expression_dag_node->result_name); @@ -157,9 +162,6 @@ std::optional analyzeAggregation(const QueryTreeNodeP } } - for (auto & node : before_aggregation_actions->getOutputs()) - available_columns_after_aggregation.emplace_back(nullptr, node->result_type, node->result_name); - /// Add expressions from aggregate functions arguments for (auto & aggregate_function_node : aggregate_function_nodes) @@ -201,14 +203,14 @@ std::optional analyzeAggregation(const QueryTreeNodeP aggregation_analysis_result.grouping_sets_parameters_list = std::move(grouping_sets_parameters_list); aggregation_analysis_result.group_by_with_constant_keys = group_by_with_constant_keys; - return aggregation_analysis_result; + return { aggregation_analysis_result, available_columns_after_aggregation }; } /** Construct window analysis result if query tree has window functions. * Actions before window functions are added into actions chain, if result is not null optional. */ std::optional analyzeWindow(const QueryTreeNodePtr & query_tree, - const ColumnsWithTypeAndName & join_tree_input_columns, + const ColumnsWithTypeAndName & current_output_columns, const PlannerContextPtr & planner_context, ActionsChain & actions_chain) { @@ -218,8 +220,7 @@ std::optional analyzeWindow(const QueryTreeNodePtr & query auto window_descriptions = extractWindowDescriptions(window_function_nodes, *planner_context); - const auto * chain_available_output_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); - const auto & window_input = chain_available_output_columns ? *chain_available_output_columns : join_tree_input_columns; + const auto & window_input = current_output_columns; PlannerActionsVisitor actions_visitor(planner_context); @@ -298,12 +299,11 @@ std::optional analyzeWindow(const QueryTreeNodePtr & query * It is client responsibility to update projection analysis result with project names actions after chain is finalized. */ ProjectionAnalysisResult analyzeProjection(const QueryNode & query_node, - const ColumnsWithTypeAndName & join_tree_input_columns, + const ColumnsWithTypeAndName & current_output_columns, const PlannerContextPtr & planner_context, ActionsChain & actions_chain) { - const auto * chain_available_output_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); - const auto & projection_input = chain_available_output_columns ? *chain_available_output_columns : join_tree_input_columns; + const auto & projection_input = current_output_columns; auto projection_actions = buildActionsDAGFromExpressionNode(query_node.getProjectionNode(), projection_input, planner_context); auto projection_columns = query_node.getProjectionColumns(); @@ -347,12 +347,11 @@ ProjectionAnalysisResult analyzeProjection(const QueryNode & query_node, * Actions before sort are added into actions chain. */ SortAnalysisResult analyzeSort(const QueryNode & query_node, - const ColumnsWithTypeAndName & join_tree_input_columns, + const ColumnsWithTypeAndName & current_output_columns, const PlannerContextPtr & planner_context, ActionsChain & actions_chain) { - const auto * chain_available_output_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); - const auto & order_by_input = chain_available_output_columns ? *chain_available_output_columns : join_tree_input_columns; + const auto & order_by_input = current_output_columns; ActionsDAGPtr before_sort_actions = std::make_shared(order_by_input); auto & before_sort_actions_outputs = before_sort_actions->getOutputs(); @@ -437,13 +436,12 @@ SortAnalysisResult analyzeSort(const QueryNode & query_node, * Actions before limit by are added into actions chain. */ LimitByAnalysisResult analyzeLimitBy(const QueryNode & query_node, - const ColumnsWithTypeAndName & join_tree_input_columns, + const ColumnsWithTypeAndName & current_output_columns, const PlannerContextPtr & planner_context, const NameSet & required_output_nodes_names, ActionsChain & actions_chain) { - const auto * chain_available_output_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); - const auto & limit_by_input = chain_available_output_columns ? *chain_available_output_columns : join_tree_input_columns; + const auto & limit_by_input = current_output_columns; auto before_limit_by_actions = buildActionsDAGFromExpressionNode(query_node.getLimitByNode(), limit_by_input, planner_context); NameSet limit_by_column_names_set; @@ -482,29 +480,43 @@ PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNo std::optional where_analysis_result_optional; std::optional where_action_step_index_optional; + const auto * input_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); + ColumnsWithTypeAndName current_output_columns = input_columns ? *input_columns : join_tree_input_columns; + if (query_node.hasWhere()) { - where_analysis_result_optional = analyzeFilter(query_node.getWhere(), join_tree_input_columns, planner_context, actions_chain); + where_analysis_result_optional = analyzeFilter(query_node.getWhere(), current_output_columns, planner_context, actions_chain); where_action_step_index_optional = actions_chain.getLastStepIndex(); + current_output_columns = actions_chain.getLastStepAvailableOutputColumns(); } - auto aggregation_analysis_result_optional = analyzeAggregation(query_tree, join_tree_input_columns, planner_context, actions_chain); + auto [aggregation_analysis_result_optional, aggregated_columns_optional] = analyzeAggregation(query_tree, current_output_columns, planner_context, actions_chain); + if (aggregated_columns_optional) + current_output_columns = std::move(*aggregated_columns_optional); std::optional having_analysis_result_optional; std::optional having_action_step_index_optional; if (query_node.hasHaving()) { - having_analysis_result_optional = analyzeFilter(query_node.getHaving(), join_tree_input_columns, planner_context, actions_chain); + having_analysis_result_optional = analyzeFilter(query_node.getHaving(), current_output_columns, planner_context, actions_chain); having_action_step_index_optional = actions_chain.getLastStepIndex(); + current_output_columns = actions_chain.getLastStepAvailableOutputColumns(); } - auto window_analysis_result_optional = analyzeWindow(query_tree, join_tree_input_columns, planner_context, actions_chain); - auto projection_analysis_result = analyzeProjection(query_node, join_tree_input_columns, planner_context, actions_chain); + auto window_analysis_result_optional = analyzeWindow(query_tree, current_output_columns, planner_context, actions_chain); + if (window_analysis_result_optional) + current_output_columns = actions_chain.getLastStepAvailableOutputColumns(); + + auto projection_analysis_result = analyzeProjection(query_node, current_output_columns, planner_context, actions_chain); + current_output_columns = actions_chain.getLastStepAvailableOutputColumns(); std::optional sort_analysis_result_optional; if (query_node.hasOrderBy()) - sort_analysis_result_optional = analyzeSort(query_node, join_tree_input_columns, planner_context, actions_chain); + { + sort_analysis_result_optional = analyzeSort(query_node, current_output_columns, planner_context, actions_chain); + current_output_columns = actions_chain.getLastStepAvailableOutputColumns(); + } std::optional limit_by_analysis_result_optional; @@ -526,14 +538,15 @@ PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNo } limit_by_analysis_result_optional = analyzeLimitBy(query_node, - join_tree_input_columns, + current_output_columns, planner_context, required_output_nodes_names, actions_chain); + current_output_columns = actions_chain.getLastStepAvailableOutputColumns(); } const auto * chain_available_output_columns = actions_chain.getLastStepAvailableOutputColumnsOrNull(); - auto project_names_input = chain_available_output_columns ? *chain_available_output_columns : join_tree_input_columns; + auto project_names_input = chain_available_output_columns ? *chain_available_output_columns : current_output_columns; bool has_with_fill = sort_analysis_result_optional.has_value() && sort_analysis_result_optional->has_with_fill; /** If there is WITH FILL we must use non constant projection columns. diff --git a/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp b/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp index 70815bb8b3b..4478f1548a4 100644 --- a/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp +++ b/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp @@ -187,7 +187,7 @@ void PushingAsyncPipelineExecutor::push(Chunk chunk) if (!is_pushed) throw Exception(ErrorCodes::LOGICAL_ERROR, - "Pipeline for PushingPipelineExecutor was finished before all data was inserted"); + "Pipeline for PushingAsyncPipelineExecutor was finished before all data was inserted"); } void PushingAsyncPipelineExecutor::push(Block block) diff --git a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp index bf0e2448082..c85c0342c8c 100644 --- a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp @@ -17,6 +17,24 @@ namespace ErrorCodes extern const int UNKNOWN_EXCEPTION; } +namespace +{ + +arrow::Compression::type getArrowCompression(FormatSettings::ArrowCompression method) +{ + switch (method) + { + case FormatSettings::ArrowCompression::NONE: + return arrow::Compression::type::UNCOMPRESSED; + case FormatSettings::ArrowCompression::ZSTD: + return arrow::Compression::type::ZSTD; + case FormatSettings::ArrowCompression::LZ4_FRAME: + return arrow::Compression::type::LZ4_FRAME; + } +} + +} + ArrowBlockOutputFormat::ArrowBlockOutputFormat(WriteBuffer & out_, const Block & header_, bool stream_, const FormatSettings & format_settings_) : IOutputFormat(header_, out_) , stream{stream_} @@ -78,12 +96,14 @@ void ArrowBlockOutputFormat::prepareWriter(const std::shared_ptr { arrow_ostream = std::make_shared(out); arrow::Result> writer_status; + arrow::ipc::IpcWriteOptions options = arrow::ipc::IpcWriteOptions::Defaults(); + options.codec = *arrow::util::Codec::Create(getArrowCompression(format_settings.arrow.output_compression_method)); // TODO: should we use arrow::ipc::IpcOptions::alignment? if (stream) - writer_status = arrow::ipc::MakeStreamWriter(arrow_ostream.get(), schema); + writer_status = arrow::ipc::MakeStreamWriter(arrow_ostream.get(), schema, options); else - writer_status = arrow::ipc::MakeFileWriter(arrow_ostream.get(), schema); + writer_status = arrow::ipc::MakeFileWriter(arrow_ostream.get(), schema,options); if (!writer_status.ok()) throw Exception(ErrorCodes::UNKNOWN_EXCEPTION, diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp index 86d9560beb9..67345a0dfb0 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp @@ -28,6 +28,34 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_COLUMN; + extern const int NOT_IMPLEMENTED; +} + +namespace +{ + +orc::CompressionKind getORCCompression(FormatSettings::ORCCompression method) +{ + if (method == FormatSettings::ORCCompression::NONE) + return orc::CompressionKind::CompressionKind_NONE; + +#if USE_SNAPPY + if (method == FormatSettings::ORCCompression::SNAPPY) + return orc::CompressionKind::CompressionKind_SNAPPY; +#endif + + if (method == FormatSettings::ORCCompression::ZSTD) + return orc::CompressionKind::CompressionKind_ZSTD; + + if (method == FormatSettings::ORCCompression::LZ4) + return orc::CompressionKind::CompressionKind_LZ4; + + if (method == FormatSettings::ORCCompression::ZLIB) + return orc::CompressionKind::CompressionKind_ZLIB; + + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported compression method"); +} + } ORCOutputStream::ORCOutputStream(WriteBuffer & out_) : out(out_) {} @@ -544,7 +572,7 @@ void ORCBlockOutputFormat::prepareWriter() { const Block & header = getPort(PortKind::Main).getHeader(); schema = orc::createStructType(); - options.setCompression(orc::CompressionKind::CompressionKind_NONE); + options.setCompression(getORCCompression(format_settings.orc.output_compression_method)); size_t columns_count = header.columns(); for (size_t i = 0; i != columns_count; ++i) schema->addStructField(header.safeGetByPosition(i).name, getORCType(recursiveRemoveLowCardinality(data_types[i]))); diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 18c81f8fd6a..759f773a574 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -14,9 +14,13 @@ namespace DB namespace ErrorCodes { extern const int UNKNOWN_EXCEPTION; + extern const int NOT_IMPLEMENTED; } -static parquet::ParquetVersion::type getParquetVersion(const FormatSettings & settings) +namespace +{ + +parquet::ParquetVersion::type getParquetVersion(const FormatSettings & settings) { switch (settings.parquet.output_version) { @@ -31,6 +35,35 @@ static parquet::ParquetVersion::type getParquetVersion(const FormatSettings & se } } +parquet::Compression::type getParquetCompression(FormatSettings::ParquetCompression method) +{ + if (method == FormatSettings::ParquetCompression::NONE) + return parquet::Compression::type::UNCOMPRESSED; + +#if USE_SNAPPY + if (method == FormatSettings::ParquetCompression::SNAPPY) + return parquet::Compression::type::SNAPPY; +#endif + +#if USE_BROTLI + if (method == FormatSettings::ParquetCompression::BROTLI) + return parquet::Compression::type::BROTLI; +#endif + + if (method == FormatSettings::ParquetCompression::ZSTD) + return parquet::Compression::type::ZSTD; + + if (method == FormatSettings::ParquetCompression::LZ4) + return parquet::Compression::type::LZ4; + + if (method == FormatSettings::ParquetCompression::GZIP) + return parquet::Compression::type::GZIP; + + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported compression method"); +} + +} + ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, const Block & header_, const FormatSettings & format_settings_) : IOutputFormat(header_, out_), format_settings{format_settings_} { @@ -60,9 +93,7 @@ void ParquetBlockOutputFormat::consume(Chunk chunk) parquet::WriterProperties::Builder builder; builder.version(getParquetVersion(format_settings)); -#if USE_SNAPPY - builder.compression(parquet::Compression::SNAPPY); -#endif + builder.compression(getParquetCompression(format_settings.parquet.output_compression_method)); auto props = builder.build(); auto status = parquet::arrow::FileWriter::Open( *arrow_table->schema(), diff --git a/src/Processors/QueryPlan/JoinStep.h b/src/Processors/QueryPlan/JoinStep.h index a814d541574..e7185f36588 100644 --- a/src/Processors/QueryPlan/JoinStep.h +++ b/src/Processors/QueryPlan/JoinStep.h @@ -49,6 +49,8 @@ public: String getName() const override { return "FilledJoin"; } void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; + const JoinPtr & getJoin() const { return join; } + private: void updateOutputStream() override; diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index d466c52725f..37bc894339f 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -314,11 +314,14 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes if (auto updated_steps = simplePushDownOverStep(parent_node, nodes, child)) return updated_steps; - if (auto * join = typeid_cast(child.get())) + auto * join = typeid_cast(child.get()); + auto * filled_join = typeid_cast(child.get()); + + if (join || filled_join) { auto join_push_down = [&](JoinKind kind) -> size_t { - const auto & table_join = join->getJoin()->getTableJoin(); + const auto & table_join = join ? join->getJoin()->getTableJoin() : filled_join->getJoin()->getTableJoin(); /// Only inner and left(/right) join are supported. Other types may generate default values for left table keys. /// So, if we push down a condition like `key != 0`, not all rows may be filtered. @@ -326,8 +329,8 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes return 0; bool is_left = kind == JoinKind::Left; - const auto & input_header = is_left ? join->getInputStreams().front().header : join->getInputStreams().back().header; - const auto & res_header = join->getOutputStream().header; + const auto & input_header = is_left ? child->getInputStreams().front().header : child->getInputStreams().back().header; + const auto & res_header = child->getOutputStream().header; Names allowed_keys; const auto & source_columns = input_header.getNames(); for (const auto & name : source_columns) @@ -372,7 +375,7 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes return updated_steps; /// For full sorting merge join we push down both to the left and right tables, because left and right streams are not independent. - if (join->allowPushDownToRight()) + if (join && join->allowPushDownToRight()) { if (size_t updated_steps = join_push_down(JoinKind::Right)) return updated_steps; diff --git a/src/Processors/Transforms/JoiningTransform.cpp b/src/Processors/Transforms/JoiningTransform.cpp index c28a84e9d5d..bba8ec6fa16 100644 --- a/src/Processors/Transforms/JoiningTransform.cpp +++ b/src/Processors/Transforms/JoiningTransform.cpp @@ -318,13 +318,24 @@ DelayedJoinedBlocksWorkerTransform::DelayedJoinedBlocksWorkerTransform(Block out IProcessor::Status DelayedJoinedBlocksWorkerTransform::prepare() { + auto & output = outputs.front(); + auto & input = inputs.front(); + + if (output.isFinished()) + { + input.close(); + return Status::Finished; + } + + if (!output.canPush()) + { + input.setNotNeeded(); + return Status::PortFull; + } + if (inputs.size() != 1 && outputs.size() != 1) throw Exception(ErrorCodes::LOGICAL_ERROR, "DelayedJoinedBlocksWorkerTransform must have exactly one input port"); - auto & output = outputs.front(); - - auto & input = inputs.front(); - if (output_chunk) { input.setNotNeeded(); @@ -397,15 +408,25 @@ DelayedJoinedBlocksTransform::DelayedJoinedBlocksTransform(size_t num_streams, J void DelayedJoinedBlocksTransform::work() { + if (finished) + return; + delayed_blocks = join->getDelayedBlocks(); finished = finished || delayed_blocks == nullptr; } - IProcessor::Status DelayedJoinedBlocksTransform::prepare() { for (auto & output : outputs) { + if (output.isFinished()) + { + /// If at least one output is finished, then we have read all data from buckets. + /// Some workers can still be busy with joining the last chunk of data in memory, + /// but after that they also will finish when they will try to get next chunk. + finished = true; + continue; + } if (!output.canPush()) return Status::PortFull; } @@ -414,6 +435,8 @@ IProcessor::Status DelayedJoinedBlocksTransform::prepare() { for (auto & output : outputs) { + if (output.isFinished()) + continue; Chunk chunk; chunk.setChunkInfo(std::make_shared()); output.push(std::move(chunk)); diff --git a/src/Processors/Transforms/buildPushingToViewsChain.cpp b/src/Processors/Transforms/buildPushingToViewsChain.cpp index 870106d794f..c27e73804ad 100644 --- a/src/Processors/Transforms/buildPushingToViewsChain.cpp +++ b/src/Processors/Transforms/buildPushingToViewsChain.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -710,6 +711,7 @@ IProcessor::Status FinalizingViewsTransform::prepare() if (!output.canPush()) return Status::PortFull; + bool materialized_views_ignore_errors = views_data->context->getSettingsRef().materialized_views_ignore_errors; size_t num_finished = 0; size_t pos = 0; for (auto & input : inputs) @@ -735,7 +737,7 @@ IProcessor::Status FinalizingViewsTransform::prepare() else statuses[i].exception = data.exception; - if (i == 0 && statuses[0].is_first) + if (i == 0 && statuses[0].is_first && !materialized_views_ignore_errors) { output.pushData(std::move(data)); return Status::PortFull; @@ -752,7 +754,7 @@ IProcessor::Status FinalizingViewsTransform::prepare() if (!statuses.empty()) return Status::Ready; - if (any_exception) + if (any_exception && !materialized_views_ignore_errors) output.pushException(any_exception); output.finish(); @@ -782,6 +784,8 @@ static std::exception_ptr addStorageToException(std::exception_ptr ptr, const St void FinalizingViewsTransform::work() { + bool materialized_views_ignore_errors = views_data->context->getSettingsRef().materialized_views_ignore_errors; + size_t i = 0; for (auto & view : views_data->views) { @@ -794,6 +798,10 @@ void FinalizingViewsTransform::work() any_exception = status.exception; view.setException(addStorageToException(status.exception, view.table_id)); + + /// Exception will be ignored, it is saved here for the system.query_views_log + if (materialized_views_ignore_errors) + tryLogException(view.exception, &Poco::Logger::get("PushingToViews"), "Cannot push to the storage, ignoring the error"); } else { diff --git a/src/Server/KeeperTCPHandler.cpp b/src/Server/KeeperTCPHandler.cpp index 0853c6ee62b..f9e11062906 100644 --- a/src/Server/KeeperTCPHandler.cpp +++ b/src/Server/KeeperTCPHandler.cpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include #ifdef POCO_HAVE_FD_EPOLL diff --git a/src/Storages/Cache/ExternalDataSourceCache.cpp b/src/Storages/Cache/ExternalDataSourceCache.cpp index 56b2e661836..1fc68a2d774 100644 --- a/src/Storages/Cache/ExternalDataSourceCache.cpp +++ b/src/Storages/Cache/ExternalDataSourceCache.cpp @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Storages/ColumnsDescription.cpp b/src/Storages/ColumnsDescription.cpp index d401840eec7..fa39e304925 100644 --- a/src/Storages/ColumnsDescription.cpp +++ b/src/Storages/ColumnsDescription.cpp @@ -383,6 +383,15 @@ NamesAndTypesList ColumnsDescription::getEphemeral() const return ret; } +NamesAndTypesList ColumnsDescription::getWithDefaultExpression() const +{ + NamesAndTypesList ret; + for (const auto & col : columns) + if (col.default_desc.expression) + ret.emplace_back(col.name, col.type); + return ret; +} + NamesAndTypesList ColumnsDescription::getAll() const { NamesAndTypesList ret; diff --git a/src/Storages/ColumnsDescription.h b/src/Storages/ColumnsDescription.h index 4f874f4b850..5551fdea2e3 100644 --- a/src/Storages/ColumnsDescription.h +++ b/src/Storages/ColumnsDescription.h @@ -132,6 +132,7 @@ public: NamesAndTypesList getInsertable() const; /// ordinary + ephemeral NamesAndTypesList getAliases() const; NamesAndTypesList getEphemeral() const; + NamesAndTypesList getWithDefaultExpression() const; // columns with default expression, for example set by `CREATE TABLE` statement NamesAndTypesList getAllPhysical() const; /// ordinary + materialized. NamesAndTypesList getAll() const; /// ordinary + materialized + aliases + ephemeral /// Returns .size0/.null/... diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp index c6f675533c6..b47028b883a 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Storages/IndicesDescription.cpp b/src/Storages/IndicesDescription.cpp index 2e07aceeaa9..a93ac248c98 100644 --- a/src/Storages/IndicesDescription.cpp +++ b/src/Storages/IndicesDescription.cpp @@ -94,14 +94,15 @@ IndexDescription IndexDescription::getIndexFromAST(const ASTPtr & definition_ast auto syntax = TreeRewriter(context).analyze(expr_list, columns.getAllPhysical()); result.expression = ExpressionAnalyzer(expr_list, syntax, context).getActions(true); - Block block_without_columns = result.expression->getSampleBlock(); + result.sample_block = result.expression->getSampleBlock(); - for (size_t i = 0; i < block_without_columns.columns(); ++i) + for (auto & elem : result.sample_block) { - const auto & column = block_without_columns.getByPosition(i); - result.column_names.emplace_back(column.name); - result.data_types.emplace_back(column.type); - result.sample_block.insert(ColumnWithTypeAndName(column.type->createColumn(), column.type, column.name)); + if (!elem.column) + elem.column = elem.type->createColumn(); + + result.column_names.push_back(elem.name); + result.data_types.push_back(elem.type); } const auto & definition_arguments = index_definition->type->arguments; diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 7b97273d8af..2afdc0dda8a 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -959,6 +959,11 @@ void registerStorageKafka(StorageFactory & factory) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "kafka_poll_max_batch_size can not be lower than 1"); } + if (args.columns.getOrdinary() != args.columns.getAll() || !args.columns.getWithDefaultExpression().empty()) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "KafkaEngine doesn't support DEFAULT/MATERIALIZED/EPHEMERAL/ALIAS expressions for columns. " + "See https://clickhouse.com/docs/en/engines/table-engines/integrations/kafka/#configuration"); + } return std::make_shared(args.table_id, args.getContext(), args.columns, std::move(kafka_settings), collection_name); }; diff --git a/src/Storages/LiveView/LiveViewSink.h b/src/Storages/LiveView/LiveViewSink.h index 1d90e35618f..e163400f2af 100644 --- a/src/Storages/LiveView/LiveViewSink.h +++ b/src/Storages/LiveView/LiveViewSink.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include namespace DB diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp index 547becf3837..2c3e452de92 100644 --- a/src/Storages/LiveView/StorageLiveView.cpp +++ b/src/Storages/LiveView/StorageLiveView.cpp @@ -27,7 +27,7 @@ limitations under the License. */ #include #include #include -#include +#include #include #include @@ -241,7 +241,18 @@ StorageLiveView::StorageLiveView( blocks_metadata_ptr = std::make_shared(); active_ptr = std::make_shared(true); - periodic_refresh_task = getContext()->getSchedulePool().createTask("LiveViewPeriodicRefreshTask", [this]{ periodicRefreshTaskFunc(); }); + periodic_refresh_task = getContext()->getSchedulePool().createTask("LiveViewPeriodicRefreshTask", + [this] + { + try + { + periodicRefreshTaskFunc(); + } + catch (...) + { + tryLogCurrentException(log, "Exception in LiveView periodic refresh task in BackgroundSchedulePool"); + } + }); periodic_refresh_task->deactivate(); } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 1da99cb4117..73007e3f178 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -525,7 +525,6 @@ void MergeTreeData::checkProperties( for (const auto & index : new_metadata.secondary_indices) { - MergeTreeIndexFactory::instance().validate(index, attach); if (indices_names.find(index.name) != indices_names.end()) diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 0d123623f05..cf009a10c27 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -31,7 +31,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp index 8f4d066baa3..7a0b1d03e79 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp @@ -1,6 +1,6 @@ #include "MergeTreeDataPartChecksum.h" #include -#include +#include #include #include #include diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index db358e1b6a9..07da66e4378 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -588,18 +588,24 @@ MergeTreeDataSelectSamplingData MergeTreeDataSelectExecutor::getSampling( * It is also important that the entire universe can be covered using SAMPLE 0.1 OFFSET 0, ... OFFSET 0.9 and similar decimals. */ + auto parallel_replicas_mode = context->getParallelReplicasMode(); /// Parallel replicas has been requested but there is no way to sample data. /// Select all data from first replica and no data from other replicas. - if (settings.parallel_replicas_count > 1 && !data.supportsSampling() && settings.parallel_replica_offset > 0) + if (settings.parallel_replicas_count > 1 && parallel_replicas_mode == Context::ParallelReplicasMode::SAMPLE_KEY + && !data.supportsSampling() && settings.parallel_replica_offset > 0) { - LOG_DEBUG(log, "Will use no data on this replica because parallel replicas processing has been requested" + LOG_DEBUG( + log, + "Will use no data on this replica because parallel replicas processing has been requested" " (the setting 'max_parallel_replicas') but the table does not support sampling and this replica is not the first."); sampling.read_nothing = true; return sampling; } - sampling.use_sampling = relative_sample_size > 0 || (settings.parallel_replicas_count > 1 && data.supportsSampling()); - bool no_data = false; /// There is nothing left after sampling. + sampling.use_sampling = relative_sample_size > 0 + || (settings.parallel_replicas_count > 1 && parallel_replicas_mode == Context::ParallelReplicasMode::SAMPLE_KEY + && data.supportsSampling()); + bool no_data = false; /// There is nothing left after sampling. if (sampling.use_sampling) { diff --git a/src/Storages/MergeTree/MergeTreeIndices.cpp b/src/Storages/MergeTree/MergeTreeIndices.cpp index 2be9ecd8de3..6ae96d00171 100644 --- a/src/Storages/MergeTree/MergeTreeIndices.cpp +++ b/src/Storages/MergeTree/MergeTreeIndices.cpp @@ -35,6 +35,7 @@ MergeTreeIndexPtr MergeTreeIndexFactory::get( { auto it = creators.find(index.type); if (it == creators.end()) + { throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown Index type '{}'. Available index types: {}", index.type, std::accumulate(creators.cbegin(), creators.cend(), std::string{}, @@ -46,6 +47,7 @@ MergeTreeIndexPtr MergeTreeIndexFactory::get( return left + ", " + right.first; }) ); + } return it->second(index); } @@ -61,8 +63,31 @@ MergeTreeIndices MergeTreeIndexFactory::getMany(const std::vectorhasArrayJoin()) + throw Exception(ErrorCodes::INCORRECT_QUERY, "Secondary index '{}' cannot contain array joins", index.name); + + try + { + index.expression->assertDeterministic(); + } + catch (Exception & e) + { + e.addMessage(fmt::format("for secondary index '{}'", index.name)); + throw; + } + + for (const auto & elem : index.sample_block) + if (elem.column && (isColumnConst(*elem.column) || elem.column->isDummy())) + throw Exception(ErrorCodes::INCORRECT_QUERY, "Secondary index '{}' cannot contain constants", index.name); + } + auto it = validators.find(index.type); if (it == validators.end()) + { throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown Index type '{}'. Available index types: {}", index.type, std::accumulate( @@ -77,6 +102,7 @@ void MergeTreeIndexFactory::validate(const IndexDescription & index, bool attach return left + ", " + right.first; }) ); + } it->second(index, attach); } diff --git a/src/Storages/MergeTree/MergeTreePartition.cpp b/src/Storages/MergeTree/MergeTreePartition.cpp index 3a6908ef32d..3b28012e7d6 100644 --- a/src/Storages/MergeTree/MergeTreePartition.cpp +++ b/src/Storages/MergeTree/MergeTreePartition.cpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index e951b8f54cf..479e50fdebb 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -64,8 +64,7 @@ void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def, ContextPtr conte auto ast = dynamic_cast(custom.getImpl()).ast; if (ast && isDiskFunction(ast)) { - const auto & ast_function = assert_cast(*ast); - auto disk_name = getOrCreateDiskFromDiskAST(ast_function, context); + auto disk_name = getOrCreateDiskFromDiskAST(ast, context); LOG_TRACE(&Poco::Logger::get("MergeTreeSettings"), "Created custom disk {}", disk_name); value = disk_name; } diff --git a/src/Storages/MergeTree/PartMetadataManagerWithCache.cpp b/src/Storages/MergeTree/PartMetadataManagerWithCache.cpp index b6260d5edb6..3a53cf25745 100644 --- a/src/Storages/MergeTree/PartMetadataManagerWithCache.cpp +++ b/src/Storages/MergeTree/PartMetadataManagerWithCache.cpp @@ -1,7 +1,7 @@ #include "PartMetadataManagerWithCache.h" #if USE_ROCKSDB -#include +#include #include #include #include diff --git a/src/Storages/MergeTree/SimpleMergeSelector.cpp b/src/Storages/MergeTree/SimpleMergeSelector.cpp index 15291622a2a..af3373fd175 100644 --- a/src/Storages/MergeTree/SimpleMergeSelector.cpp +++ b/src/Storages/MergeTree/SimpleMergeSelector.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include @@ -28,7 +28,7 @@ struct Estimator { double difference = std::abs(log2(static_cast(sum_size) / size_prev_at_left)); if (difference < settings.heuristic_to_align_parts_max_absolute_difference_in_powers_of_two) - current_score *= interpolateLinear(settings.heuristic_to_align_parts_max_score_adjustment, 1, + current_score *= std::lerp(settings.heuristic_to_align_parts_max_score_adjustment, 1, difference / settings.heuristic_to_align_parts_max_absolute_difference_in_powers_of_two); } @@ -115,8 +115,8 @@ bool allow( // std::cerr << "size_normalized: " << size_normalized << "\n"; /// Calculate boundaries for age - double min_age_to_lower_base = interpolateLinear(settings.min_age_to_lower_base_at_min_size, settings.min_age_to_lower_base_at_max_size, size_normalized); - double max_age_to_lower_base = interpolateLinear(settings.max_age_to_lower_base_at_min_size, settings.max_age_to_lower_base_at_max_size, size_normalized); + double min_age_to_lower_base = std::lerp(settings.min_age_to_lower_base_at_min_size, settings.min_age_to_lower_base_at_max_size, size_normalized); + double max_age_to_lower_base = std::lerp(settings.max_age_to_lower_base_at_min_size, settings.max_age_to_lower_base_at_max_size, size_normalized); // std::cerr << "min_age_to_lower_base: " << min_age_to_lower_base << "\n"; // std::cerr << "max_age_to_lower_base: " << max_age_to_lower_base << "\n"; @@ -137,7 +137,7 @@ bool allow( // std::cerr << "combined_ratio: " << combined_ratio << "\n"; - double lowered_base = interpolateLinear(settings.base, 2.0, combined_ratio); + double lowered_base = std::lerp(settings.base, 2.0, combined_ratio); // std::cerr << "------- lowered_base: " << lowered_base << "\n"; diff --git a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp index 9c6eeceb605..d048c94ac75 100644 --- a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp +++ b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp @@ -2,7 +2,7 @@ #include "StorageMaterializedPostgreSQL.h" #include -#include +#include #include #include #include diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index 40ea84ec68b..e3996950e79 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -207,6 +207,8 @@ struct SelectQueryInfo /// /// Configured in StorageDistributed::getQueryProcessingStage() ClusterPtr optimized_cluster; + /// should we use custom key with the cluster + bool use_custom_key = false; mutable ParallelReplicasReadingCoordinatorPtr coordinator; @@ -218,6 +220,8 @@ struct SelectQueryInfo /// It is needed for PK analysis based on row_level_policy and additional_filters. ASTs filter_asts; + ASTPtr parallel_replica_custom_key_ast; + /// Filter actions dag for current storage ActionsDAGPtr filter_actions_dag; diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 0bef3784adb..4eb454e5156 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -51,6 +51,7 @@ #include #include #include +#include #include #include #include @@ -65,6 +66,8 @@ #include #include #include +#include + #include #include #include @@ -82,6 +85,7 @@ #include #include +#include #include #include @@ -262,7 +266,6 @@ size_t getClusterQueriedNodes(const Settings & settings, const ClusterPtr & clus } - /// For destruction of std::unique_ptr of type that is incomplete in class definition. StorageDistributed::~StorageDistributed() = default; @@ -400,29 +403,38 @@ QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage( const auto & settings = local_context->getSettingsRef(); ClusterPtr cluster = getCluster(); - query_info.cluster = cluster; size_t nodes = getClusterQueriedNodes(settings, cluster); - /// Always calculate optimized cluster here, to avoid conditions during read() - /// (Anyway it will be calculated in the read()) - if (nodes > 1 && settings.optimize_skip_unused_shards) + if (query_info.use_custom_key) { - ClusterPtr optimized_cluster = getOptimizedCluster(local_context, storage_snapshot, query_info.query); - if (optimized_cluster) - { - LOG_DEBUG(log, "Skipping irrelevant shards - the query will be sent to the following shards of the cluster (shard numbers): {}", - makeFormattedListOfShards(optimized_cluster)); + LOG_INFO(log, "Single shard cluster used with custom_key, transforming replicas into virtual shards"); + query_info.cluster = cluster->getClusterWithReplicasAsShards(settings, settings.max_parallel_replicas); + } + else + { + query_info.cluster = cluster; - cluster = optimized_cluster; - query_info.optimized_cluster = cluster; - - nodes = getClusterQueriedNodes(settings, cluster); - } - else + if (nodes > 1 && settings.optimize_skip_unused_shards) { - LOG_DEBUG(log, "Unable to figure out irrelevant shards from WHERE/PREWHERE clauses - the query will be sent to all shards of the cluster{}", - has_sharding_key ? "" : " (no sharding key)"); + /// Always calculate optimized cluster here, to avoid conditions during read() + /// (Anyway it will be calculated in the read()) + ClusterPtr optimized_cluster = getOptimizedCluster(local_context, storage_snapshot, query_info.query); + if (optimized_cluster) + { + LOG_DEBUG(log, "Skipping irrelevant shards - the query will be sent to the following shards of the cluster (shard numbers): {}", + makeFormattedListOfShards(optimized_cluster)); + + cluster = optimized_cluster; + query_info.optimized_cluster = cluster; + + nodes = getClusterQueriedNodes(settings, cluster); + } + else + { + LOG_DEBUG(log, "Unable to figure out irrelevant shards from WHERE/PREWHERE clauses - the query will be sent to all shards of the cluster{}", + has_sharding_key ? "" : " (no sharding key)"); + } } } @@ -728,13 +740,36 @@ void StorageDistributed::read( storage_snapshot, processed_stage); + auto settings = local_context->getSettingsRef(); + + ClusterProxy::AdditionalShardFilterGenerator additional_shard_filter_generator; + if (query_info.use_custom_key) + { + if (auto custom_key_ast = parseCustomKeyForTable(settings.parallel_replicas_custom_key, *local_context)) + { + if (query_info.getCluster()->getShardCount() == 1) + { + // we are reading from single shard with multiple replicas but didn't transform replicas + // into virtual shards with custom_key set + throw Exception(ErrorCodes::LOGICAL_ERROR, "Replicas weren't transformed into virtual shards"); + } + + additional_shard_filter_generator = + [&, custom_key_ast = std::move(custom_key_ast), shard_count = query_info.cluster->getShardCount()](uint64_t shard_num) -> ASTPtr + { + return getCustomKeyFilterForParallelReplica( + shard_count, shard_num - 1, custom_key_ast, settings.parallel_replicas_custom_key_filter_type, *this, local_context); + }; + } + } + ClusterProxy::executeQuery( query_plan, header, processed_stage, main_table, remote_table_function_ptr, select_stream_factory, log, modified_query_ast, local_context, query_info, sharding_key_expr, sharding_key_column_name, - query_info.cluster); + query_info.cluster, additional_shard_filter_generator); /// This is a bug, it is possible only when there is no shards to query, and this is handled earlier. if (!query_plan.isInitialized()) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 6c6ff30fd04..5065add20e9 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/Storages/System/StorageSystemParts.cpp b/src/Storages/System/StorageSystemParts.cpp index f6854e7d5d0..86ecb336b51 100644 --- a/src/Storages/System/StorageSystemParts.cpp +++ b/src/Storages/System/StorageSystemParts.cpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/Storages/System/StorageSystemProjectionParts.cpp b/src/Storages/System/StorageSystemProjectionParts.cpp index 37c62ba5eb0..d2c6c3ef287 100644 --- a/src/Storages/System/StorageSystemProjectionParts.cpp +++ b/src/Storages/System/StorageSystemProjectionParts.cpp @@ -11,7 +11,7 @@ #include #include #include -#include +#include namespace DB { diff --git a/tests/ci/autoscale_runners_lambda/app.py b/tests/ci/autoscale_runners_lambda/app.py new file mode 100644 index 00000000000..7e3af3f6779 --- /dev/null +++ b/tests/ci/autoscale_runners_lambda/app.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 + +"""The lambda to decrease/increase ASG desired capacity based on current queue""" + +import json +import logging +import time +from dataclasses import dataclass +from pprint import pformat +from typing import Any, List, Literal, Optional, Tuple + +import boto3 # type: ignore +import requests # type: ignore + +RUNNER_TYPE_LABELS = [ + "builder", + "func-tester", + "func-tester-aarch64", + "fuzzer-unit-tester", + "stress-tester", + "style-checker", + "style-checker-aarch64", +] + +# 4 HOUR - is a balance to get the most precise values +# - Our longest possible running check is around 5h on the worst scenario +# - The long queue won't be wiped out and replaced, so the measurmenet is fine +# - If the data is spoiled by something, we are from the bills perspective +QUEUE_QUERY = f"""SELECT + last_status AS status, + toUInt32(count()) AS length, + labels +FROM +( + SELECT + arraySort(groupArray(status))[-1] AS last_status, + labels, + id, + html_url + FROM default.workflow_jobs + WHERE has(labels, 'self-hosted') + AND hasAny({RUNNER_TYPE_LABELS}, labels) + AND started_at > now() - INTERVAL 4 HOUR + GROUP BY ALL + HAVING last_status IN ('in_progress', 'queued') +) +GROUP BY ALL +ORDER BY labels, last_status""" + + +@dataclass +class Queue: + status: Literal["in_progress", "queued"] + lentgh: int + label: str + + +def get_scales(runner_type: str) -> Tuple[int, int]: + "returns the multipliers for scaling down and up ASG by types" + # Scaling down is quicker on the lack of running jobs than scaling up on + # queue + scale_down = 3 + scale_up = 5 + if runner_type == "style-checker": + # the style checkers have so many noise, so it scales up too quickly + scale_down = 2 + scale_up = 10 + return scale_down, scale_up + + +### VENDORING +def get_parameter_from_ssm(name, decrypt=True, client=None): + if not client: + client = boto3.client("ssm", region_name="us-east-1") + return client.get_parameter(Name=name, WithDecryption=decrypt)["Parameter"]["Value"] + + +class CHException(Exception): + pass + + +class ClickHouseHelper: + def __init__( + self, + url: Optional[str] = None, + user: Optional[str] = None, + password: Optional[str] = None, + ): + self.url = url + self.auth = {} + if user: + self.auth["X-ClickHouse-User"] = user + if password: + self.auth["X-ClickHouse-Key"] = password + + def _select_and_get_json_each_row(self, db, query): + params = { + "database": db, + "query": query, + "default_format": "JSONEachRow", + } + for i in range(5): + response = None + try: + response = requests.get(self.url, params=params, headers=self.auth) + response.raise_for_status() + return response.text + except Exception as ex: + logging.warning("Cannot fetch data with exception %s", str(ex)) + if response: + logging.warning("Reponse text %s", response.text) + time.sleep(0.1 * i) + + raise CHException("Cannot fetch data from clickhouse") + + def select_json_each_row(self, db, query): + text = self._select_and_get_json_each_row(db, query) + result = [] + for line in text.split("\n"): + if line: + result.append(json.loads(line)) + return result + + +CH_CLIENT = ClickHouseHelper(get_parameter_from_ssm("clickhouse-test-stat-url"), "play") + + +def set_capacity( + runner_type: str, queues: List[Queue], client: Any, dry_run: bool = True +) -> None: + assert len(queues) in (1, 2) + assert all(q.label == runner_type for q in queues) + as_groups = client.describe_auto_scaling_groups( + Filters=[ + {"Name": "tag-key", "Values": ["github:runner-type"]}, + {"Name": "tag-value", "Values": [runner_type]}, + ] + )["AutoScalingGroups"] + assert len(as_groups) == 1 + asg = as_groups[0] + running = 0 + queued = 0 + for q in queues: + if q.status == "in_progress": + running = q.lentgh + continue + if q.status == "queued": + queued = q.lentgh + continue + raise ValueError("Queue status is not in ['in_progress', 'queued']") + + scale_down, scale_up = get_scales(runner_type) + # How much nodes are free (positive) or need to be added (negative) + capacity_reserve = asg["DesiredCapacity"] - running - queued + stop = False + if capacity_reserve < 0: + # This part is about scaling up + capacity_deficit = -capacity_reserve + # It looks that we are still OK, since no queued jobs exist + stop = stop or queued == 0 + # Are we already at the capacity limits + stop = stop or asg["MaxSize"] <= asg["DesiredCapacity"] + # Let's calculate a new desired capacity + desired_capacity = asg["DesiredCapacity"] + (capacity_deficit // scale_up) + desired_capacity = max(desired_capacity, asg["MinSize"]) + desired_capacity = min(desired_capacity, asg["MaxSize"]) + # Finally, should the capacity be even changed + stop = stop or asg["DesiredCapacity"] == desired_capacity + if stop: + return + logging.info( + "The ASG %s capacity will be increased to %s, current capacity=%s, " + "maximum capacity=%s, running jobs=%s, queue size=%s", + asg["AutoScalingGroupName"], + desired_capacity, + asg["DesiredCapacity"], + asg["MaxSize"], + running, + queued, + ) + if not dry_run: + client.set_desired_capacity( + AutoScalingGroupName=asg["AutoScalingGroupName"], + DesiredCapacity=desired_capacity, + ) + return + + # Now we will calculate if we need to scale down + stop = stop or asg["DesiredCapacity"] == asg["MinSize"] + desired_capacity = asg["DesiredCapacity"] - (capacity_reserve // scale_down) + desired_capacity = max(desired_capacity, asg["MinSize"]) + desired_capacity = min(desired_capacity, asg["MaxSize"]) + stop = stop or asg["DesiredCapacity"] == desired_capacity + if stop: + return + + logging.info( + "The ASG %s capacity will be decreased to %s, current capacity=%s, " + "minimum capacity=%s, running jobs=%s, queue size=%s", + asg["AutoScalingGroupName"], + desired_capacity, + asg["DesiredCapacity"], + asg["MinSize"], + running, + queued, + ) + if not dry_run: + client.set_desired_capacity( + AutoScalingGroupName=asg["AutoScalingGroupName"], + DesiredCapacity=desired_capacity, + ) + + +def main(dry_run: bool = True) -> None: + logging.getLogger().setLevel(logging.INFO) + asg_client = boto3.client("autoscaling") + try: + global CH_CLIENT + queues = CH_CLIENT.select_json_each_row("default", QUEUE_QUERY) + except CHException as ex: + logging.exception( + "Got an exception on insert, tryuing to update the client " + "credentials and repeat", + exc_info=ex, + ) + CH_CLIENT = ClickHouseHelper( + get_parameter_from_ssm("clickhouse-test-stat-url"), "play" + ) + queues = CH_CLIENT.select_json_each_row("default", QUEUE_QUERY) + + logging.info("Received queue data:\n%s", pformat(queues, width=120)) + for runner_type in RUNNER_TYPE_LABELS: + runner_queues = [ + Queue(queue["status"], queue["length"], runner_type) + for queue in queues + if runner_type in queue["labels"] + ] + runner_queues = runner_queues or [Queue("in_progress", 0, runner_type)] + set_capacity(runner_type, runner_queues, asg_client, dry_run) + + +def handler(event: dict, context: Any) -> None: + _ = event + _ = context + return main(False) diff --git a/tests/ci/autoscale_runners_lambda/build_and_deploy_archive.sh b/tests/ci/autoscale_runners_lambda/build_and_deploy_archive.sh new file mode 120000 index 00000000000..96ba3fa024e --- /dev/null +++ b/tests/ci/autoscale_runners_lambda/build_and_deploy_archive.sh @@ -0,0 +1 @@ +../team_keys_lambda/build_and_deploy_archive.sh \ No newline at end of file diff --git a/tests/ci/autoscale_runners_lambda/requirements.txt b/tests/ci/autoscale_runners_lambda/requirements.txt new file mode 100644 index 00000000000..f2293605cf1 --- /dev/null +++ b/tests/ci/autoscale_runners_lambda/requirements.txt @@ -0,0 +1 @@ +requests diff --git a/tests/ci/autoscale_runners_lambda_test.py b/tests/ci/autoscale_runners_lambda_test.py new file mode 100644 index 00000000000..7efa0004745 --- /dev/null +++ b/tests/ci/autoscale_runners_lambda_test.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python + +import unittest +from dataclasses import dataclass +from typing import Any, List + +from autoscale_runners_lambda.app import set_capacity, Queue + + +@dataclass +class TestCase: + name: str + min_size: int + desired_capacity: int + max_size: int + queues: List[Queue] + expected_capacity: int + + +class TestSetCapacity(unittest.TestCase): + class FakeClient: + def __init__(self): + self._expected_data = {} # type: dict + self._expected_capacity = -1 + + @property + def expected_data(self) -> dict: + """a one-time property""" + data, self._expected_data = self._expected_data, {} + return data + + @expected_data.setter + def expected_data(self, value: dict) -> None: + self._expected_data = value + + @property + def expected_capacity(self) -> int: + """one-time property""" + capacity, self._expected_capacity = self._expected_capacity, -1 + return capacity + + def describe_auto_scaling_groups(self, **kwargs: Any) -> dict: + _ = kwargs + return self.expected_data + + def set_desired_capacity(self, **kwargs: Any) -> None: + self._expected_capacity = kwargs["DesiredCapacity"] + + def data_helper( + self, name: str, min_size: int, desired_capacity: int, max_size: int + ) -> None: + self.expected_data = { + "AutoScalingGroups": [ + { + "AutoScalingGroupName": name, + "DesiredCapacity": desired_capacity, + "MinSize": min_size, + "MaxSize": max_size, + } + ] + } + + def setUp(self): + self.client = self.FakeClient() + + def test_normal_cases(self): + test_cases = ( + # Do not change capacity + TestCase("noqueue", 1, 13, 20, [Queue("in_progress", 155, "noqueue")], -1), + TestCase("w/reserve", 1, 13, 20, [Queue("queued", 17, "w/reserve")], -1), + # Increase capacity + TestCase("increase", 1, 13, 20, [Queue("queued", 23, "increase")], 15), + TestCase("increase", 1, 13, 20, [Queue("queued", 18, "increase")], 14), + TestCase("increase", 1, 13, 20, [Queue("queued", 183, "increase")], 20), + TestCase( + "increase-w/o reserve", + 1, + 13, + 20, + [ + Queue("in_progress", 11, "increase-w/o reserve"), + Queue("queued", 12, "increase-w/o reserve"), + ], + 15, + ), + TestCase("lower-min", 10, 5, 20, [Queue("queued", 5, "lower-min")], 10), + # Decrease capacity + TestCase("w/reserve", 1, 13, 20, [Queue("queued", 5, "w/reserve")], 11), + TestCase("w/reserve", 1, 23, 20, [Queue("queued", 17, "w/reserve")], 20), + TestCase("decrease", 1, 13, 20, [Queue("in_progress", 3, "decrease")], 10), + TestCase("decrease", 1, 13, 20, [Queue("in_progress", 5, "decrease")], 11), + ) + for t in test_cases: + self.client.data_helper(t.name, t.min_size, t.desired_capacity, t.max_size) + set_capacity(t.name, t.queues, self.client, False) + self.assertEqual(t.expected_capacity, self.client.expected_capacity, t.name) + + def test_exceptions(self): + test_cases = ( + ( + TestCase( + "different names", + 1, + 1, + 1, + [Queue("queued", 5, "another name")], + -1, + ), + AssertionError, + ), + (TestCase("wrong queue len", 1, 1, 1, [], -1), AssertionError), + ( + TestCase( + "wrong queue", 1, 1, 1, [Queue("wrong", 1, "wrong queue")], -1 # type: ignore + ), + ValueError, + ), + ) + for t, error in test_cases: + with self.assertRaises(error): + self.client.data_helper( + t.name, t.min_size, t.desired_capacity, t.max_size + ) + set_capacity(t.name, t.queues, self.client, False) + + with self.assertRaises(AssertionError): + self.client.expected_data = {"AutoScalingGroups": [1, 2]} + set_capacity( + "wrong number of ASGs", + [Queue("queued", 1, "wrong number of ASGs")], + self.client, + ) diff --git a/tests/ci/workflow_jobs_lambda/app.py b/tests/ci/workflow_jobs_lambda/app.py index f6589576806..9436e01ad53 100644 --- a/tests/ci/workflow_jobs_lambda/app.py +++ b/tests/ci/workflow_jobs_lambda/app.py @@ -213,7 +213,7 @@ def send_event_workflow_job(workflow_job: WorkflowJob) -> None: # `head_sha` String, # `url` String, # `html_url` String, - # `status` Enum8('queued' = 1, 'in_progress' = 2, 'completed' = 3, 'waiting' = 4), + # `status` Enum8('waiting' = 1, 'queued' = 2, 'in_progress' = 3, 'completed' = 4), # `conclusion` LowCardinality(String), # `started_at` DateTime, # `completed_at` DateTime, diff --git a/tests/clickhouse-test b/tests/clickhouse-test index e30e709f363..c1103da1552 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -118,6 +118,7 @@ def clickhouse_execute_http( "http_connection_timeout": timeout, "http_receive_timeout": timeout, "http_send_timeout": timeout, + "output_format_parallel_formatting": 0, } if settings is not None: params.update(settings) diff --git a/tests/integration/test_disk_configuration/test.py b/tests/integration/test_disk_configuration/test.py index 34f8bea219f..6ebe994dc68 100644 --- a/tests/integration/test_disk_configuration/test.py +++ b/tests/integration/test_disk_configuration/test.py @@ -294,6 +294,65 @@ def test_merge_tree_custom_disk_setting(start_cluster): ).strip() ) + node1.query(f"DROP TABLE {TABLE_NAME} SYNC") + node1.query(f"DROP TABLE {TABLE_NAME}_2 SYNC") + node1.query(f"DROP TABLE {TABLE_NAME}_3 SYNC") + node1.query(f"DROP TABLE {TABLE_NAME}_4 SYNC") + node2.query(f"DROP TABLE {TABLE_NAME}_4 SYNC") + + +def test_merge_tree_nested_custom_disk_setting(start_cluster): + node = cluster.instances["node1"] + + minio = cluster.minio_client + for obj in list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)): + minio.remove_object(cluster.minio_bucket, obj.object_name) + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + == 0 + ) + + node.query( + f""" + DROP TABLE IF EXISTS {TABLE_NAME} SYNC; + CREATE TABLE {TABLE_NAME} (a Int32) + ENGINE = MergeTree() order by tuple() + SETTINGS disk = disk( + type=cache, + max_size='1Gi', + path='/var/lib/clickhouse/custom_disk_cache/', + disk=disk( + type=s3, + endpoint='http://minio1:9001/root/data/', + access_key_id='minio', + secret_access_key='minio123')); + """ + ) + + node.query(f"INSERT INTO {TABLE_NAME} SELECT number FROM numbers(100)") + node.query("SYSTEM DROP FILESYSTEM CACHE") + + # Check cache is filled + assert 0 == int(node.query("SELECT count() FROM system.filesystem_cache")) + assert 100 == int(node.query(f"SELECT count() FROM {TABLE_NAME}")) + node.query(f"SELECT * FROM {TABLE_NAME}") + assert 0 < int(node.query("SELECT count() FROM system.filesystem_cache")) + + # Check s3 is filled + assert ( + len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) > 0 + ) + + node.restart_clickhouse() + + assert 100 == int(node.query(f"SELECT count() FROM {TABLE_NAME}")) + + expected = """ + SETTINGS disk = disk(type = cache, max_size = \\'[HIDDEN]\\', path = \\'[HIDDEN]\\', disk = disk(type = s3, endpoint = \\'[HIDDEN]\\' + """ + assert expected.strip() in node.query(f"SHOW CREATE TABLE {TABLE_NAME}").strip() + node.query(f"DROP TABLE {TABLE_NAME} SYNC") + def test_merge_tree_setting_override(start_cluster): node = cluster.instances["node3"] @@ -367,3 +426,4 @@ def test_merge_tree_setting_override(start_cluster): assert ( len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) > 0 ) + node.query(f"DROP TABLE {TABLE_NAME} SYNC") diff --git a/tests/integration/test_filesystem_layout/test.py b/tests/integration/test_filesystem_layout/test.py index 898bbc40eb9..2be478f95d0 100644 --- a/tests/integration/test_filesystem_layout/test.py +++ b/tests/integration/test_filesystem_layout/test.py @@ -44,8 +44,6 @@ def test_file_path_escaping(started_cluster): ] ) - -def test_file_path_escaping_atomic_db(started_cluster): node.query("CREATE DATABASE IF NOT EXISTS `test 2` ENGINE = Atomic") node.query( """ diff --git a/tests/integration/test_grpc_protocol/test.py b/tests/integration/test_grpc_protocol/test.py index a1bc0d42a46..137d585f7d1 100644 --- a/tests/integration/test_grpc_protocol/test.py +++ b/tests/integration/test_grpc_protocol/test.py @@ -594,8 +594,6 @@ def test_cancel_while_processing_input(): stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) result = stub.ExecuteQueryWithStreamInput(send_query_info()) assert result.cancelled == True - assert result.progress.written_rows == 6 - assert query("SELECT a FROM t ORDER BY a") == "1\n2\n3\n4\n5\n6\n" def test_cancel_while_generating_output(): diff --git a/tests/integration/test_log_levels_update/test.py b/tests/integration/test_log_levels_update/test.py index 176733cd7cb..4b83b6431fc 100644 --- a/tests/integration/test_log_levels_update/test.py +++ b/tests/integration/test_log_levels_update/test.py @@ -10,7 +10,7 @@ node = cluster.add_instance( config = """ - information + debug /var/log/clickhouse-server/clickhouse-server.log """ @@ -63,4 +63,4 @@ def test_log_levels_update(start_cluster): log = get_log(node) assert len(log) > 0 - assert not re.search("(|)", log) + assert not re.search("", log) diff --git a/tests/integration/test_parallel_replicas_custom_key/__init__.py b/tests/integration/test_parallel_replicas_custom_key/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_parallel_replicas_custom_key/configs/remote_servers.xml b/tests/integration/test_parallel_replicas_custom_key/configs/remote_servers.xml new file mode 100644 index 00000000000..308db461498 --- /dev/null +++ b/tests/integration/test_parallel_replicas_custom_key/configs/remote_servers.xml @@ -0,0 +1,50 @@ + + + + + false + + n1 + 9000 + + + n2 + 9000 + + + + false + + n3 + 9000 + + + n4 + 9000 + + + + + + false + + n1 + 9000 + + + n2 + 9000 + + + n3 + 9000 + + + n4 + 9000 + + + + + + diff --git a/tests/integration/test_parallel_replicas_custom_key/test.py b/tests/integration/test_parallel_replicas_custom_key/test.py new file mode 100644 index 00000000000..baac2661506 --- /dev/null +++ b/tests/integration/test_parallel_replicas_custom_key/test.py @@ -0,0 +1,94 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +nodes = [ + cluster.add_instance( + f"n{i}", main_configs=["configs/remote_servers.xml"], with_zookeeper=True + ) + for i in range(1, 5) +] + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def create_tables(cluster): + n1 = nodes[0] + n1.query("DROP TABLE IF EXISTS dist_table") + n1.query(f"DROP TABLE IF EXISTS test_table ON CLUSTER {cluster}") + + n1.query( + f"CREATE TABLE test_table ON CLUSTER {cluster} (key Int32, value String) Engine=MergeTree ORDER BY (key, sipHash64(value))" + ) + n1.query( + f""" + CREATE TABLE dist_table AS test_table + Engine=Distributed( + {cluster}, + currentDatabase(), + test_table, + rand() + ) + """ + ) + + +def insert_data(cluster, row_num): + create_tables(cluster) + n1 = nodes[0] + n1.query( + f"INSERT INTO dist_table SELECT number % 4, number FROM numbers({row_num})" + ) + n1.query("SYSTEM FLUSH DISTRIBUTED dist_table") + + +@pytest.mark.parametrize("custom_key", ["sipHash64(key)", "key"]) +@pytest.mark.parametrize("filter_type", ["default", "range"]) +@pytest.mark.parametrize( + "cluster", + ["test_multiple_shards_multiple_replicas", "test_single_shard_multiple_replicas"], +) +def test_parallel_replicas_custom_key(start_cluster, cluster, custom_key, filter_type): + for node in nodes: + node.rotate_logs() + + row_num = 1000 + insert_data(cluster, row_num) + + expected_result = "" + for i in range(4): + expected_result += f"{i}\t250\n" + + n1 = nodes[0] + assert ( + n1.query( + "SELECT key, count() FROM dist_table GROUP BY key ORDER BY key", + settings={ + "prefer_localhost_replica": 0, + "max_parallel_replicas": 4, + "parallel_replicas_custom_key": custom_key, + "parallel_replicas_custom_key_filter_type": filter_type, + }, + ) + == expected_result + ) + + if cluster == "test_multiple_shards_multiple_replicas": + # we simply process query on all replicas for each shard by appending the filter on replica + assert all( + node.contains_in_log("Processing query on a replica using custom_key") + for node in nodes + ) + else: + # we first transform all replicas into shards and then append for each shard filter + assert n1.contains_in_log( + "Single shard cluster used with custom_key, transforming replicas into virtual shards" + ) diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 9f617369859..51952ac1eb7 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -285,6 +285,56 @@ def avro_confluent_message(schema_registry_client, value): # Tests +def test_kafka_prohibited_column_types(kafka_cluster): + def assert_returned_exception(e): + assert e.value.returncode == 36 + assert ( + "KafkaEngine doesn't support DEFAULT/MATERIALIZED/EPHEMERAL/ALIAS expressions for columns." + in str(e.value) + ) + + # check column with DEFAULT expression + with pytest.raises(QueryRuntimeException) as exception: + instance.query( + """ + CREATE TABLE test.kafka (a Int, b Int DEFAULT 0) + ENGINE = Kafka('{kafka_broker}:19092', '{kafka_topic_new}', '{kafka_group_name_new}', '{kafka_format_json_each_row}', '\\n') + """ + ) + assert_returned_exception(exception) + + # check EPHEMERAL + with pytest.raises(QueryRuntimeException) as exception: + instance.query( + """ + CREATE TABLE test.kafka (a Int, b Int EPHEMERAL) + ENGINE = Kafka('{kafka_broker}:19092', '{kafka_topic_new}', '{kafka_group_name_new}', '{kafka_format_json_each_row}', '\\n') + """ + ) + assert_returned_exception(exception) + + # check ALIAS + with pytest.raises(QueryRuntimeException) as exception: + instance.query( + """ + CREATE TABLE test.kafka (a Int, b String Alias toString(a)) + ENGINE = Kafka('{kafka_broker}:19092', '{kafka_topic_new}', '{kafka_group_name_new}', '{kafka_format_json_each_row}', '\\n') + """ + ) + assert_returned_exception(exception) + + # check MATERIALIZED + # check ALIAS + with pytest.raises(QueryRuntimeException) as exception: + instance.query( + """ + CREATE TABLE test.kafka (a Int, b String MATERIALIZED toString(a)) + ENGINE = Kafka('{kafka_broker}:19092', '{kafka_topic_new}', '{kafka_group_name_new}', '{kafka_format_json_each_row}', '\\n') + """ + ) + assert_returned_exception(exception) + + def test_kafka_settings_old_syntax(kafka_cluster): assert TSV( instance.query( diff --git a/tests/queries/0_stateless/00534_filimonov.data b/tests/queries/0_stateless/00534_filimonov.data index 911a8e4d1f3..eb4500877e5 100644 --- a/tests/queries/0_stateless/00534_filimonov.data +++ b/tests/queries/0_stateless/00534_filimonov.data @@ -276,10 +276,14 @@ SELECT runningDifference(CAST( 0 AS Nullable(Int8))); SELECT runningDifference(CAST( 0 AS Nullable(Int16))); SELECT runningDifference(CAST( 0 AS Nullable(Int32))); SELECT runningDifference(CAST( 0 AS Nullable(Int64))); +SELECT runningDifference(CAST( 0 AS Nullable(Int128))); +SELECT runningDifference(CAST( 0 AS Nullable(Int256))); SELECT runningDifference(CAST( 0 AS Nullable(UInt8))); SELECT runningDifference(CAST( 0 AS Nullable(UInt16))); SELECT runningDifference(CAST( 0 AS Nullable(UInt32))); SELECT runningDifference(CAST( 0 AS Nullable(UInt64))); +SELECT runningDifference(CAST( 0 AS Nullable(UInt128))); +SELECT runningDifference(CAST( 0 AS Nullable(UInt256))); SELECT runningDifference(CAST( 0 AS Nullable(Float32))); SELECT runningDifference(CAST( 0 AS Nullable(Float64))); SELECT runningDifference(CAST( 0 AS Nullable(Date))); @@ -288,10 +292,14 @@ SELECT runningDifference(CAST(NULL AS Nullable(Int8))); SELECT runningDifference(CAST(NULL AS Nullable(Int16))); SELECT runningDifference(CAST(NULL AS Nullable(Int32))); SELECT runningDifference(CAST(NULL AS Nullable(Int64))); +SELECT runningDifference(CAST(NULL AS Nullable(Int128))); +SELECT runningDifference(CAST(NULL AS Nullable(Int256))); SELECT runningDifference(CAST(NULL AS Nullable(UInt8))); SELECT runningDifference(CAST(NULL AS Nullable(UInt16))); SELECT runningDifference(CAST(NULL AS Nullable(UInt32))); SELECT runningDifference(CAST(NULL AS Nullable(UInt64))); +SELECT runningDifference(CAST(NULL AS Nullable(UInt128))); +SELECT runningDifference(CAST(NULL AS Nullable(UInt256)); SELECT runningDifference(CAST(NULL AS Nullable(Float32))); SELECT runningDifference(CAST(NULL AS Nullable(Float64))); SELECT runningDifference(CAST(NULL AS Nullable(Date))); diff --git a/tests/queries/0_stateless/00621_regression_for_in_operator.reference b/tests/queries/0_stateless/00621_regression_for_in_operator.reference index 90f0a70449a..ab8bcf307eb 100644 --- a/tests/queries/0_stateless/00621_regression_for_in_operator.reference +++ b/tests/queries/0_stateless/00621_regression_for_in_operator.reference @@ -3,3 +3,43 @@ 2 2 2 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, table_name: default.regression_for_in_operator_view + WHERE + FUNCTION id: 4, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 6, column_name: g, result_type: String, source_id: 3 + CONSTANT id: 7, constant_value: Tuple_(\'5\', \'6\'), constant_value_type: Tuple(String, String) + SETTINGS allow_experimental_analyzer=1 +2 +2 +QUERY id: 0 + PROJECTION COLUMNS + count() UInt64 + PROJECTION + LIST id: 1, nodes: 1 + FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64 + JOIN TREE + TABLE id: 3, table_name: default.regression_for_in_operator_view + WHERE + FUNCTION id: 4, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + FUNCTION id: 6, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 2 + COLUMN id: 8, column_name: g, result_type: String, source_id: 3 + CONSTANT id: 9, constant_value: \'5\', constant_value_type: String + FUNCTION id: 10, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 11, nodes: 2 + COLUMN id: 8, column_name: g, result_type: String, source_id: 3 + CONSTANT id: 12, constant_value: \'6\', constant_value_type: String + SETTINGS allow_experimental_analyzer=1 diff --git a/tests/queries/0_stateless/00621_regression_for_in_operator.sql b/tests/queries/0_stateless/00621_regression_for_in_operator.sql index 273f930a90f..db1bcb4a39a 100644 --- a/tests/queries/0_stateless/00621_regression_for_in_operator.sql +++ b/tests/queries/0_stateless/00621_regression_for_in_operator.sql @@ -12,9 +12,13 @@ SELECT count() FROM regression_for_in_operator_view WHERE g IN ('5','6'); SET optimize_min_equality_disjunction_chain_length = 1; SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6'; +SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6' SETTINGS allow_experimental_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6' SETTINGS allow_experimental_analyzer = 1; SET optimize_min_equality_disjunction_chain_length = 3; SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6'; +SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6' SETTINGS allow_experimental_analyzer = 1; +EXPLAIN QUERY TREE SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6' SETTINGS allow_experimental_analyzer = 1; DROP TABLE regression_for_in_operator_view; DROP TABLE regression_for_in_operator; diff --git a/tests/queries/0_stateless/00653_running_difference.reference b/tests/queries/0_stateless/00653_running_difference.reference index e2833e0bb3e..624ce92ce0f 100644 --- a/tests/queries/0_stateless/00653_running_difference.reference +++ b/tests/queries/0_stateless/00653_running_difference.reference @@ -19,6 +19,30 @@ \N \N 2 +- +0 +1 +4 +5 +170141183460469231731687303715884105717 +- +0 +1 +4 +5 +170141183460469231731687303715884105718 +- +0 +1 +4 +5 +170141183460469231731687303715884105717 +- +0 +1 +4 +5 +170141183460469231731687303715884105718 --Date Difference-- \N \N diff --git a/tests/queries/0_stateless/00653_running_difference.sql b/tests/queries/0_stateless/00653_running_difference.sql index f2b4a7300b2..1f18cfc42a7 100644 --- a/tests/queries/0_stateless/00653_running_difference.sql +++ b/tests/queries/0_stateless/00653_running_difference.sql @@ -5,6 +5,14 @@ select '-'; select runningDifference(x) from (select arrayJoin([Null, 1]) as x); select '-'; select runningDifference(x) from (select arrayJoin([Null, Null, 1, 3, Null, Null, 5]) as x); +select '-'; +select runningDifference(x) from (select arrayJoin([0, 1, 5, 10, 170141183460469231731687303715884105727]::Array(UInt128)) as x); +select '-'; +select runningDifference(x) from (select arrayJoin([0, 1, 5, 10, 170141183460469231731687303715884105728]::Array(UInt256)) as x); +select '-'; +select runningDifference(x) from (select arrayJoin([0, 1, 5, 10, 170141183460469231731687303715884105727]::Array(Int128)) as x); +select '-'; +select runningDifference(x) from (select arrayJoin([0, 1, 5, 10, 170141183460469231731687303715884105728]::Array(Int256)) as x); select '--Date Difference--'; select runningDifference(x) from (select arrayJoin([Null, Null, toDate('1970-1-1'), toDate('1970-12-31'), Null, Null, toDate('2010-8-9')]) as x); select '-'; diff --git a/tests/queries/0_stateless/00736_disjunction_optimisation.reference b/tests/queries/0_stateless/00736_disjunction_optimisation.reference index afd698b425e..84477a64057 100644 --- a/tests/queries/0_stateless/00736_disjunction_optimisation.reference +++ b/tests/queries/0_stateless/00736_disjunction_optimisation.reference @@ -25,6 +25,81 @@ 3 21 3 22 3 23 +QUERY id: 0 + PROJECTION COLUMNS + k UInt64 + s UInt64 + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: k, result_type: UInt64, source_id: 3 + COLUMN id: 4, column_name: s, result_type: UInt64, source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.bug + WHERE + FUNCTION id: 5, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 6, nodes: 2 + FUNCTION id: 7, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 2 + COLUMN id: 9, column_name: k, result_type: UInt64, source_id: 3 + CONSTANT id: 10, constant_value: Tuple_(UInt64_1, UInt64_2, UInt64_3), constant_value_type: Tuple(UInt8, UInt8, UInt8) + FUNCTION id: 11, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 12, nodes: 2 + COLUMN id: 13, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 14, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8) + SETTINGS allow_experimental_analyzer=1 +1 21 +1 22 +1 23 +2 21 +2 22 +2 23 +3 21 +3 22 +3 23 +1 21 +1 22 +1 23 +2 21 +2 22 +2 23 +3 21 +3 22 +3 23 +QUERY id: 0 + PROJECTION COLUMNS + k UInt64 + s UInt64 + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: k, result_type: UInt64, source_id: 3 + COLUMN id: 4, column_name: s, result_type: UInt64, source_id: 3 + JOIN TREE + QUERY id: 3, is_subquery: 1 + PROJECTION COLUMNS + k UInt64 + s UInt64 + PROJECTION + LIST id: 5, nodes: 2 + COLUMN id: 6, column_name: k, result_type: UInt64, source_id: 7 + COLUMN id: 8, column_name: s, result_type: UInt64, source_id: 7 + JOIN TREE + TABLE id: 7, table_name: default.bug + WHERE + FUNCTION id: 9, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 10, nodes: 2 + COLUMN id: 11, column_name: k, result_type: UInt64, source_id: 7 + CONSTANT id: 12, constant_value: Tuple_(UInt64_1, UInt64_2, UInt64_3), constant_value_type: Tuple(UInt8, UInt8, UInt8) + WHERE + FUNCTION id: 13, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 14, nodes: 2 + COLUMN id: 15, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 16, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8) + SETTINGS allow_experimental_analyzer=1 1 1 21 1 1 1 1 1 22 0 1 1 1 1 23 0 0 1 @@ -34,42 +109,6 @@ 3 1 21 1 1 1 3 1 22 0 1 1 3 1 23 0 0 1 -21 1 -22 1 -23 1 -21 1 -22 1 -23 1 -21 1 -22 1 -23 1 -1 21 -1 22 -1 23 -2 21 -2 22 -2 23 -3 21 -3 22 -3 23 -1 21 -1 22 -1 23 -2 21 -2 22 -2 23 -3 21 -3 22 -3 23 -1 21 -1 22 -1 23 -2 21 -2 22 -2 23 -3 21 -3 22 -3 23 1 1 21 1 1 1 1 1 22 0 1 1 1 1 23 0 0 1 @@ -79,6 +118,41 @@ 3 1 21 1 1 1 3 1 22 0 1 1 3 1 23 0 0 1 +QUERY id: 0 + PROJECTION COLUMNS + k UInt64 + or(equals(k, 1), equals(k, 2), equals(k, 3)) UInt8 + s UInt64 + equals(s, 21) UInt8 + or(equals(s, 21), equals(s, 22)) UInt8 + or(equals(s, 21), equals(s, 22), equals(s, 23)) UInt8 + PROJECTION + LIST id: 1, nodes: 6 + COLUMN id: 2, column_name: k, result_type: UInt64, source_id: 3 + FUNCTION id: 4, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 2, column_name: k, result_type: UInt64, source_id: 3 + CONSTANT id: 6, constant_value: Tuple_(UInt64_1, UInt64_2, UInt64_3), constant_value_type: Tuple(UInt8, UInt8, UInt8) + COLUMN id: 7, column_name: s, result_type: UInt64, source_id: 3 + FUNCTION id: 8, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 9, nodes: 2 + COLUMN id: 7, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 10, constant_value: UInt64_21, constant_value_type: UInt8 + FUNCTION id: 11, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 12, nodes: 2 + COLUMN id: 7, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 13, constant_value: Tuple_(UInt64_21, UInt64_22), constant_value_type: Tuple(UInt8, UInt8) + FUNCTION id: 14, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 15, nodes: 2 + COLUMN id: 7, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 16, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8) + JOIN TREE + TABLE id: 3, table_name: default.bug + SETTINGS allow_experimental_analyzer=1 21 1 22 1 23 1 @@ -88,3 +162,256 @@ 21 1 22 1 23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +QUERY id: 0 + PROJECTION COLUMNS + s UInt64 + or(equals(s, 21), equals(s, 22), equals(s, 23)) UInt8 + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: s, result_type: UInt64, source_id: 3 + FUNCTION id: 4, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 2, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 6, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8) + JOIN TREE + TABLE id: 3, table_name: default.bug + SETTINGS allow_experimental_analyzer=1 +1 21 +1 22 +1 23 +2 21 +2 22 +2 23 +3 21 +3 22 +3 23 +1 21 +1 22 +1 23 +2 21 +2 22 +2 23 +3 21 +3 22 +3 23 +1 21 +1 22 +1 23 +2 21 +2 22 +2 23 +3 21 +3 22 +3 23 +QUERY id: 0 + PROJECTION COLUMNS + k UInt64 + s UInt64 + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: k, result_type: UInt64, source_id: 3 + COLUMN id: 4, column_name: s, result_type: UInt64, source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.bug + WHERE + FUNCTION id: 5, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 6, nodes: 2 + FUNCTION id: 7, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 2 + COLUMN id: 9, column_name: k, result_type: UInt64, source_id: 3 + CONSTANT id: 10, constant_value: Tuple_(UInt64_1, UInt64_2, UInt64_3), constant_value_type: Tuple(UInt8, UInt8, UInt8) + FUNCTION id: 11, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 12, nodes: 2 + COLUMN id: 13, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 14, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8) + SETTINGS allow_experimental_analyzer=1 +1 21 +1 22 +1 23 +2 21 +2 22 +2 23 +3 21 +3 22 +3 23 +1 21 +1 22 +1 23 +2 21 +2 22 +2 23 +3 21 +3 22 +3 23 +QUERY id: 0 + PROJECTION COLUMNS + k UInt64 + s UInt64 + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: k, result_type: UInt64, source_id: 3 + COLUMN id: 4, column_name: s, result_type: UInt64, source_id: 3 + JOIN TREE + QUERY id: 3, is_subquery: 1 + PROJECTION COLUMNS + k UInt64 + s UInt64 + PROJECTION + LIST id: 5, nodes: 2 + COLUMN id: 6, column_name: k, result_type: UInt64, source_id: 7 + COLUMN id: 8, column_name: s, result_type: UInt64, source_id: 7 + JOIN TREE + TABLE id: 7, table_name: default.bug + WHERE + FUNCTION id: 9, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 10, nodes: 2 + COLUMN id: 11, column_name: k, result_type: UInt64, source_id: 7 + CONSTANT id: 12, constant_value: Tuple_(UInt64_1, UInt64_2, UInt64_3), constant_value_type: Tuple(UInt8, UInt8, UInt8) + WHERE + FUNCTION id: 13, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 14, nodes: 2 + COLUMN id: 15, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 16, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8) + SETTINGS allow_experimental_analyzer=1 +1 1 21 1 1 1 +1 1 22 0 1 1 +1 1 23 0 0 1 +2 1 21 1 1 1 +2 1 22 0 1 1 +2 1 23 0 0 1 +3 1 21 1 1 1 +3 1 22 0 1 1 +3 1 23 0 0 1 +1 1 21 1 1 1 +1 1 22 0 1 1 +1 1 23 0 0 1 +2 1 21 1 1 1 +2 1 22 0 1 1 +2 1 23 0 0 1 +3 1 21 1 1 1 +3 1 22 0 1 1 +3 1 23 0 0 1 +QUERY id: 0 + PROJECTION COLUMNS + k UInt64 + or(equals(k, 1), equals(k, 2), equals(k, 3)) UInt8 + s UInt64 + equals(s, 21) UInt8 + or(equals(s, 21), equals(s, 22)) UInt8 + or(equals(s, 21), equals(s, 22), equals(s, 23)) UInt8 + PROJECTION + LIST id: 1, nodes: 6 + COLUMN id: 2, column_name: k, result_type: UInt64, source_id: 3 + FUNCTION id: 4, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 2, column_name: k, result_type: UInt64, source_id: 3 + CONSTANT id: 6, constant_value: Tuple_(UInt64_1, UInt64_2, UInt64_3), constant_value_type: Tuple(UInt8, UInt8, UInt8) + COLUMN id: 7, column_name: s, result_type: UInt64, source_id: 3 + FUNCTION id: 8, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 9, nodes: 2 + COLUMN id: 7, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 10, constant_value: UInt64_21, constant_value_type: UInt8 + FUNCTION id: 11, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 12, nodes: 2 + FUNCTION id: 13, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 14, nodes: 2 + COLUMN id: 7, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 15, constant_value: UInt64_21, constant_value_type: UInt8 + FUNCTION id: 16, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 17, nodes: 2 + COLUMN id: 7, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 18, constant_value: UInt64_22, constant_value_type: UInt8 + FUNCTION id: 19, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 20, nodes: 2 + COLUMN id: 7, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 21, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8) + JOIN TREE + TABLE id: 3, table_name: default.bug + SETTINGS allow_experimental_analyzer=1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +QUERY id: 0 + PROJECTION COLUMNS + s UInt64 + or(equals(s, 21), equals(s, 22), equals(s, 23)) UInt8 + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: s, result_type: UInt64, source_id: 3 + FUNCTION id: 4, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 2, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 6, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8) + JOIN TREE + TABLE id: 3, table_name: default.bug + SETTINGS allow_experimental_analyzer=1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +21 1 +22 1 +23 1 +QUERY id: 0 + PROJECTION COLUMNS + s UInt64 + or(equals(s, 21), equals(22, s), equals(23, s)) UInt8 + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: s, result_type: UInt64, source_id: 3 + FUNCTION id: 4, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 2, column_name: s, result_type: UInt64, source_id: 3 + CONSTANT id: 6, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8) + JOIN TREE + TABLE id: 3, table_name: default.bug + SETTINGS allow_experimental_analyzer=1 diff --git a/tests/queries/0_stateless/00736_disjunction_optimisation.sql b/tests/queries/0_stateless/00736_disjunction_optimisation.sql index 700221ef7f0..e5bfc81f7ae 100644 --- a/tests/queries/0_stateless/00736_disjunction_optimisation.sql +++ b/tests/queries/0_stateless/00736_disjunction_optimisation.sql @@ -5,17 +5,45 @@ insert into bug values(1,21),(1,22),(1,23),(2,21),(2,22),(2,23),(3,21),(3,22),(3 set optimize_min_equality_disjunction_chain_length = 2; select * from bug; + select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23); +select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1; +explain query tree select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;; + select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23); +select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;; +explain query tree select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;; + select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug; +select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;; +explain query tree select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;; + select s, (s=21 or s=22 or s=23) from bug; +select s, (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;; +explain query tree select s, (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;; set optimize_min_equality_disjunction_chain_length = 3; select * from bug; + select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23); +select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1; +explain query tree select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;; + select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23); +select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;; +explain query tree select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;; + select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug; +select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;; +explain query tree select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;; + select s, (s=21 or s=22 or s=23) from bug; +select s, (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;; +explain query tree select s, (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;; + +select s, (s=21 or 22=s or 23=s) from bug; +select s, (s=21 or 22=s or 23=s) from bug SETTINGS allow_experimental_analyzer = 1;; +explain query tree select s, (s=21 or 22=s or 23=s) from bug SETTINGS allow_experimental_analyzer = 1;; DROP TABLE bug; diff --git a/tests/queries/0_stateless/00965_logs_level_bugfix.reference b/tests/queries/0_stateless/00965_logs_level_bugfix.reference index 52396b3fe79..affd41b780b 100644 --- a/tests/queries/0_stateless/00965_logs_level_bugfix.reference +++ b/tests/queries/0_stateless/00965_logs_level_bugfix.reference @@ -2,7 +2,6 @@ . . - . - diff --git a/tests/queries/0_stateless/01013_repeat_function.reference b/tests/queries/0_stateless/01013_repeat_function.reference index 46bb248a99a..ea0dadd524f 100644 --- a/tests/queries/0_stateless/01013_repeat_function.reference +++ b/tests/queries/0_stateless/01013_repeat_function.reference @@ -1,7 +1,7 @@ abcabcabcabcabcabcabcabcabcabc abcabcabc -sdfggsdfgg -xywq + + abcabcabcabcabcabcabcabcabcabcabcabc sdfggsdfggsdfggsdfggsdfggsdfggsdfggsdfggsdfggsdfgg @@ -20,8 +20,8 @@ sdfggsdfggsdfggsdfggsdfggsdfggsdfggsdfggsdfggsdfgg xywqxywqxywqxywqxywqxywqxywqxywqxywqxywq plkfplkfplkfplkfplkfplkfplkfplkfplkfplkf abcabcabc -abcabc -abc + + abcabcabcabcabcabcabcabcabcabcabcabc abcabcabcabcabcabcabcabcabcabc diff --git a/tests/queries/0_stateless/01013_repeat_function.sql b/tests/queries/0_stateless/01013_repeat_function.sql index 85b0c16b4ab..b29cc032f28 100644 --- a/tests/queries/0_stateless/01013_repeat_function.sql +++ b/tests/queries/0_stateless/01013_repeat_function.sql @@ -3,20 +3,20 @@ DROP TABLE IF EXISTS defaults; CREATE TABLE defaults ( strings String, - u8 UInt8, + i8 Int8, u16 UInt16, u32 UInt32, u64 UInt64 )ENGINE = Memory(); -INSERT INTO defaults values ('abc', 3, 12, 4, 56) ('sdfgg', 2, 10, 21, 200) ('xywq', 1, 4, 9, 5) ('plkf', 0, 5, 7,77); +INSERT INTO defaults values ('abc', 3, 12, 4, 56) ('sdfgg', -2, 10, 21, 200) ('xywq', -1, 4, 9, 5) ('plkf', 0, 5, 7,77); -SELECT repeat(strings, u8) FROM defaults; +SELECT repeat(strings, i8) FROM defaults; SELECT repeat(strings, u16) FROM defaults; SELECT repeat(strings, u32) from defaults; SELECT repeat(strings, u64) FROM defaults; SELECT repeat(strings, 10) FROM defaults; -SELECT repeat('abc', u8) FROM defaults; +SELECT repeat('abc', i8) FROM defaults; SELECT repeat('abc', u16) FROM defaults; SELECT repeat('abc', u32) FROM defaults; SELECT repeat('abc', u64) FROM defaults; diff --git a/tests/queries/0_stateless/01308_orc_output_format_arrays.reference b/tests/queries/0_stateless/01308_orc_output_format_arrays.reference index 1f9646ac112..7feea7cec35 100644 Binary files a/tests/queries/0_stateless/01308_orc_output_format_arrays.reference and b/tests/queries/0_stateless/01308_orc_output_format_arrays.reference differ diff --git a/tests/queries/0_stateless/01308_orc_output_format_arrays.sh b/tests/queries/0_stateless/01308_orc_output_format_arrays.sh index 1d9aea353b6..498854874cf 100755 --- a/tests/queries/0_stateless/01308_orc_output_format_arrays.sh +++ b/tests/queries/0_stateless/01308_orc_output_format_arrays.sh @@ -11,7 +11,7 @@ $CLICKHOUSE_CLIENT --query="CREATE TABLE orc (array1 Array(Int32), array2 Array( $CLICKHOUSE_CLIENT --query="INSERT INTO orc VALUES ([1,2,3,4,5], [[1,2], [3,4], [5]]), ([42], [[42, 42], [42]])"; -$CLICKHOUSE_CLIENT --query="SELECT * FROM orc FORMAT ORC"; +$CLICKHOUSE_CLIENT --query="SELECT * FROM orc FORMAT ORC SETTINGS output_format_orc_compression_method='none'" | md5sum; $CLICKHOUSE_CLIENT --query="DROP TABLE orc"; diff --git a/tests/queries/0_stateless/01705_normalize_create_alter_function_names.reference b/tests/queries/0_stateless/01705_normalize_create_alter_function_names.reference index b6f5fe99ca1..b5b93c34c00 100644 --- a/tests/queries/0_stateless/01705_normalize_create_alter_function_names.reference +++ b/tests/queries/0_stateless/01705_normalize_create_alter_function_names.reference @@ -1,2 +1,2 @@ -CREATE TABLE default.x\n(\n `i` Int32,\n INDEX mm rand() TYPE minmax GRANULARITY 1,\n INDEX nn rand() TYPE minmax GRANULARITY 1,\n PROJECTION p\n (\n SELECT max(i)\n ),\n PROJECTION p2\n (\n SELECT min(i)\n )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/default/x\', \'r\')\nORDER BY i\nSETTINGS index_granularity = 8192 -metadata format version: 1\ndate column: \nsampling expression: \nindex granularity: 8192\nmode: 0\nsign column: \nprimary key: i\ndata format version: 1\npartition key: \nindices: mm rand() TYPE minmax GRANULARITY 1, nn rand() TYPE minmax GRANULARITY 1\nprojections: p (SELECT max(i)), p2 (SELECT min(i))\ngranularity bytes: 10485760\n +CREATE TABLE default.x\n(\n `i` Int32,\n INDEX mm log2(i) TYPE minmax GRANULARITY 1,\n INDEX nn log2(i) TYPE minmax GRANULARITY 1,\n PROJECTION p\n (\n SELECT max(i)\n ),\n PROJECTION p2\n (\n SELECT min(i)\n )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/default/x\', \'r\')\nORDER BY i\nSETTINGS index_granularity = 8192 +metadata format version: 1\ndate column: \nsampling expression: \nindex granularity: 8192\nmode: 0\nsign column: \nprimary key: i\ndata format version: 1\npartition key: \nindices: mm log2(i) TYPE minmax GRANULARITY 1, nn log2(i) TYPE minmax GRANULARITY 1\nprojections: p (SELECT max(i)), p2 (SELECT min(i))\ngranularity bytes: 10485760\n diff --git a/tests/queries/0_stateless/01705_normalize_create_alter_function_names.sql b/tests/queries/0_stateless/01705_normalize_create_alter_function_names.sql index 683bd271405..be0f7e8b710 100644 --- a/tests/queries/0_stateless/01705_normalize_create_alter_function_names.sql +++ b/tests/queries/0_stateless/01705_normalize_create_alter_function_names.sql @@ -2,9 +2,9 @@ drop table if exists x; -create table x(i int, index mm RAND() type minmax granularity 1, projection p (select MAX(i))) engine ReplicatedMergeTree('/clickhouse/tables/{database}/x', 'r') order by i; +create table x(i int, index mm LOG2(i) type minmax granularity 1, projection p (select MAX(i))) engine ReplicatedMergeTree('/clickhouse/tables/{database}/x', 'r') order by i; -alter table x add index nn RAND() type minmax granularity 1, add projection p2 (select MIN(i)); +alter table x add index nn LOG2(i) type minmax granularity 1, add projection p2 (select MIN(i)); show create x; diff --git a/tests/queries/0_stateless/02232_dist_insert_send_logs_level_hung.sh b/tests/queries/0_stateless/02232_dist_insert_send_logs_level_hung.sh index 5ed94148bc1..734cef06214 100755 --- a/tests/queries/0_stateless/02232_dist_insert_send_logs_level_hung.sh +++ b/tests/queries/0_stateless/02232_dist_insert_send_logs_level_hung.sh @@ -49,7 +49,16 @@ insert_client_opts=( timeout 250s $CLICKHOUSE_CLIENT "${client_opts[@]}" "${insert_client_opts[@]}" -q "insert into function remote('127.2', currentDatabase(), in_02232) select * from numbers(1e6)" # Kill underlying query of remote() to make KILL faster -timeout 60s $CLICKHOUSE_CLIENT "${client_opts[@]}" -q "KILL QUERY WHERE Settings['log_comment'] = '$CLICKHOUSE_LOG_COMMENT' SYNC" --format Null +# This test is reproducing very interesting bahaviour. +# The block size is 1, so the secondary query creates InterpreterSelectQuery for each row due to pushing to the MV. +# It works extremely slow, and the initial query produces new blocks and writes them to the socket much faster +# then the secondary query can read and process them. Therefore, it fills network buffers in the kernel. +# Once a buffer in the kernel is full, send(...) blocks until the secondary query will finish processing data +# that it already has in ReadBufferFromPocoSocket and call recv. +# Or until the kernel will decide to resize the buffer (seems like it has non-trivial rules for that). +# Anyway, it may look like the initial query got stuck, but actually it did not. +# Moreover, the initial query cannot be killed at that point, so KILL QUERY ... SYNC will get "stuck" as well. +timeout 30s $CLICKHOUSE_CLIENT "${client_opts[@]}" -q "KILL QUERY WHERE query like '%INSERT INTO $CLICKHOUSE_DATABASE.in_02232%' SYNC" --format Null echo $? $CLICKHOUSE_CLIENT "${client_opts[@]}" -nm -q " diff --git a/tests/queries/0_stateless/02343_group_by_use_nulls.sql b/tests/queries/0_stateless/02343_group_by_use_nulls.sql index a979a78be0d..e1d4021a943 100644 --- a/tests/queries/0_stateless/02343_group_by_use_nulls.sql +++ b/tests/queries/0_stateless/02343_group_by_use_nulls.sql @@ -1,3 +1,4 @@ +set optimize_group_by_function_keys=0; -- { echoOn } SELECT number, number % 2, sum(number) AS val FROM numbers(10) diff --git a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh index 8262cd7eab5..65d0b3f434f 100755 --- a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh +++ b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Tags: no-fasttest, no-stress +# Tags: no-fasttest, no-asan, no-tsan, no-msan, no-ubsan, no-debug +# FIXME https://github.com/ClickHouse/ClickHouse/issues/47207 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02426_orc_bug.reference b/tests/queries/0_stateless/02426_orc_bug.reference index e5ad2b49289..baa88da2158 100644 Binary files a/tests/queries/0_stateless/02426_orc_bug.reference and b/tests/queries/0_stateless/02426_orc_bug.reference differ diff --git a/tests/queries/0_stateless/02426_orc_bug.sh b/tests/queries/0_stateless/02426_orc_bug.sh new file mode 100755 index 00000000000..7a7ad9f1783 --- /dev/null +++ b/tests/queries/0_stateless/02426_orc_bug.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --query="SELECT arrayJoin([[], [1]]) FORMAT ORC SETTINGS output_format_orc_compression_method='none'" | md5sum; + diff --git a/tests/queries/0_stateless/02426_orc_bug.sql b/tests/queries/0_stateless/02426_orc_bug.sql deleted file mode 100644 index 7016f1ceb70..00000000000 --- a/tests/queries/0_stateless/02426_orc_bug.sql +++ /dev/null @@ -1,3 +0,0 @@ --- Tags: no-fasttest - -SELECT arrayJoin([[], [1]]) FORMAT ORC; diff --git a/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.reference b/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.reference index dcfcac737c3..84589668d64 100644 --- a/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.reference +++ b/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.reference @@ -1,6 +1,62 @@ SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a IN (\'x\', \'y\') +QUERY id: 0 + PROJECTION COLUMNS + a LowCardinality(String) + PROJECTION + LIST id: 1, nodes: 1 + COLUMN id: 2, column_name: a, result_type: LowCardinality(String), source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.t_logical_expressions_optimizer_low_cardinality + WHERE + FUNCTION id: 4, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 2, column_name: a, result_type: LowCardinality(String), source_id: 3 + CONSTANT id: 6, constant_value: Tuple_(\'x\', \'y\'), constant_value_type: Tuple(String, String) + SETTINGS allow_experimental_analyzer=1 +SELECT a +FROM t_logical_expressions_optimizer_low_cardinality +WHERE (a = \'x\') OR (\'y\' = a) +QUERY id: 0 + PROJECTION COLUMNS + a LowCardinality(String) + PROJECTION + LIST id: 1, nodes: 1 + COLUMN id: 2, column_name: a, result_type: LowCardinality(String), source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.t_logical_expressions_optimizer_low_cardinality + WHERE + FUNCTION id: 4, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + COLUMN id: 2, column_name: a, result_type: LowCardinality(String), source_id: 3 + CONSTANT id: 6, constant_value: Tuple_(\'x\', \'y\'), constant_value_type: Tuple(String, String) + SETTINGS allow_experimental_analyzer=1 SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE (b = 0) OR (b = 1) +QUERY id: 0 + PROJECTION COLUMNS + a LowCardinality(String) + PROJECTION + LIST id: 1, nodes: 1 + COLUMN id: 2, column_name: a, result_type: LowCardinality(String), source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.t_logical_expressions_optimizer_low_cardinality + WHERE + FUNCTION id: 4, function_name: or, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 5, nodes: 2 + FUNCTION id: 6, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 7, nodes: 2 + COLUMN id: 8, column_name: b, result_type: UInt32, source_id: 3 + CONSTANT id: 9, constant_value: UInt64_0, constant_value_type: UInt8 + FUNCTION id: 10, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 11, nodes: 2 + COLUMN id: 8, column_name: b, result_type: UInt32, source_id: 3 + CONSTANT id: 12, constant_value: UInt64_1, constant_value_type: UInt8 + SETTINGS allow_experimental_analyzer=1 diff --git a/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.sql b/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.sql index be355a05675..14f8ad830e7 100644 --- a/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.sql +++ b/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.sql @@ -4,7 +4,11 @@ CREATE TABLE t_logical_expressions_optimizer_low_cardinality (a LowCardinality(S -- LowCardinality case, ignore optimize_min_equality_disjunction_chain_length limit, optimzer applied EXPLAIN SYNTAX SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a = 'x' OR a = 'y'; +EXPLAIN QUERY TREE SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a = 'x' OR a = 'y' SETTINGS allow_experimental_analyzer = 1; +EXPLAIN SYNTAX SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a = 'x' OR 'y' = a; +EXPLAIN QUERY TREE SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a = 'x' OR 'y' = a SETTINGS allow_experimental_analyzer = 1; -- Non-LowCardinality case, optimizer not applied for short chains EXPLAIN SYNTAX SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE b = 0 OR b = 1; +EXPLAIN QUERY TREE SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE b = 0 OR b = 1 SETTINGS allow_experimental_analyzer = 1; DROP TABLE t_logical_expressions_optimizer_low_cardinality; diff --git a/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.reference b/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.reference new file mode 100644 index 00000000000..50755627996 --- /dev/null +++ b/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.reference @@ -0,0 +1,256 @@ +-- { echoOn } +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY ROLLUP(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; +0 0 0 +1 1 1 +2 0 2 +3 1 3 +4 0 4 +5 1 5 +6 0 6 +7 1 7 +8 0 8 +9 1 9 +\N \N 45 +set optimize_group_by_function_keys = 0; +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY ROLLUP(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; +0 0 0 +0 \N 0 +1 1 1 +1 \N 1 +2 0 2 +2 \N 2 +3 1 3 +3 \N 3 +4 0 4 +4 \N 4 +5 1 5 +5 \N 5 +6 0 6 +6 \N 6 +7 1 7 +7 \N 7 +8 0 8 +8 \N 8 +9 1 9 +9 \N 9 +\N \N 45 +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY ROLLUP(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=0; +0 0 0 +0 0 0 +0 0 45 +1 0 1 +1 1 1 +2 0 2 +2 0 2 +3 0 3 +3 1 3 +4 0 4 +4 0 4 +5 0 5 +5 1 5 +6 0 6 +6 0 6 +7 0 7 +7 1 7 +8 0 8 +8 0 8 +9 0 9 +9 1 9 +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY CUBE(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; +0 0 0 +0 \N 0 +1 1 1 +1 \N 1 +2 0 2 +2 \N 2 +3 1 3 +3 \N 3 +4 0 4 +4 \N 4 +5 1 5 +5 \N 5 +6 0 6 +6 \N 6 +7 1 7 +7 \N 7 +8 0 8 +8 \N 8 +9 1 9 +9 \N 9 +\N 0 20 +\N 1 25 +\N \N 45 +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY CUBE(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=0; +0 0 0 +0 0 0 +0 0 20 +0 0 45 +0 1 25 +1 0 1 +1 1 1 +2 0 2 +2 0 2 +3 0 3 +3 1 3 +4 0 4 +4 0 4 +5 0 5 +5 1 5 +6 0 6 +6 0 6 +7 0 7 +7 1 7 +8 0 8 +8 0 8 +9 0 9 +9 1 9 +SELECT + number, + number % 2, + sum(number) AS val +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls = 1; +0 \N 0 +1 \N 1 +2 \N 2 +3 \N 3 +4 \N 4 +5 \N 5 +6 \N 6 +7 \N 7 +8 \N 8 +9 \N 9 +\N 0 20 +\N 1 25 +SELECT + number, + number % 2, + sum(number) AS val +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls = 0; +0 0 0 +0 0 20 +0 1 25 +1 0 1 +2 0 2 +3 0 3 +4 0 4 +5 0 5 +6 0 6 +7 0 7 +8 0 8 +9 0 9 +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY ROLLUP(number, number % 2) WITH TOTALS +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; +0 0 0 +0 \N 0 +1 1 1 +1 \N 1 +2 0 2 +2 \N 2 +3 1 3 +3 \N 3 +4 0 4 +4 \N 4 +5 1 5 +5 \N 5 +6 0 6 +6 \N 6 +7 1 7 +7 \N 7 +8 0 8 +8 \N 8 +9 1 9 +9 \N 9 +\N \N 45 + +0 0 45 +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY CUBE(number, number % 2) WITH TOTALS +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; +0 0 0 +0 \N 0 +1 1 1 +1 \N 1 +2 0 2 +2 \N 2 +3 1 3 +3 \N 3 +4 0 4 +4 \N 4 +5 1 5 +5 \N 5 +6 0 6 +6 \N 6 +7 1 7 +7 \N 7 +8 0 8 +8 \N 8 +9 1 9 +9 \N 9 +\N 0 20 +\N 1 25 +\N \N 45 + +0 0 45 +SELECT + number, + number % 2, + sum(number) AS val +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY 1, tuple(val) +SETTINGS group_by_use_nulls = 1, max_bytes_before_external_sort=10; +0 \N 0 +1 \N 1 +2 \N 2 +3 \N 3 +4 \N 4 +5 \N 5 +6 \N 6 +7 \N 7 +8 \N 8 +9 \N 9 +\N 0 20 +\N 1 25 diff --git a/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.sql b/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.sql new file mode 100644 index 00000000000..a4d4f2f8bc9 --- /dev/null +++ b/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.sql @@ -0,0 +1,85 @@ +SET allow_experimental_analyzer=1; + +-- { echoOn } +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY ROLLUP(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; + +set optimize_group_by_function_keys = 0; + +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY ROLLUP(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; + +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY ROLLUP(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=0; + +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY CUBE(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; + +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY CUBE(number, number % 2) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=0; + +SELECT + number, + number % 2, + sum(number) AS val +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls = 1; + +SELECT + number, + number % 2, + sum(number) AS val +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls = 0; + +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY ROLLUP(number, number % 2) WITH TOTALS +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; + +SELECT number, number % 2, sum(number) AS val +FROM numbers(10) +GROUP BY CUBE(number, number % 2) WITH TOTALS +ORDER BY (number, number % 2, val) +SETTINGS group_by_use_nulls=1; + +SELECT + number, + number % 2, + sum(number) AS val +FROM numbers(10) +GROUP BY + GROUPING SETS ( + (number), + (number % 2) + ) +ORDER BY 1, tuple(val) +SETTINGS group_by_use_nulls = 1, max_bytes_before_external_sort=10; diff --git a/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key.reference b/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key.reference new file mode 100644 index 00000000000..8d0f56ba185 --- /dev/null +++ b/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key.reference @@ -0,0 +1,173 @@ +query='SELECT * FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key)' with custom_key='sipHash64(x)' +filter_type='default' max_replicas=1 prefer_localhost_replica=0 +Hello +filter_type='default' max_replicas=2 prefer_localhost_replica=0 +Hello +filter_type='default' max_replicas=3 prefer_localhost_replica=0 +Hello +filter_type='range' max_replicas=1 prefer_localhost_replica=0 +Hello +filter_type='range' max_replicas=2 prefer_localhost_replica=0 +Hello +filter_type='range' max_replicas=3 prefer_localhost_replica=0 +Hello +filter_type='default' max_replicas=1 prefer_localhost_replica=1 +Hello +filter_type='default' max_replicas=2 prefer_localhost_replica=1 +Hello +filter_type='default' max_replicas=3 prefer_localhost_replica=1 +Hello +filter_type='range' max_replicas=1 prefer_localhost_replica=1 +Hello +filter_type='range' max_replicas=2 prefer_localhost_replica=1 +Hello +filter_type='range' max_replicas=3 prefer_localhost_replica=1 +Hello +query='SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key) GROUP BY y ORDER BY y' with custom_key='y' +filter_type='default' max_replicas=1 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=2 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=3 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=1 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=2 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=3 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=1 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=2 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=3 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=1 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=2 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=3 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +query='SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key) GROUP BY y ORDER BY y' with custom_key='cityHash64(y)' +filter_type='default' max_replicas=1 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=2 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=3 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=1 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=2 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=3 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=1 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=2 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=3 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=1 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=2 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=3 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +query='SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key) GROUP BY y ORDER BY y' with custom_key='cityHash64(y) + 1' +filter_type='default' max_replicas=1 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=2 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=3 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=1 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=2 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=3 prefer_localhost_replica=0 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=1 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=2 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='default' max_replicas=3 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=1 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=2 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +filter_type='range' max_replicas=3 prefer_localhost_replica=1 +0 334 +1 333 +2 333 +1 diff --git a/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key.sh b/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key.sh new file mode 100755 index 00000000000..3035a191c8f --- /dev/null +++ b/tests/queries/0_stateless/02535_max_parallel_replicas_custom_key.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Tags: no-parallel, long + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +function run_with_custom_key { + echo "query='$1' with custom_key='$2'" + for prefer_localhost_replica in 0 1; do + for filter_type in 'default' 'range'; do + for max_replicas in {1..3}; do + echo "filter_type='$filter_type' max_replicas=$max_replicas prefer_localhost_replica=$prefer_localhost_replica" + query="$1 SETTINGS max_parallel_replicas=$max_replicas\ + , parallel_replicas_custom_key='$2'\ + , parallel_replicas_custom_key_filter_type='$filter_type'\ + , prefer_localhost_replica=$prefer_localhost_replica" + $CLICKHOUSE_CLIENT --query="$query" + done + done + done +} + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS 02535_custom_key"; + +$CLICKHOUSE_CLIENT --query="CREATE TABLE 02535_custom_key (x String) ENGINE = MergeTree ORDER BY x"; +$CLICKHOUSE_CLIENT --query="INSERT INTO 02535_custom_key VALUES ('Hello')"; + +run_with_custom_key "SELECT * FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key)" "sipHash64(x)" + +$CLICKHOUSE_CLIENT --query="DROP TABLE 02535_custom_key" + +$CLICKHOUSE_CLIENT --query="CREATE TABLE 02535_custom_key (x String, y Int32) ENGINE = MergeTree ORDER BY cityHash64(x)" +$CLICKHOUSE_CLIENT --query="INSERT INTO 02535_custom_key SELECT toString(number), number % 3 FROM numbers(1000)" + +function run_count_with_custom_key { + run_with_custom_key "SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key) GROUP BY y ORDER BY y" "$1" +} + +run_count_with_custom_key "y" +run_count_with_custom_key "cityHash64(y)" +run_count_with_custom_key "cityHash64(y) + 1" + +$CLICKHOUSE_CLIENT --query="SELECT count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key) as t1 JOIN 02535_custom_key USING y" --parallel_replicas_custom_key="y" --send_logs_level="trace" 2>&1 | grep -Fac "Joins are not supported with parallel replicas" + +$CLICKHOUSE_CLIENT --query="DROP TABLE 02535_custom_key" diff --git a/tests/queries/0_stateless/02570_fallback_from_async_insert.sh b/tests/queries/0_stateless/02570_fallback_from_async_insert.sh index 9c158d6241b..d7c8944b89d 100755 --- a/tests/queries/0_stateless/02570_fallback_from_async_insert.sh +++ b/tests/queries/0_stateless/02570_fallback_from_async_insert.sh @@ -47,6 +47,7 @@ $CLICKHOUSE_CLIENT --query "SYSTEM FLUSH LOGS" $CLICKHOUSE_CLIENT --query " SELECT 'id_' || splitByChar('_', query_id)[1] AS id FROM system.text_log WHERE query_id LIKE '%$query_id_suffix' AND message LIKE '%$message%' + ORDER BY id " $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS t_async_insert_fallback" diff --git a/tests/queries/0_stateless/02572_materialized_views_ignore_errors.reference b/tests/queries/0_stateless/02572_materialized_views_ignore_errors.reference new file mode 100644 index 00000000000..fc2e6b78122 --- /dev/null +++ b/tests/queries/0_stateless/02572_materialized_views_ignore_errors.reference @@ -0,0 +1,28 @@ +-- { echoOn } +select * from data_02572 order by key; +insert into data_02572 settings materialized_views_ignore_errors=1 values (2); +select * from data_02572 order by key; +2 +-- check system.query_views_log +system flush logs; +-- lower(status) to pass through clickhouse-test "exception" check +select lower(status::String), errorCodeToName(exception_code) +from system.query_views_log where + view_name = concatWithSeparator('.', currentDatabase(), 'push_to_proxy_mv_02572') and + view_target = concatWithSeparator('.', currentDatabase(), 'proxy_02572') + order by event_date, event_time +; +exceptionwhileprocessing UNKNOWN_TABLE +-- materialized_views_ignore_errors=0 +insert into data_02572 values (1); -- { serverError UNKNOWN_TABLE } +select * from data_02572 order by key; +1 +2 +create table receiver_02572 as data_02572; +insert into data_02572 values (3); +select * from data_02572 order by key; +1 +2 +3 +select * from receiver_02572 order by key; +3 diff --git a/tests/queries/0_stateless/02572_materialized_views_ignore_errors.sql b/tests/queries/0_stateless/02572_materialized_views_ignore_errors.sql new file mode 100644 index 00000000000..2d1f824b9b1 --- /dev/null +++ b/tests/queries/0_stateless/02572_materialized_views_ignore_errors.sql @@ -0,0 +1,40 @@ +set prefer_localhost_replica=1; + +drop table if exists data_02572; +drop table if exists proxy_02572; +drop table if exists push_to_proxy_mv_02572; +drop table if exists receiver_02572; + +create table data_02572 (key Int) engine=Memory(); + +create table proxy_02572 (key Int) engine=Distributed('test_shard_localhost', currentDatabase(), 'receiver_02572'); +-- ensure that insert fails +insert into proxy_02572 values (1); -- { serverError UNKNOWN_TABLE } + +-- proxy data with MV +create materialized view push_to_proxy_mv_02572 to proxy_02572 as select * from data_02572; + +-- { echoOn } +select * from data_02572 order by key; + +insert into data_02572 settings materialized_views_ignore_errors=1 values (2); +select * from data_02572 order by key; +-- check system.query_views_log +system flush logs; +-- lower(status) to pass through clickhouse-test "exception" check +select lower(status::String), errorCodeToName(exception_code) +from system.query_views_log where + view_name = concatWithSeparator('.', currentDatabase(), 'push_to_proxy_mv_02572') and + view_target = concatWithSeparator('.', currentDatabase(), 'proxy_02572') + order by event_date, event_time +; + +-- materialized_views_ignore_errors=0 +insert into data_02572 values (1); -- { serverError UNKNOWN_TABLE } +select * from data_02572 order by key; + +create table receiver_02572 as data_02572; + +insert into data_02572 values (3); +select * from data_02572 order by key; +select * from receiver_02572 order by key; diff --git a/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.reference b/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.reference new file mode 100644 index 00000000000..d5446e756a3 --- /dev/null +++ b/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.reference @@ -0,0 +1,2 @@ +10 querystart OK +10 queryfinish OK diff --git a/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.sql b/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.sql new file mode 100644 index 00000000000..9568bc7af1a --- /dev/null +++ b/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.sql @@ -0,0 +1,26 @@ +-- Tags: no-parallel, no-replicated-database +-- Tag no-parallel: due to attaching to system.query_log +-- Tag no-replicated-database: Replicated database will has extra queries + +-- Attach MV to system.query_log and check that writing query_log will not fail + +set log_queries=1; + +drop table if exists log_proxy_02572; +drop table if exists push_to_logs_proxy_mv_02572; + +create table log_proxy_02572 as system.query_log engine=Distributed('test_shard_localhost', currentDatabase(), 'receiver_02572'); +create materialized view push_to_logs_proxy_mv_02572 to log_proxy_02572 as select * from system.query_log; + +select 1 format Null; +system flush logs; +system flush logs; + +drop table log_proxy_02572; +drop table push_to_logs_proxy_mv_02572; + +system flush logs; +-- lower() to pass through clickhouse-test "exception" check +select count(), lower(type::String), errorCodeToName(exception_code) + from system.query_log + where current_database = currentDatabase() group by 2, 3; diff --git a/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.reference b/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.reference new file mode 100644 index 00000000000..492b12dba56 --- /dev/null +++ b/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.reference @@ -0,0 +1,14 @@ +10 +10 +10 +10 +10 +10 +10 +10 +10 +10 +10 +10 +10 +10 diff --git a/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.sh b/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.sh new file mode 100755 index 00000000000..89b5147f026 --- /dev/null +++ b/tests/queries/0_stateless/02581_parquet_arrow_orc_compressions.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='none'" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='lz4'" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='snappy'" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='zstd'" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='brotli'" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Parquet settings output_format_parquet_compression_method='gzip'" | $CLICKHOUSE_LOCAL --input-format=Parquet -q "select count() from table" + +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format ORC settings output_format_orc_compression_method='none'" | $CLICKHOUSE_LOCAL --input-format=ORC -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format ORC settings output_format_orc_compression_method='lz4'" | $CLICKHOUSE_LOCAL --input-format=ORC -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format ORC settings output_format_orc_compression_method='zstd'" | $CLICKHOUSE_LOCAL --input-format=ORC -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format ORC settings output_format_orc_compression_method='zlib'" | $CLICKHOUSE_LOCAL --input-format=ORC -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format ORC settings output_format_orc_compression_method='snappy'" | $CLICKHOUSE_LOCAL --input-format=ORC -q "select count() from table" + + +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Arrow settings output_format_arrow_compression_method='none'" | $CLICKHOUSE_LOCAL --input-format=Arrow -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Arrow settings output_format_arrow_compression_method='lz4_frame'" | $CLICKHOUSE_LOCAL --input-format=Arrow -q "select count() from table" +$CLICKHOUSE_LOCAL -q "select * from numbers(10) format Arrow settings output_format_arrow_compression_method='zstd'" | $CLICKHOUSE_LOCAL --input-format=Arrow -q "select count() from table" + diff --git a/tests/queries/0_stateless/25337_width_bucket.reference b/tests/queries/0_stateless/02581_width_bucket.reference similarity index 100% rename from tests/queries/0_stateless/25337_width_bucket.reference rename to tests/queries/0_stateless/02581_width_bucket.reference diff --git a/tests/queries/0_stateless/25337_width_bucket.sql b/tests/queries/0_stateless/02581_width_bucket.sql similarity index 100% rename from tests/queries/0_stateless/25337_width_bucket.sql rename to tests/queries/0_stateless/02581_width_bucket.sql diff --git a/tests/queries/0_stateless/25339_analyzer_join_subquery_empty_column_list.reference b/tests/queries/0_stateless/02582_analyzer_join_subquery_empty_column_list.reference similarity index 100% rename from tests/queries/0_stateless/25339_analyzer_join_subquery_empty_column_list.reference rename to tests/queries/0_stateless/02582_analyzer_join_subquery_empty_column_list.reference diff --git a/tests/queries/0_stateless/25339_analyzer_join_subquery_empty_column_list.sql b/tests/queries/0_stateless/02582_analyzer_join_subquery_empty_column_list.sql similarity index 100% rename from tests/queries/0_stateless/25339_analyzer_join_subquery_empty_column_list.sql rename to tests/queries/0_stateless/02582_analyzer_join_subquery_empty_column_list.sql diff --git a/tests/queries/0_stateless/02664_async_reading_with_small_limit.reference b/tests/queries/0_stateless/02582_async_reading_with_small_limit.reference similarity index 100% rename from tests/queries/0_stateless/02664_async_reading_with_small_limit.reference rename to tests/queries/0_stateless/02582_async_reading_with_small_limit.reference diff --git a/tests/queries/0_stateless/02664_async_reading_with_small_limit.sql b/tests/queries/0_stateless/02582_async_reading_with_small_limit.sql similarity index 100% rename from tests/queries/0_stateless/02664_async_reading_with_small_limit.sql rename to tests/queries/0_stateless/02582_async_reading_with_small_limit.sql diff --git a/tests/queries/0_stateless/02670_map_literal_cast.reference b/tests/queries/0_stateless/02583_map_literal_cast.reference similarity index 100% rename from tests/queries/0_stateless/02670_map_literal_cast.reference rename to tests/queries/0_stateless/02583_map_literal_cast.reference diff --git a/tests/queries/0_stateless/02670_map_literal_cast.sql b/tests/queries/0_stateless/02583_map_literal_cast.sql similarity index 100% rename from tests/queries/0_stateless/02670_map_literal_cast.sql rename to tests/queries/0_stateless/02583_map_literal_cast.sql diff --git a/tests/queries/0_stateless/02674_range_ipv4.reference b/tests/queries/0_stateless/02584_range_ipv4.reference similarity index 100% rename from tests/queries/0_stateless/02674_range_ipv4.reference rename to tests/queries/0_stateless/02584_range_ipv4.reference diff --git a/tests/queries/0_stateless/02674_range_ipv4.sql b/tests/queries/0_stateless/02584_range_ipv4.sql similarity index 100% rename from tests/queries/0_stateless/02674_range_ipv4.sql rename to tests/queries/0_stateless/02584_range_ipv4.sql diff --git a/tests/queries/0_stateless/02585_query_status_deadlock.reference b/tests/queries/0_stateless/02585_query_status_deadlock.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02585_query_status_deadlock.sh b/tests/queries/0_stateless/02585_query_status_deadlock.sh new file mode 100755 index 00000000000..227ecb1c1b2 --- /dev/null +++ b/tests/queries/0_stateless/02585_query_status_deadlock.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +QUERY_ID="${CLICKHOUSE_DATABASE}_test_02585_query_to_kill_id_1" + +$CLICKHOUSE_CLIENT --query_id="$QUERY_ID" -n -q " +create temporary table tmp as select * from numbers(500000000); +select * from remote('127.0.0.2', 'system.numbers_mt') where number in (select * from tmp);" &> /dev/null & + +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS" + +while true +do + res=$($CLICKHOUSE_CLIENT -q "select query, event_time from system.query_log where query_id = '$QUERY_ID' and query like 'select%' limit 1") + if [ -n "$res" ]; then + break + fi + sleep 1 +done + +$CLICKHOUSE_CLIENT -q "kill query where query_id = '$QUERY_ID' sync" &> /dev/null + diff --git a/tests/queries/0_stateless/02668_logical_optimizer_removing_redundant_checks.reference b/tests/queries/0_stateless/02668_logical_optimizer_removing_redundant_checks.reference new file mode 100644 index 00000000000..d083e178586 --- /dev/null +++ b/tests/queries/0_stateless/02668_logical_optimizer_removing_redundant_checks.reference @@ -0,0 +1,89 @@ +1 test +3 another +QUERY id: 0 + PROJECTION COLUMNS + a Int32 + b LowCardinality(String) + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: a, result_type: Int32, source_id: 3 + COLUMN id: 4, column_name: b, result_type: LowCardinality(String), source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.02668_logical_optimizer + WHERE + FUNCTION id: 5, function_name: in, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 6, nodes: 2 + COLUMN id: 7, column_name: a, result_type: Int32, source_id: 3 + CONSTANT id: 8, constant_value: Tuple_(UInt64_1, UInt64_3), constant_value_type: Tuple(UInt8, UInt8) +1 test +QUERY id: 0 + PROJECTION COLUMNS + a Int32 + b LowCardinality(String) + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: a, result_type: Int32, source_id: 3 + COLUMN id: 4, column_name: b, result_type: LowCardinality(String), source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.02668_logical_optimizer + WHERE + FUNCTION id: 5, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 6, nodes: 2 + COLUMN id: 7, column_name: a, result_type: Int32, source_id: 3 + CONSTANT id: 8, constant_value: UInt64_1, constant_value_type: UInt8 +QUERY id: 0 + PROJECTION COLUMNS + a Int32 + b LowCardinality(String) + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: a, result_type: Int32, source_id: 3 + COLUMN id: 4, column_name: b, result_type: LowCardinality(String), source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.02668_logical_optimizer + WHERE + CONSTANT id: 5, constant_value: UInt64_0, constant_value_type: UInt8 +3 another +QUERY id: 0 + PROJECTION COLUMNS + a Int32 + b LowCardinality(String) + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: a, result_type: Int32, source_id: 3 + COLUMN id: 4, column_name: b, result_type: LowCardinality(String), source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.02668_logical_optimizer + WHERE + FUNCTION id: 5, function_name: and, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 6, nodes: 2 + FUNCTION id: 7, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 8, nodes: 2 + COLUMN id: 9, column_name: a, result_type: Int32, source_id: 3 + CONSTANT id: 10, constant_value: UInt64_3, constant_value_type: UInt8 + FUNCTION id: 11, function_name: equals, function_type: ordinary, result_type: LowCardinality(UInt8) + ARGUMENTS + LIST id: 12, nodes: 2 + COLUMN id: 13, column_name: b, result_type: LowCardinality(String), source_id: 3 + CONSTANT id: 14, constant_value: \'another\', constant_value_type: String +2 test2 +QUERY id: 0 + PROJECTION COLUMNS + a Int32 + b LowCardinality(String) + PROJECTION + LIST id: 1, nodes: 2 + COLUMN id: 2, column_name: a, result_type: Int32, source_id: 3 + COLUMN id: 4, column_name: b, result_type: LowCardinality(String), source_id: 3 + JOIN TREE + TABLE id: 3, table_name: default.02668_logical_optimizer + WHERE + FUNCTION id: 5, function_name: equals, function_type: ordinary, result_type: UInt8 + ARGUMENTS + LIST id: 6, nodes: 2 + COLUMN id: 7, column_name: a, result_type: Int32, source_id: 3 + CONSTANT id: 8, constant_value: UInt64_2, constant_value_type: UInt8 diff --git a/tests/queries/0_stateless/02668_logical_optimizer_removing_redundant_checks.sql b/tests/queries/0_stateless/02668_logical_optimizer_removing_redundant_checks.sql new file mode 100644 index 00000000000..f20ef412215 --- /dev/null +++ b/tests/queries/0_stateless/02668_logical_optimizer_removing_redundant_checks.sql @@ -0,0 +1,26 @@ +SET allow_experimental_analyzer = 1; + +DROP TABLE IF EXISTS 02668_logical_optimizer; + +CREATE TABLE 02668_logical_optimizer +(a Int32, b LowCardinality(String)) +ENGINE=Memory; + +INSERT INTO 02668_logical_optimizer VALUES (1, 'test'), (2, 'test2'), (3, 'another'); + +SET optimize_min_equality_disjunction_chain_length = 2; + +SELECT * FROM 02668_logical_optimizer WHERE a = 1 OR 3 = a OR 1 = a; +EXPLAIN QUERY TREE SELECT * FROM 02668_logical_optimizer WHERE a = 1 OR 3 = a OR 1 = a; + +SELECT * FROM 02668_logical_optimizer WHERE a = 1 OR 1 = a; +EXPLAIN QUERY TREE SELECT * FROM 02668_logical_optimizer WHERE a = 1 OR 1 = a; + +SELECT * FROM 02668_logical_optimizer WHERE a = 1 AND 2 = a; +EXPLAIN QUERY TREE SELECT * FROM 02668_logical_optimizer WHERE a = 1 AND 2 = a; + +SELECT * FROM 02668_logical_optimizer WHERE 3 = a AND b = 'another' AND a = 3; +EXPLAIN QUERY TREE SELECT * FROM 02668_logical_optimizer WHERE a = 3 AND b = 'another' AND a = 3; + +SELECT * FROM 02668_logical_optimizer WHERE a = 2 AND 2 = a; +EXPLAIN QUERY TREE SELECT * FROM 02668_logical_optimizer WHERE a = 2 AND 2 = a; diff --git a/tests/queries/0_stateless/02670_constant_skip_index.reference b/tests/queries/0_stateless/02670_constant_skip_index.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02670_constant_skip_index.sql b/tests/queries/0_stateless/02670_constant_skip_index.sql new file mode 100644 index 00000000000..97dd2ab33c9 --- /dev/null +++ b/tests/queries/0_stateless/02670_constant_skip_index.sql @@ -0,0 +1,25 @@ + +DROP TABLE IF EXISTS t_constant_index; + +CREATE TABLE t_constant_index +( + id UInt64, + INDEX t_constant_index 'foo' TYPE set(2) GRANULARITY 1 +) ENGINE = MergeTree +ORDER BY id; -- { serverError INCORRECT_QUERY } + +CREATE TABLE t_constant_index +( + id UInt64, + INDEX t_constant_index id + rand() TYPE set(2) GRANULARITY 1 +) ENGINE = MergeTree +ORDER BY id; -- { serverError BAD_ARGUMENTS } + +CREATE TABLE t_constant_index +( + id UInt64, + INDEX t_constant_index id * 2 TYPE set(2) GRANULARITY 1 +) ENGINE = MergeTree +ORDER BY id; + +DROP TABLE t_constant_index; diff --git a/tests/queries/0_stateless/02675_predicate_push_down_filled_join_fix.reference b/tests/queries/0_stateless/02675_predicate_push_down_filled_join_fix.reference new file mode 100644 index 00000000000..ecdb62c5cb5 --- /dev/null +++ b/tests/queries/0_stateless/02675_predicate_push_down_filled_join_fix.reference @@ -0,0 +1,33 @@ +Expression ((Project names + (Projection + ))) +Header: t1.id UInt64 + t1.value String + t2.value String +Actions: INPUT : 0 -> t1.id_0 UInt64 : 0 + INPUT : 1 -> t1.value_1 String : 1 + INPUT : 2 -> t2.value_2 String : 2 + ALIAS t1.id_0 :: 0 -> t1.id UInt64 : 3 + ALIAS t1.value_1 :: 1 -> t1.value String : 0 + ALIAS t2.value_2 :: 2 -> t2.value String : 1 +Positions: 3 0 1 + FilledJoin (Filled JOIN) + Header: t1.id_0 UInt64 + t1.value_1 String + t2.value_2 String + Filter (( + (JOIN actions + Change column names to column identifiers))) + Header: t1.id_0 UInt64 + t1.value_1 String + Filter column: equals(t1.id_0, 0_UInt8) (removed) + Actions: INPUT : 0 -> id UInt64 : 0 + INPUT : 1 -> value String : 1 + COLUMN Const(UInt8) -> 0_UInt8 UInt8 : 2 + ALIAS id :: 0 -> t1.id_0 UInt64 : 3 + ALIAS value :: 1 -> t1.value_1 String : 0 + FUNCTION equals(t1.id_0 : 3, 0_UInt8 :: 2) -> equals(t1.id_0, 0_UInt8) UInt8 : 1 + Positions: 1 3 0 + ReadFromMergeTree (default.test_table) + Header: id UInt64 + value String + ReadType: Default + Parts: 1 + Granules: 1 +0 Value JoinValue diff --git a/tests/queries/0_stateless/02675_predicate_push_down_filled_join_fix.sql b/tests/queries/0_stateless/02675_predicate_push_down_filled_join_fix.sql new file mode 100644 index 00000000000..78cb423216b --- /dev/null +++ b/tests/queries/0_stateless/02675_predicate_push_down_filled_join_fix.sql @@ -0,0 +1,26 @@ +SET allow_experimental_analyzer = 1; + +DROP TABLE IF EXISTS test_table; +CREATE TABLE test_table +( + id UInt64, + value String +) ENGINE=MergeTree ORDER BY id; + +INSERT INTO test_table VALUES (0, 'Value'); + +DROP TABLE IF EXISTS test_table_join; +CREATE TABLE test_table_join +( + id UInt64, + value String +) ENGINE = Join(All, inner, id); + +INSERT INTO test_table_join VALUES (0, 'JoinValue'); + +EXPLAIN header = 1, actions = 1 SELECT t1.id, t1.value, t2.value FROM test_table AS t1 INNER JOIN test_table_join AS t2 ON t1.id = t2.id WHERE t1.id = 0; + +SELECT t1.id, t1.value, t2.value FROM test_table AS t1 INNER JOIN test_table_join AS t2 ON t1.id = t2.id WHERE t1.id = 0; + +DROP TABLE test_table_join; +DROP TABLE test_table; diff --git a/tests/queries/0_stateless/25340_grace_hash_limit_race.reference b/tests/queries/0_stateless/25340_grace_hash_limit_race.reference new file mode 100644 index 00000000000..83b33d238da --- /dev/null +++ b/tests/queries/0_stateless/25340_grace_hash_limit_race.reference @@ -0,0 +1 @@ +1000 diff --git a/tests/queries/0_stateless/25340_grace_hash_limit_race.sql b/tests/queries/0_stateless/25340_grace_hash_limit_race.sql new file mode 100644 index 00000000000..55262ab2455 --- /dev/null +++ b/tests/queries/0_stateless/25340_grace_hash_limit_race.sql @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS test_grace_hash; + +CREATE TABLE test_grace_hash (id UInt32, value UInt64) ENGINE = MergeTree ORDER BY id; + +INSERT INTO test_grace_hash SELECT number, number % 100 = 0 FROM numbers(100000); + +SET join_algorithm = 'grace_hash'; + +SELECT count() FROM ( + SELECT f.id FROM test_grace_hash AS f + LEFT JOIN test_grace_hash AS d + ON f.id = d.id + LIMIT 1000 +); + +DROP TABLE test_grace_hash; diff --git a/tests/queries/1_stateful/00173_group_by_use_nulls.reference b/tests/queries/1_stateful/00173_group_by_use_nulls.reference index 02723bf14dd..e82b996ad3c 100644 --- a/tests/queries/1_stateful/00173_group_by_use_nulls.reference +++ b/tests/queries/1_stateful/00173_group_by_use_nulls.reference @@ -8,3 +8,25 @@ 59183 1336 33010362 1336 800784 1336 +-- { echoOn } +set allow_experimental_analyzer = 1; +SELECT + CounterID AS k, + quantileBFloat16(0.5)(ResolutionWidth) +FROM remote('127.0.0.{1,2}', test, hits) +GROUP BY k +ORDER BY + count() DESC, + CounterID ASC +LIMIT 10 +SETTINGS group_by_use_nulls = 1; +1704509 1384 +732797 1336 +598875 1384 +792887 1336 +3807842 1336 +25703952 1336 +716829 1384 +59183 1336 +33010362 1336 +800784 1336 diff --git a/tests/queries/1_stateful/00173_group_by_use_nulls.sql b/tests/queries/1_stateful/00173_group_by_use_nulls.sql index 7acacc4e579..8531e9efaf8 100644 --- a/tests/queries/1_stateful/00173_group_by_use_nulls.sql +++ b/tests/queries/1_stateful/00173_group_by_use_nulls.sql @@ -8,3 +8,28 @@ ORDER BY CounterID ASC LIMIT 10 SETTINGS group_by_use_nulls = 1; + +SELECT + CounterID AS k, + quantileBFloat16(0.5)(ResolutionWidth) +FROM test.hits +GROUP BY k +ORDER BY + count() DESC, + CounterID ASC +LIMIT 10 +SETTINGS group_by_use_nulls = 1 FORMAT Null; + +-- { echoOn } +set allow_experimental_analyzer = 1; + +SELECT + CounterID AS k, + quantileBFloat16(0.5)(ResolutionWidth) +FROM remote('127.0.0.{1,2}', test, hits) +GROUP BY k +ORDER BY + count() DESC, + CounterID ASC +LIMIT 10 +SETTINGS group_by_use_nulls = 1; diff --git a/utils/checksum-for-compressed-block/main.cpp b/utils/checksum-for-compressed-block/main.cpp index 27a2154340e..4f9923e7638 100644 --- a/utils/checksum-for-compressed-block/main.cpp +++ b/utils/checksum-for-compressed-block/main.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include /** A tool to easily prove if "Checksum doesn't match: corrupted data" diff --git a/utils/self-extracting-executable/decompressor.cpp b/utils/self-extracting-executable/decompressor.cpp index 37fbd043814..d41b9b1ebe1 100644 --- a/utils/self-extracting-executable/decompressor.cpp +++ b/utils/self-extracting-executable/decompressor.cpp @@ -168,6 +168,10 @@ int decompress(char * input, char * output, off_t start, off_t end, size_t max_n return 0; } +bool isSudo() +{ + return geteuid() == 0; +} /// Read data about files and decomrpess them. int decompressFiles(int input_fd, char * path, char * name, bool & have_compressed_analoge, bool & has_exec, char * decompressed_suffix, uint64_t * decompressed_umask) @@ -220,6 +224,8 @@ int decompressFiles(int input_fd, char * path, char * name, bool & have_compress return 1; } + bool is_sudo = isSudo(); + FileData file_info; /// Decompress files with appropriate file names for (size_t i = 0; i < le64toh(metadata.number_of_files); ++i) @@ -319,6 +325,9 @@ int decompressFiles(int input_fd, char * path, char * name, bool & have_compress perror("fsync"); if (0 != close(output_fd)) perror("close"); + + if (is_sudo) + chown(file_name, info_in.st_uid, info_in.st_gid); } if (0 != munmap(input, info_in.st_size)) @@ -414,6 +423,13 @@ int main(int/* argc*/, char* argv[]) else name = file_path; + struct stat input_info; + if (0 != stat(self, &input_info)) + { + perror("stat"); + return 1; + } + #if !defined(OS_DARWIN) && !defined(OS_FREEBSD) /// get inode of this executable uint64_t inode = getInode(self); @@ -441,13 +457,6 @@ int main(int/* argc*/, char* argv[]) return 1; } - struct stat input_info; - if (0 != stat(self, &input_info)) - { - perror("stat"); - return 1; - } - /// inconsistency in WSL1 Ubuntu - inode reported in /proc/self/maps is a 64bit to /// 32bit conversion of input_info.st_ino if (input_info.st_ino & 0xFFFFFFFF00000000 && !(inode & 0xFFFFFFFF00000000)) @@ -532,6 +541,9 @@ int main(int/* argc*/, char* argv[]) return 1; } + if (isSudo()) + chown(static_cast(self), input_info.st_uid, input_info.st_gid); + if (has_exec) { #if !defined(OS_DARWIN) && !defined(OS_FREEBSD) diff --git a/utils/wikistat-loader/main.cpp b/utils/wikistat-loader/main.cpp index 31ade014c74..493f1df05da 100644 --- a/utils/wikistat-loader/main.cpp +++ b/utils/wikistat-loader/main.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include #include