From 2eecaee08dc1f52786527674d3e9263df846cdb1 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 19 Feb 2021 16:24:57 +0300 Subject: [PATCH] Better tests for protobuf format. --- src/Columns/ColumnFixedString.cpp | 15 - src/Columns/ColumnFixedString.h | 2 - src/DataTypes/DataTypeFixedString.cpp | 19 +- src/DataTypes/DataTypeFixedString.h | 6 + src/Formats/ProtobufSerializer.cpp | 17 +- .../00825_protobuf_format_array_3dim.sh | 12 +- .../00825_protobuf_format_array_of_arrays.sh | 18 +- .../00825_protobuf_format_enum_mapping.sh | 10 +- .../00825_protobuf_format_input.insh | 5 - .../00825_protobuf_format_input.reference | 18 - .../00825_protobuf_format_input.sh | 77 --- .../00825_protobuf_format_input_single.insh | 12 - .../0_stateless/00825_protobuf_format_map.sh | 18 +- .../00825_protobuf_format_nested_optional.sh | 10 +- ...tobuf_format_no_length_delimiter.reference | 13 + ...825_protobuf_format_no_length_delimiter.sh | 52 ++ .../00825_protobuf_format_output.reference | Bin 3726 -> 0 bytes .../00825_protobuf_format_output.sh | 79 --- .../00825_protobuf_format_persons.reference | 569 ++++++++++++++++++ .../00825_protobuf_format_persons.sh | 118 ++++ .../00825_protobuf_format_squares.reference | 24 + .../00825_protobuf_format_squares.sh | 34 ++ .../00825_protobuf_format_table_default.sh | 10 +- .../00825_protobuf_format_array_3dim.proto | 0 ...0825_protobuf_format_array_of_arrays.proto | 0 .../00825_protobuf_format_enum_mapping.proto | 0 .../00825_protobuf_format_map.proto | 0 ...0825_protobuf_format_nested_optional.proto | 0 ..._protobuf_format_no_length_delimiter.proto | 6 + .../00825_protobuf_format_persons.proto} | 6 - ...825_protobuf_format_persons_syntax2.proto} | 0 .../00825_protobuf_format_squares.proto | 6 + .../00825_protobuf_format_syntax2.proto | 63 ++ .../00825_protobuf_format_table_default.proto | 0 .../protobuf_length_delimited_encoder.py | 8 +- 35 files changed, 964 insertions(+), 263 deletions(-) delete mode 100644 tests/queries/0_stateless/00825_protobuf_format_input.insh delete mode 100644 tests/queries/0_stateless/00825_protobuf_format_input.reference delete mode 100755 tests/queries/0_stateless/00825_protobuf_format_input.sh delete mode 100644 tests/queries/0_stateless/00825_protobuf_format_input_single.insh create mode 100644 tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.reference create mode 100755 tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.sh delete mode 100644 tests/queries/0_stateless/00825_protobuf_format_output.reference delete mode 100755 tests/queries/0_stateless/00825_protobuf_format_output.sh create mode 100644 tests/queries/0_stateless/00825_protobuf_format_persons.reference create mode 100755 tests/queries/0_stateless/00825_protobuf_format_persons.sh create mode 100644 tests/queries/0_stateless/00825_protobuf_format_squares.reference create mode 100755 tests/queries/0_stateless/00825_protobuf_format_squares.sh rename tests/queries/0_stateless/{ => format_schemas}/00825_protobuf_format_array_3dim.proto (100%) rename tests/queries/0_stateless/{ => format_schemas}/00825_protobuf_format_array_of_arrays.proto (100%) rename tests/queries/0_stateless/{ => format_schemas}/00825_protobuf_format_enum_mapping.proto (100%) rename tests/queries/0_stateless/{ => format_schemas}/00825_protobuf_format_map.proto (100%) rename tests/queries/0_stateless/{ => format_schemas}/00825_protobuf_format_nested_optional.proto (100%) create mode 100644 tests/queries/0_stateless/format_schemas/00825_protobuf_format_no_length_delimiter.proto rename tests/queries/0_stateless/{00825_protobuf_format.proto => format_schemas/00825_protobuf_format_persons.proto} (97%) rename tests/queries/0_stateless/{00825_protobuf_format_syntax2.proto => format_schemas/00825_protobuf_format_persons_syntax2.proto} (100%) create mode 100644 tests/queries/0_stateless/format_schemas/00825_protobuf_format_squares.proto create mode 100644 tests/queries/0_stateless/format_schemas/00825_protobuf_format_syntax2.proto rename tests/queries/0_stateless/{ => format_schemas}/00825_protobuf_format_table_default.proto (100%) diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index c4a7f923867..84bd0561f01 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -474,19 +474,4 @@ ColumnPtr ColumnFixedString::compress() const }); } - -void ColumnFixedString::alignStringLength(ColumnFixedString::Chars & data, size_t n, size_t old_size) -{ - size_t length = data.size() - old_size; - if (length < n) - { - data.resize_fill(old_size + n); - } - else if (length > n) - { - data.resize_assume_reserved(old_size); - throw Exception("Too large value for FixedString(" + std::to_string(n) + ")", ErrorCodes::TOO_LARGE_STRING_SIZE); - } -} - } diff --git a/src/Columns/ColumnFixedString.h b/src/Columns/ColumnFixedString.h index d9f6619b2d1..5e7b7f360f0 100644 --- a/src/Columns/ColumnFixedString.h +++ b/src/Columns/ColumnFixedString.h @@ -184,8 +184,6 @@ public: const Chars & getChars() const { return chars; } size_t getN() const { return n; } - - static void alignStringLength(ColumnFixedString::Chars & data, size_t n, size_t old_size); }; } diff --git a/src/DataTypes/DataTypeFixedString.cpp b/src/DataTypes/DataTypeFixedString.cpp index 21cfe855169..87e989d1dd2 100644 --- a/src/DataTypes/DataTypeFixedString.cpp +++ b/src/DataTypes/DataTypeFixedString.cpp @@ -25,6 +25,7 @@ namespace ErrorCodes extern const int CANNOT_READ_ALL_DATA; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int UNEXPECTED_AST_STRUCTURE; + extern const int TOO_LARGE_STRING_SIZE; } @@ -120,13 +121,21 @@ void DataTypeFixedString::serializeTextEscaped(const IColumn & column, size_t ro } -static inline void alignStringLength(const DataTypeFixedString & type, - ColumnFixedString::Chars & data, - size_t string_start) +void DataTypeFixedString::alignStringLength(PaddedPODArray & chars, size_t old_size) const { - ColumnFixedString::alignStringLength(data, type.getN(), string_start); + size_t length = chars.size() - old_size; + if (length < n) + { + chars.resize_fill(old_size + n); + } + else if (length > n) + { + chars.resize_assume_reserved(old_size); + throw Exception("Too large value for FixedString(" + std::to_string(n) + ")", ErrorCodes::TOO_LARGE_STRING_SIZE); + } } + template static inline void read(const DataTypeFixedString & self, IColumn & column, Reader && reader) { @@ -135,7 +144,7 @@ static inline void read(const DataTypeFixedString & self, IColumn & column, Read try { reader(data); - alignStringLength(self, data, prev_size); + self.alignStringLength(data, prev_size); } catch (...) { diff --git a/src/DataTypes/DataTypeFixedString.h b/src/DataTypes/DataTypeFixedString.h index af82e4b5d11..5c80a0e346a 100644 --- a/src/DataTypes/DataTypeFixedString.h +++ b/src/DataTypes/DataTypeFixedString.h @@ -1,6 +1,7 @@ #pragma once #include +#include #define MAX_FIXEDSTRING_SIZE 0xFFFFFF @@ -82,6 +83,11 @@ public: bool isCategorial() const override { return true; } bool canBeInsideNullable() const override { return true; } bool canBeInsideLowCardinality() const override { return true; } + + /// Makes sure that the length of a newly inserted string to `chars` is equal to getN(). + /// If the length is less than getN() the function will add zero characters up to getN(). + /// If the length is greater than getN() the function will throw an exception. + void alignStringLength(PaddedPODArray & chars, size_t old_size) const; }; } diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp index 82149460773..66efa76fe97 100644 --- a/src/Formats/ProtobufSerializer.cpp +++ b/src/Formats/ProtobufSerializer.cpp @@ -525,16 +525,16 @@ namespace { public: using ColumnType = std::conditional_t; - using StringDataType = std::conditional_t; ProtobufSerializerString( - const StringDataType & string_data_type_, + const std::shared_ptr & fixed_string_data_type_, const google::protobuf::FieldDescriptor & field_descriptor_, const ProtobufReaderOrWriter & reader_or_writer_) : ProtobufSerializerSingleValue(field_descriptor_, reader_or_writer_) + , fixed_string_data_type(fixed_string_data_type_) + , n(fixed_string_data_type->getN()) { static_assert(is_fixed_string, "This constructor for FixedString only"); - n = string_data_type_.getN(); setFunctions(); prepareEnumMapping(); } @@ -583,11 +583,11 @@ namespace { if (row_num < old_size) { - ColumnFixedString::alignStringLength(text_buffer, n, 0); + fixed_string_data_type->alignStringLength(text_buffer, 0); memcpy(data.data() + row_num * n, text_buffer.data(), n); } else - ColumnFixedString::alignStringLength(data, n, old_data_size); + fixed_string_data_type->alignStringLength(data, old_data_size); } else { @@ -817,7 +817,7 @@ namespace auto str = default_function(); arr.insert(str.data(), str.data() + str.size()); if constexpr (is_fixed_string) - ColumnFixedString::alignStringLength(arr, n, 0); + fixed_string_data_type->alignStringLength(arr, 0); default_string = std::move(arr); } return *default_string; @@ -865,7 +865,8 @@ namespace str.insert(name.data(), name.data() + name.length()); } - size_t n = 0; + const std::shared_ptr fixed_string_data_type; + const size_t n = 0; std::function write_function; std::function &)> read_function; std::function default_function; @@ -2765,7 +2766,7 @@ namespace case TypeIndex::DateTime: return std::make_unique(field_descriptor, reader_or_writer); case TypeIndex::DateTime64: return std::make_unique(assert_cast(*data_type), field_descriptor, reader_or_writer); case TypeIndex::String: return std::make_unique>(field_descriptor, reader_or_writer); - case TypeIndex::FixedString: return std::make_unique>(assert_cast(*data_type), field_descriptor, reader_or_writer); + case TypeIndex::FixedString: return std::make_unique>(typeid_cast>(data_type), field_descriptor, reader_or_writer); case TypeIndex::Enum8: return std::make_unique>(typeid_cast>(data_type), field_descriptor, reader_or_writer); case TypeIndex::Enum16: return std::make_unique>(typeid_cast>(data_type), field_descriptor, reader_or_writer); case TypeIndex::Decimal32: return std::make_unique>(assert_cast &>(*data_type), field_descriptor, reader_or_writer); diff --git a/tests/queries/0_stateless/00825_protobuf_format_array_3dim.sh b/tests/queries/0_stateless/00825_protobuf_format_array_3dim.sh index 903217ca939..762f8a937e4 100755 --- a/tests/queries/0_stateless/00825_protobuf_format_array_3dim.sh +++ b/tests/queries/0_stateless/00825_protobuf_format_array_3dim.sh @@ -1,18 +1,19 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SCHEMADIR=$CURDIR/format_schemas # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -eo pipefail # Run the client. -$CLICKHOUSE_CLIENT --multiquery <<'EOF' +$CLICKHOUSE_CLIENT --multiquery < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM array_3dim_protobuf_00825 FORMAT Protobuf SETTINGS format_schema = '$SCHEMADIR/00825_protobuf_format_array_3dim:ABC'" > "$BINARY_FILE_PATH" # Check the output in the protobuf format echo -$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$CURDIR/00825_protobuf_format_array_3dim:ABC" --input "$BINARY_FILE_PATH" +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/00825_protobuf_format_array_3dim:ABC" --input "$BINARY_FILE_PATH" # Check the input in the protobuf format (now the table contains the same data twice). echo -$CLICKHOUSE_CLIENT --query "INSERT INTO array_3dim_protobuf_00825 FORMAT Protobuf SETTINGS format_schema='$CURDIR/00825_protobuf_format_array_3dim:ABC'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "INSERT INTO array_3dim_protobuf_00825 FORMAT Protobuf SETTINGS format_schema='$SCHEMADIR/00825_protobuf_format_array_3dim:ABC'" < "$BINARY_FILE_PATH" $CLICKHOUSE_CLIENT --query "SELECT * FROM array_3dim_protobuf_00825" rm "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "DROP TABLE array_3dim_protobuf_00825" diff --git a/tests/queries/0_stateless/00825_protobuf_format_array_of_arrays.sh b/tests/queries/0_stateless/00825_protobuf_format_array_of_arrays.sh index 0b386723091..243446f9438 100755 --- a/tests/queries/0_stateless/00825_protobuf_format_array_of_arrays.sh +++ b/tests/queries/0_stateless/00825_protobuf_format_array_of_arrays.sh @@ -3,18 +3,21 @@ # https://github.com/ClickHouse/ClickHouse/issues/9069 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SCHEMADIR=$CURDIR/format_schemas # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -eo pipefail # Run the client. -$CLICKHOUSE_CLIENT --multiquery <<'EOF' +$CLICKHOUSE_CLIENT --multiquery < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM array_of_arrays_protobuf_00825 FORMAT Protobuf SETTINGS format_schema = '$SCHEMADIR/00825_protobuf_format_array_of_arrays:AA'" > "$BINARY_FILE_PATH" # Check the output in the protobuf format echo -$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$CURDIR/00825_protobuf_format_array_of_arrays:AA" --input "$BINARY_FILE_PATH" +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/00825_protobuf_format_array_of_arrays:AA" --input "$BINARY_FILE_PATH" # Check the input in the protobuf format (now the table contains the same data twice). echo -$CLICKHOUSE_CLIENT --query "INSERT INTO array_of_arrays_protobuf_00825 FORMAT Protobuf SETTINGS format_schema='$CURDIR/00825_protobuf_format_array_of_arrays:AA'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "INSERT INTO array_of_arrays_protobuf_00825 FORMAT Protobuf SETTINGS format_schema='$SCHEMADIR/00825_protobuf_format_array_of_arrays:AA'" < "$BINARY_FILE_PATH" $CLICKHOUSE_CLIENT --query "SELECT * FROM array_of_arrays_protobuf_00825" rm "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "DROP TABLE array_of_arrays_protobuf_00825" diff --git a/tests/queries/0_stateless/00825_protobuf_format_enum_mapping.sh b/tests/queries/0_stateless/00825_protobuf_format_enum_mapping.sh index cbb387a62a5..300f82e5ca2 100755 --- a/tests/queries/0_stateless/00825_protobuf_format_enum_mapping.sh +++ b/tests/queries/0_stateless/00825_protobuf_format_enum_mapping.sh @@ -3,13 +3,14 @@ # https://github.com/ClickHouse/ClickHouse/issues/7438 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SCHEMADIR=$CURDIR/format_schemas # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -eo pipefail # Run the client. -$CLICKHOUSE_CLIENT --multiquery <<'EOF' +$CLICKHOUSE_CLIENT --multiquery < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM enum_mapping_protobuf_00825 FORMAT Protobuf SETTINGS format_schema = '$SCHEMADIR/00825_protobuf_format_enum_mapping:Message'" > "$BINARY_FILE_PATH" # Check the output in the protobuf format echo -$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$CURDIR/00825_protobuf_format_enum_mapping:Message" --input "$BINARY_FILE_PATH" +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/00825_protobuf_format_enum_mapping:Message" --input "$BINARY_FILE_PATH" # Check the input in the protobuf format (now the table contains the same data twice). echo -$CLICKHOUSE_CLIENT --query "INSERT INTO enum_mapping_protobuf_00825 FORMAT Protobuf SETTINGS format_schema='$CURDIR/00825_protobuf_format_enum_mapping:Message'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "INSERT INTO enum_mapping_protobuf_00825 FORMAT Protobuf SETTINGS format_schema='$SCHEMADIR/00825_protobuf_format_enum_mapping:Message'" < "$BINARY_FILE_PATH" $CLICKHOUSE_CLIENT --query "SELECT * FROM enum_mapping_protobuf_00825" rm "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "DROP TABLE enum_mapping_protobuf_00825" diff --git a/tests/queries/0_stateless/00825_protobuf_format_input.insh b/tests/queries/0_stateless/00825_protobuf_format_input.insh deleted file mode 100644 index 39a2f17c98f..00000000000 --- a/tests/queries/0_stateless/00825_protobuf_format_input.insh +++ /dev/null @@ -1,5 +0,0 @@ -echo -ne '\xf3\x01\x0a\x24\x61\x37\x35\x32\x32\x31\x35\x38\x2d\x33\x64\x34\x31\x2d\x34\x62\x37\x37\x2d\x61\x64\x36\x39\x2d\x36\x63\x35\x39\x38\x65\x65\x35\x35\x63\x34\x39\x12\x04\x49\x76\x61\x6e\x1a\x06\x50\x65\x74\x72\x6f\x76\x20\x01\x28\xaf\x1f\x32\x03\x70\x6e\x67\x3a\x0c\x2b\x37\x34\x39\x35\x31\x32\x33\x34\x35\x36\x37\x40\x01\x4d\xfc\xd0\x30\x5c\x50\x26\x58\x09\x62\x09\x59\x65\x73\x74\x65\x72\x64\x61\x79\x62\x07\x46\x6c\x6f\x77\x65\x72\x73\x6a\x04\xff\x01\x00\x00\x72\x06\x4d\x6f\x73\x63\x6f\x77\x7a\x08\x4b\x03\x5f\x42\x72\x7d\x16\x42\x81\x01\x1f\x85\xeb\x51\xb8\x1e\x09\x40\x89\x01\x33\x33\x33\x33\x33\xc3\x6a\x40\x95\x01\xcd\xcc\xcc\x3d\x9d\x01\x9a\x99\xb9\x40\xa0\x01\x80\xc4\xd7\x8d\x7f\xaa\x01\x0c\x0a\x05\x6d\x65\x74\x65\x72\x15\x00\x00\x80\x3f\xaa\x01\x11\x0a\x0a\x63\x65\x6e\x74\x69\x6d\x65\x74\x65\x72\x15\x0a\xd7\x23\x3c\xaa\x01\x10\x0a\x09\x6b\x69\x6c\x6f\x6d\x65\x74\x65\x72\x15\x00\x00\x7a\x44\xb2\x01\x10\x0a\x0e\xa2\x06\x0b\x0a\x09\x08\xf4\x03\x12\x04\xf5\x03\xf6\x03\x7e\x0a\x24\x63\x36\x39\x34\x61\x64\x38\x61\x2d\x66\x37\x31\x34\x2d\x34\x65\x61\x33\x2d\x39\x30\x37\x64\x2d\x66\x64\x35\x34\x66\x62\x32\x35\x64\x39\x62\x35\x12\x07\x4e\x61\x74\x61\x6c\x69\x61\x1a\x08\x53\x6f\x6b\x6f\x6c\x6f\x76\x61\x28\xa6\x3f\x32\x03\x6a\x70\x67\x50\x1a\x58\x0b\x6a\x04\x64\xc8\x01\x32\x72\x08\x50\x6c\x79\x6d\x6f\x75\x74\x68\x7a\x08\x6a\x9d\x49\x42\x46\x8c\x84\xc0\x81\x01\x6e\x86\x1b\xf0\xf9\x21\x09\x40\x95\x01\x42\x60\xe5\x3b\x9d\x01\xcd\xcc\xac\x40\xa0\x01\xff\xff\xa9\xce\x93\x8c\x09\xc0\x01\x0a\x24\x61\x37\x64\x61\x31\x61\x61\x36\x2d\x66\x34\x32\x35\x2d\x34\x37\x38\x39\x2d\x38\x39\x34\x37\x2d\x62\x30\x33\x34\x37\x38\x36\x65\x64\x33\x37\x34\x12\x06\x56\x61\x73\x69\x6c\x79\x1a\x07\x53\x69\x64\x6f\x72\x6f\x76\x20\x01\x28\xfb\x48\x32\x03\x62\x6d\x70\x3a\x0d\x2b\x34\x34\x32\x30\x31\x32\x33\x34\x35\x36\x37\x38\x40\x01\x4d\x50\xe0\x27\x5c\x50\x17\x58\x04\x62\x05\x53\x75\x6e\x6e\x79\x6a\x05\xfa\x01\xf4\x01\x0a\x72\x08\x4d\x75\x72\x6d\x61\x6e\x73\x6b\x7a\x08\xfd\xf0\x89\x42\xc8\x4c\x04\x42\x81\x01\x11\x2d\x44\x54\xfb\x21\x09\x40\x89\x01\x00\x00\x00\xe8\x76\x48\x37\x42\x95\x01\x00\x00\x48\x44\x9d\x01\xcd\xcc\x4c\xc0\xa0\x01\x80\xd4\x9f\x93\x01\xaa\x01\x0c\x0a\x05\x70\x6f\x75\x6e\x64\x15\x00\x00\x80\x41\xb2\x01\x0a\x0a\x08\xa2\x06\x05\x0a\x03\x08\xf7\x03' | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:Person'" -echo -ne '\xb3\x01\x12\x05\x46\x72\x69\x64\x61\x28\x99\xe1\xf3\xd1\x0b\x52\x08\x45\x72\x6d\x61\x6b\x6f\x76\x61\x72\x0c\x00\x00\xdc\x42\x00\x00\x52\x43\x00\x00\x94\x42\x79\x48\xce\x3d\x51\x00\x00\x00\x00\xc8\x02\x14\xc2\x05\x08\x00\x00\x80\x44\x00\x00\x80\x49\x9a\x06\x02\x4b\x42\x9a\x06\x02\x4d\x42\xa1\x06\x00\x00\x00\x00\x00\x00\xe0\x3f\xa8\x06\x2a\xa8\x06\xa8\xff\xff\xff\xff\xff\xff\xff\xff\x01\xb0\x06\x01\xbd\x06\x25\x06\x49\x40\xfa\x06\x02\x34\x30\x90\x08\xe2\x08\xe1\x08\x89\xe6\x6e\xdd\x01\x00\x00\x00\xb0\x09\xc3\x19\xd0\x0c\xb7\x02\xe2\x12\x24\x32\x30\x66\x63\x64\x39\x35\x61\x2d\x33\x33\x32\x64\x2d\x34\x31\x64\x62\x2d\x61\x39\x65\x63\x2d\x31\x36\x31\x66\x36\x34\x34\x64\x30\x35\x39\x63\xa0\x38\xbc\x05\xaa\x38\x02\xbd\x05\xb4\x01\x08\x01\x12\x06\x49\x73\x6f\x6c\x64\x65\x52\x07\x4c\x61\x76\x72\x6f\x76\x61\x72\x0c\x00\x00\x7f\x43\x00\x00\x00\x00\x00\x00\x7f\x43\xaa\x01\x03\x61\x62\x63\xc8\x02\x32\xc2\x05\x08\x00\x00\x00\x41\x00\x00\x80\x3f\x9a\x06\x04\x42\x79\x74\x65\x9a\x06\x03\x42\x69\x74\xa1\x06\x00\x00\x00\x00\x00\x00\x12\x40\xa8\x06\x1a\xa8\x06\xb0\xff\xff\xff\xff\xff\xff\xff\xff\x01\xb0\x06\x01\xbd\x06\xf9\x0f\x49\x40\xc2\x06\x01\x2c\xfa\x06\x02\x33\x32\x90\x08\x78\xe1\x08\x39\x4e\x2b\xfe\xe4\xf5\xff\xff\xb0\x09\xe8\x30\xd8\x12\x01\xe2\x12\x24\x37\x63\x66\x61\x36\x38\x35\x36\x2d\x61\x35\x34\x61\x2d\x34\x37\x38\x36\x2d\x62\x38\x65\x35\x2d\x37\x34\x35\x31\x35\x39\x64\x35\x32\x32\x37\x38\xa0\x38\xbe\x05\xc2\x3e\x05\x15\x00\x00\xb6\x42' | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:AltPerson'" -echo -ne '\xa5\x02\x0a\x24\x61\x61\x30\x65\x35\x61\x30\x36\x2d\x63\x61\x62\x32\x2d\x34\x30\x33\x34\x2d\x61\x36\x61\x32\x2d\x34\x38\x65\x38\x32\x62\x39\x31\x36\x36\x34\x65\x12\x06\x4c\x65\x6f\x6e\x69\x64\x1a\x08\x4b\x69\x72\x69\x6c\x6c\x6f\x76\x22\x04\x6d\x61\x6c\x65\x2a\x0a\x31\x39\x38\x33\x2d\x30\x36\x2d\x32\x34\x3a\x0c\x2b\x37\x34\x39\x35\x30\x32\x37\x35\x38\x36\x34\x42\x01\x31\x4a\x13\x32\x30\x31\x39\x2d\x30\x32\x2d\x30\x34\x20\x30\x39\x3a\x34\x35\x3a\x30\x30\x52\x02\x33\x35\x5a\x06\x63\x61\x6e\x63\x65\x72\x62\x07\x37\x20\x72\x69\x6e\x67\x73\x62\x08\x45\x61\x73\x74\x73\x69\x64\x65\x62\x0b\x4c\x61\x73\x74\x20\x48\x75\x72\x72\x61\x68\x6a\x01\x30\x6a\x01\x30\x6a\x03\x32\x35\x35\x72\x09\x53\x61\x6e\x20\x44\x69\x65\x67\x6f\x7a\x09\x33\x32\x2e\x38\x32\x33\x39\x34\x33\x7a\x0b\x2d\x31\x31\x37\x2e\x30\x38\x31\x33\x32\x37\x82\x01\x09\x33\x2e\x31\x34\x31\x35\x39\x32\x37\x8a\x01\x08\x31\x35\x30\x30\x30\x30\x30\x30\x92\x01\x06\x31\x38\x36\x2e\x37\x35\x9a\x01\x04\x2d\x32\x2e\x31\xa2\x01\x0b\x32\x30\x36\x35\x39\x38\x32\x39\x33\x33\x31\xaa\x01\x18\x0a\x06\x6d\x69\x6e\x75\x74\x65\x0a\x04\x68\x6f\x75\x72\x12\x02\x36\x30\x12\x04\x33\x36\x30\x30\xb2\x01\x08\x0a\x06\x12\x04\x31\x38\x30\x30' | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:StrPerson'" -echo -ne '\xdd\x01\x0a\x24\x33\x66\x61\x65\x65\x30\x36\x34\x2d\x63\x34\x66\x37\x2d\x34\x64\x33\x34\x2d\x62\x36\x66\x33\x2d\x38\x64\x38\x31\x63\x32\x62\x36\x61\x31\x35\x64\x12\x04\x4e\x69\x63\x6b\x1a\x0a\x4b\x6f\x6c\x65\x73\x6e\x69\x6b\x6f\x76\x20\x01\x28\xda\x52\x32\x03\x62\x6d\x70\x3a\x0c\x34\x31\x32\x2d\x36\x38\x37\x2d\x35\x30\x30\x37\x40\x01\x4d\x2f\x27\xf2\x5b\x50\x14\x58\x09\x62\x06\x48\x61\x76\x61\x6e\x61\x68\x80\x01\x68\x00\x68\x80\x01\x72\x0a\x50\x69\x74\x74\x73\x62\x75\x72\x67\x68\x7a\x08\x9b\x11\x22\x42\x1f\xe6\x9f\xc2\x81\x01\x28\x2d\x44\x54\xfb\x21\x09\x40\x89\x01\x00\x00\x00\xe8\x76\x48\x27\x42\x95\x01\x00\x00\x43\x44\x9d\x01\x66\x66\x92\x41\xa0\x01\xce\xdf\xb8\xba\x01\xab\x01\x0d\xcd\xcc\xe2\x41\x0d\xcd\xcc\x4c\x3e\x0d\x00\x00\x80\x3f\x12\x05\x6f\x75\x6e\x63\x65\x12\x05\x63\x61\x72\x61\x74\x12\x04\x67\x72\x61\x6d\xac\x01\xb3\x01\x0b\xa2\x06\x05\x0b\x08\x96\x4a\x0c\x0c\xb4\x01' | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format_syntax2:Syntax2Person'" -echo -ne '\x04\x08\x02\x10\x04\x00\x04\x08\x03\x10\x09' | $CLICKHOUSE_CLIENT --query="INSERT INTO in_squares_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:NumberAndSquare'" diff --git a/tests/queries/0_stateless/00825_protobuf_format_input.reference b/tests/queries/0_stateless/00825_protobuf_format_input.reference deleted file mode 100644 index 75fef3f8ac3..00000000000 --- a/tests/queries/0_stateless/00825_protobuf_format_input.reference +++ /dev/null @@ -1,18 +0,0 @@ -a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 bmp +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.970680,33.074982] 3.14159265358979 100000000000.00 800 -3.2 154400000 ['pound'] [16] 503 [] -c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 jpg \N 0 \N 26 pisces [] [100,200,50] Plymouth [50.403724,-4.142123] 3.14159 \N 0.007 5.4 -20000000000000 [] [] \N [] -aa0e5a06-cab2-4034-a6a2-48e82b91664e Leonid Kirillov male 1983-06-24 \N +74950275864\0 1 2019-02-04 09:45:00 35 cancer ['7 rings','Eastside','Last Hurrah'] [0,0,255] San Diego [32.823943,-117.081327] 3.1415927 15000000.00 186.75 -2.1 20659829331 ['minute','hour'] [60,3600] \N [1800] -20fcd95a-332d-41db-a9ec-161f644d059c Frida Ermakova female 1978-12-12 \N 3124555929\0\0\0 0 2013-03-11 16:30:00 40 sagittarius [] [110,210,74] [42.000000,-88.000000] 3.1410000324249268 311.00 0.5 10.0 8010000009 ['KB','MB'] [1024,1048576] 700 [701] -a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753216,37.622504] 3.14 214.10 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] -3faee064-c4f7-4d34-b6f3-8d81c2b6a15d Nick Kolesnikov male 1998-12-26 bmp 412-687-5007\0 1 2018-11-19 05:59:59 20 capricorn ['Havana'] [128,0,128] Pittsburgh [40.517192,-79.949456] 3.1415926535898 50000000000.00 780 18.3 195500007 ['ounce','carat','gram'] [28.35,0.2,1] 9494 [] -7cfa6856-a54a-4786-b8e5-745159d52278 Isolde Lavrova female 1987-02-09 \N \N 1 \N 32 aquarius [] [255,0,255] [26.000000,-80.000000] 3.1415998935699463 \N 4.5 25.0 -11111111111111 ['Byte','Bit'] [8,1] 702 [] -0 0 -2 4 -3 9 -a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 bmp +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.970680,33.074982] 3.14159265358979 100000000000.00 800 -3.2 154400000 ['pound'] [16] 503 [] -c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 jpg \N 0 \N 26 pisces [] [100,200,50] Plymouth [50.403724,-4.142123] 3.14159 \N 0.007 5.4 -20000000000000 [] [] \N [] -a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753216,37.622504] 3.14 214.10 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] -3faee064-c4f7-4d34-b6f3-8d81c2b6a15d Nick Kolesnikov male 1998-12-26 bmp 412-687-5007\0 1 2018-11-19 05:59:59 20 capricorn ['Havana'] [128,0,128] Pittsburgh [40.517192,-79.949456] 3.1415926535898 50000000000.00 780 18.3 195500007 ['ounce','carat','gram'] [28.35,0.2,1] 9494 [] -2 4 -3 9 -ok -ok diff --git a/tests/queries/0_stateless/00825_protobuf_format_input.sh b/tests/queries/0_stateless/00825_protobuf_format_input.sh deleted file mode 100755 index 5a85a852cb1..00000000000 --- a/tests/queries/0_stateless/00825_protobuf_format_input.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh - -set -eo pipefail - -# Run the client. -$CLICKHOUSE_CLIENT --multiquery <<'EOF' -DROP TABLE IF EXISTS in_persons_00825; -DROP TABLE IF EXISTS in_squares_00825; - -CREATE TABLE in_persons_00825 (uuid UUID, - name String, - surname String, - gender Enum8('male'=1, 'female'=0), - birthDate Date, - photo Nullable(String), - phoneNumber Nullable(FixedString(13)), - isOnline UInt8, - visitTime Nullable(DateTime), - age UInt8, - zodiacSign Enum16('aries'=321, 'taurus'=420, 'gemini'=521, 'cancer'=621, 'leo'=723, 'virgo'=823, - 'libra'=923, 'scorpius'=1023, 'sagittarius'=1122, 'capricorn'=1222, 'aquarius'=120, - 'pisces'=219), - songs Array(String), - color Array(UInt8), - hometown LowCardinality(String), - location Array(Decimal32(6)), - pi Nullable(Float64), - lotteryWin Nullable(Decimal64(2)), - someRatio Float32, - temperature Decimal32(1), - randomBigNumber Int64, - measureUnits Nested (unit String, coef Float32), - nestiness_a_b_c_d Nullable(UInt32), - `nestiness_a_B.c_E` Array(UInt32) - ) ENGINE = MergeTree ORDER BY tuple(); - -CREATE TABLE in_squares_00825 (number UInt32, square UInt32) ENGINE = MergeTree ORDER BY tuple(); -EOF - -# To generate the file 00825_protobuf_format_input.insh use the following commands: -# ninja ProtobufDelimitedMessagesSerializer -# build/utils/test-data-generator/ProtobufDelimitedMessagesSerializer -# shellcheck source=./00825_protobuf_format_input.insh -source "$CURDIR"/00825_protobuf_format_input.insh - -$CLICKHOUSE_CLIENT --query "SELECT * FROM in_persons_00825 ORDER BY uuid;" -$CLICKHOUSE_CLIENT --query "SELECT * FROM in_squares_00825 ORDER BY number;" - -$CLICKHOUSE_CLIENT --query "TRUNCATE TABLE in_persons_00825;" -$CLICKHOUSE_CLIENT --query "TRUNCATE TABLE in_squares_00825;" - -# shellcheck source=./00825_protobuf_format_input_single.insh -source "$CURDIR"/00825_protobuf_format_input_single.insh - -$CLICKHOUSE_CLIENT --query "SELECT * FROM in_persons_00825 ORDER BY uuid;" -$CLICKHOUSE_CLIENT --query "SELECT * FROM in_squares_00825 ORDER BY number;" - -# Try to input malformed data. -set +eo pipefail -echo -ne '\xe0\x80\x3f\x0b' \ - | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:Person'" 2>&1 \ - | grep -qF "Protobuf messages are corrupted" && echo "ok" || echo "fail" -set -eo pipefail - -# Try to input malformed data for ProtobufSingle -set +eo pipefail -echo -ne '\xff\xff\x3f\x0b' \ - | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT ProtobufSingle SETTINGS format_schema = '$CURDIR/00825_protobuf_format:Person'" 2>&1 \ - | grep -qF "Protobuf messages are corrupted" && echo "ok" || echo "fail" -set -eo pipefail - -$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS in_persons_00825;" -$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS in_squares_00825;" diff --git a/tests/queries/0_stateless/00825_protobuf_format_input_single.insh b/tests/queries/0_stateless/00825_protobuf_format_input_single.insh deleted file mode 100644 index 6c4dfec05aa..00000000000 --- a/tests/queries/0_stateless/00825_protobuf_format_input_single.insh +++ /dev/null @@ -1,12 +0,0 @@ -echo -ne '\x0a\x24\x61\x37\x35\x32\x32\x31\x35\x38\x2d\x33\x64\x34\x31\x2d\x34\x62\x37\x37\x2d\x61\x64\x36\x39\x2d\x36\x63\x35\x39\x38\x65\x65\x35\x35\x63\x34\x39\x12\x04\x49\x76\x61\x6e\x1a\x06\x50\x65\x74\x72\x6f\x76\x20\x01\x28\xaf\x1f\x32\x03\x70\x6e\x67\x3a\x0c\x2b\x37\x34\x39\x35\x31\x32\x33\x34\x35\x36\x37\x40\x01\x4d\xfc\xd0\x30\x5c\x50\x26\x58\x09\x62\x09\x59\x65\x73\x74\x65\x72\x64\x61\x79\x62\x07\x46\x6c\x6f\x77\x65\x72\x73\x6a\x04\xff\x01\x00\x00\x72\x06\x4d\x6f\x73\x63\x6f\x77\x7a\x08\x4b\x03\x5f\x42\x72\x7d\x16\x42\x81\x01\x1f\x85\xeb\x51\xb8\x1e\x09\x40\x89\x01\x33\x33\x33\x33\x33\xc3\x6a\x40\x95\x01\xcd\xcc\xcc\x3d\x9d\x01\x9a\x99\xb9\x40\xa0\x01\x80\xc4\xd7\x8d\x7f\xaa\x01\x0c\x0a\x05\x6d\x65\x74\x65\x72\x15\x00\x00\x80\x3f\xaa\x01\x11\x0a\x0a\x63\x65\x6e\x74\x69\x6d\x65\x74\x65\x72\x15\x0a\xd7\x23\x3c\xaa\x01\x10\x0a\x09\x6b\x69\x6c\x6f\x6d\x65\x74\x65\x72\x15\x00\x00\x7a\x44\xb2\x01\x10\x0a\x0e\xa2\x06\x0b\x0a\x09\x08\xf4\x03\x12\x04\xf5\x03\xf6\x03' | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT ProtobufSingle SETTINGS format_schema = '$CURDIR/00825_protobuf_format:Person'" -echo -ne '\x0a\x24\x63\x36\x39\x34\x61\x64\x38\x61\x2d\x66\x37\x31\x34\x2d\x34\x65\x61\x33\x2d\x39\x30\x37\x64\x2d\x66\x64\x35\x34\x66\x62\x32\x35\x64\x39\x62\x35\x12\x07\x4e\x61\x74\x61\x6c\x69\x61\x1a\x08\x53\x6f\x6b\x6f\x6c\x6f\x76\x61\x28\xa6\x3f\x32\x03\x6a\x70\x67\x50\x1a\x58\x0b\x6a\x04\x64\xc8\x01\x32\x72\x08\x50\x6c\x79\x6d\x6f\x75\x74\x68\x7a\x08\x6a\x9d\x49\x42\x46\x8c\x84\xc0\x81\x01\x6e\x86\x1b\xf0\xf9\x21\x09\x40\x95\x01\x42\x60\xe5\x3b\x9d\x01\xcd\xcc\xac\x40\xa0\x01\xff\xff\xa9\xce\x93\x8c\x09' | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT ProtobufSingle SETTINGS format_schema = '$CURDIR/00825_protobuf_format:Person'" -echo -ne '\x0a\x24\x61\x37\x64\x61\x31\x61\x61\x36\x2d\x66\x34\x32\x35\x2d\x34\x37\x38\x39\x2d\x38\x39\x34\x37\x2d\x62\x30\x33\x34\x37\x38\x36\x65\x64\x33\x37\x34\x12\x06\x56\x61\x73\x69\x6c\x79\x1a\x07\x53\x69\x64\x6f\x72\x6f\x76\x20\x01\x28\xfb\x48\x32\x03\x62\x6d\x70\x3a\x0d\x2b\x34\x34\x32\x30\x31\x32\x33\x34\x35\x36\x37\x38\x40\x01\x4d\x50\xe0\x27\x5c\x50\x17\x58\x04\x62\x05\x53\x75\x6e\x6e\x79\x6a\x05\xfa\x01\xf4\x01\x0a\x72\x08\x4d\x75\x72\x6d\x61\x6e\x73\x6b\x7a\x08\xfd\xf0\x89\x42\xc8\x4c\x04\x42\x81\x01\x11\x2d\x44\x54\xfb\x21\x09\x40\x89\x01\x00\x00\x00\xe8\x76\x48\x37\x42\x95\x01\x00\x00\x48\x44\x9d\x01\xcd\xcc\x4c\xc0\xa0\x01\x80\xd4\x9f\x93\x01\xaa\x01\x0c\x0a\x05\x70\x6f\x75\x6e\x64\x15\x00\x00\x80\x41\xb2\x01\x0a\x0a\x08\xa2\x06\x05\x0a\x03\x08\xf7\x03' | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT ProtobufSingle SETTINGS format_schema = '$CURDIR/00825_protobuf_format:Person'" -echo -ne '\x0a\x24\x33\x66\x61\x65\x65\x30\x36\x34\x2d\x63\x34\x66\x37\x2d\x34\x64\x33\x34\x2d\x62\x36\x66\x33\x2d\x38\x64\x38\x31\x63\x32\x62\x36\x61\x31\x35\x64\x12\x04\x4e\x69\x63\x6b\x1a\x0a\x4b\x6f\x6c\x65\x73\x6e\x69\x6b\x6f\x76\x20\x01\x28\xda\x52\x32\x03\x62\x6d\x70\x3a\x0c\x34\x31\x32\x2d\x36\x38\x37\x2d\x35\x30\x30\x37\x40\x01\x4d\x2f\x27\xf2\x5b\x50\x14\x58\x09\x62\x06\x48\x61\x76\x61\x6e\x61\x68\x80\x01\x68\x00\x68\x80\x01\x72\x0a\x50\x69\x74\x74\x73\x62\x75\x72\x67\x68\x7a\x08\x9b\x11\x22\x42\x1f\xe6\x9f\xc2\x81\x01\x28\x2d\x44\x54\xfb\x21\x09\x40\x89\x01\x00\x00\x00\xe8\x76\x48\x27\x42\x95\x01\x00\x00\x43\x44\x9d\x01\x66\x66\x92\x41\xa0\x01\xce\xdf\xb8\xba\x01\xab\x01\x0d\xcd\xcc\xe2\x41\x0d\xcd\xcc\x4c\x3e\x0d\x00\x00\x80\x3f\x12\x05\x6f\x75\x6e\x63\x65\x12\x05\x63\x61\x72\x61\x74\x12\x04\x67\x72\x61\x6d\xac\x01\xb3\x01\x0b\xa2\x06\x05\x0b\x08\x96\x4a\x0c\x0c\xb4\x01' | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT ProtobufSingle SETTINGS format_schema = '$CURDIR/00825_protobuf_format_syntax2:Syntax2Person'" - -echo -ne '\x08\x02\x10\x04' | $CLICKHOUSE_CLIENT --query="INSERT INTO in_squares_00825 FORMAT ProtobufSingle SETTINGS format_schema = '$CURDIR/00825_protobuf_format:NumberAndSquare'" -echo -ne '\x08\x03\x10\x09' | $CLICKHOUSE_CLIENT --query="INSERT INTO in_squares_00825 FORMAT ProtobufSingle SETTINGS format_schema = '$CURDIR/00825_protobuf_format:NumberAndSquare'" - -### Actually empty Protobuf message is a valid message (with all values default). -### It will work in Kafka but clickhouse-client forbids that: -### Code: 108. DB::Exception: No data to insert -## echo -ne '' | $CLICKHOUSE_CLIENT --query="INSERT INTO in_squares_00825 FORMAT ProtobufSingle SETTINGS format_schema = '$CURDIR/00825_protobuf_format:NumberAndSquare'" \ No newline at end of file diff --git a/tests/queries/0_stateless/00825_protobuf_format_map.sh b/tests/queries/0_stateless/00825_protobuf_format_map.sh index 5df25c41750..da7cacf76b8 100755 --- a/tests/queries/0_stateless/00825_protobuf_format_map.sh +++ b/tests/queries/0_stateless/00825_protobuf_format_map.sh @@ -3,29 +3,30 @@ # https://github.com/ClickHouse/ClickHouse/issues/6497 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SCHEMADIR=$CURDIR/format_schemas # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -eo pipefail # Run the client. -$CLICKHOUSE_CLIENT --multiquery <<'EOF' +$CLICKHOUSE_CLIENT --multiquery < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM map_protobuf_00825 FORMAT Protobuf SETTINGS format_schema = '$SCHEMADIR/00825_protobuf_format_map:Message'" > "$BINARY_FILE_PATH" # Check the output in the protobuf format echo @@ -34,7 +35,8 @@ hexdump -C $BINARY_FILE_PATH # Check the input in the protobuf format (now the table contains the same data twice). echo -$CLICKHOUSE_CLIENT --query "INSERT INTO map_00825 FORMAT Protobuf SETTINGS format_schema='$CURDIR/00825_protobuf_format_map:Message'" < "$BINARY_FILE_PATH" -$CLICKHOUSE_CLIENT --query "SELECT * FROM map_00825" +$CLICKHOUSE_CLIENT --query "INSERT INTO map_protobuf_00825 FORMAT Protobuf SETTINGS format_schema='$SCHEMADIR/00825_protobuf_format_map:Message'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM map_protobuf_00825" rm "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "DROP TABLE map_protobuf_00825" diff --git a/tests/queries/0_stateless/00825_protobuf_format_nested_optional.sh b/tests/queries/0_stateless/00825_protobuf_format_nested_optional.sh index 58ded92f2c1..b33db75b5c9 100755 --- a/tests/queries/0_stateless/00825_protobuf_format_nested_optional.sh +++ b/tests/queries/0_stateless/00825_protobuf_format_nested_optional.sh @@ -3,13 +3,14 @@ # https://github.com/ClickHouse/ClickHouse/issues/6497 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SCHEMADIR=$CURDIR/format_schemas # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -eo pipefail # Run the client. -$CLICKHOUSE_CLIENT --multiquery <<'EOF' +$CLICKHOUSE_CLIENT --multiquery < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM nested_optional_protobuf_00825 FORMAT Protobuf SETTINGS format_schema = '$SCHEMADIR/00825_protobuf_format_nested_optional:Message'" > "$BINARY_FILE_PATH" # Check the output in the protobuf format echo -$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$CURDIR/00825_protobuf_format_nested_optional:Message" --input "$BINARY_FILE_PATH" +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/00825_protobuf_format_nested_optional:Message" --input "$BINARY_FILE_PATH" # Check the input in the protobuf format (now the table contains the same data twice). echo -$CLICKHOUSE_CLIENT --query "INSERT INTO nested_optional_protobuf_00825 FORMAT Protobuf SETTINGS format_schema='$CURDIR/00825_protobuf_format_nested_optional:Message'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "INSERT INTO nested_optional_protobuf_00825 FORMAT Protobuf SETTINGS format_schema='$SCHEMADIR/00825_protobuf_format_nested_optional:Message'" < "$BINARY_FILE_PATH" $CLICKHOUSE_CLIENT --query "SELECT * FROM nested_optional_protobuf_00825" rm "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "DROP TABLE nested_optional_protobuf_00825" diff --git a/tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.reference b/tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.reference new file mode 100644 index 00000000000..ba84d41eb51 --- /dev/null +++ b/tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.reference @@ -0,0 +1,13 @@ +1000 1K +2000 2K +3000 3K + +Binary representation: +00000000 08 e8 07 12 02 31 4b |.....1K| +00000007 + +x: 1000 +str: "1K" + +Roundtrip: +1000 1K diff --git a/tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.sh b/tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.sh new file mode 100755 index 00000000000..b95d35e8256 --- /dev/null +++ b/tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SCHEMADIR=$CURDIR/format_schemas +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -eo pipefail + +# Run the client. +$CLICKHOUSE_CLIENT --multiquery < "$BINARY_FILE_PATH" + +# Check the output in the ProtobufSingle format +echo +echo "Binary representation:" +hexdump -C $BINARY_FILE_PATH + +echo +(cd $SCHEMADIR && protoc --decode Message 00825_protobuf_format_no_length_delimiter.proto) < $BINARY_FILE_PATH + +# Check the input in the ProtobufSingle format. +echo +echo "Roundtrip:" +$CLICKHOUSE_CLIENT --query "CREATE TABLE roundtrip_no_length_delimiter_protobuf_00825 AS no_length_delimiter_protobuf_00825" +$CLICKHOUSE_CLIENT --query "INSERT INTO roundtrip_no_length_delimiter_protobuf_00825 FORMAT ProtobufSingle SETTINGS format_schema='$SCHEMADIR/00825_protobuf_format_no_length_delimiter:Message'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM roundtrip_no_length_delimiter_protobuf_00825" +rm "$BINARY_FILE_PATH" + +# The ProtobufSingle format can't be used to write multiple rows because this format doesn't have any row delimiter. +$CLICKHOUSE_CLIENT --multiquery --testmode > /dev/null <c7uVQdUpWmqfBS_dn2f4DyeWgSvvZUx1G`$q*q5M_gQ(`{0Jmyf0nVI#tr z`jWg{?!Axa_dL%%zso!l*P0q7M5el=L{(W*!i_(Z{c-_qdW~!QN zwffTbKn=7X_j1*_gBm#%gm*>tsa}F8Tn&4AoDdm7*B2Qho8er+J(b1jZ?7e zzA>&`6!>}8>t0~5vagG4BYs0Qqq-?|X|gJ*mZ?Yv)}m5Zl&W1}LZe2QddphOJ~J6N ztND<4`2R$ zwE$HFU-|K=qfpCnsA6&PvvCdwm#J14tqlVsnlS?a~;hiG0ku4 zPj{QC-W?hKkITcs*^O>sd!^Laegn`AgBr|F15H}+Fl;q7Iyg5@1KVG}e+XsS?y>t* zQHSk@DI|*g80QgL{uIdzbc8D^`2h##jUG92s(zf)#<|B%b2AG~ zf$N-0(GT#S3*Yn1m2ZDz;h0I`#HzW9Z_p3kJM#6YKF?0*SySvqOM160>Frc78w75k zil!g{&jJbX$6o~?WSqOXm>+18L#Mg9%4P=?@gLwXApkX99l7w!;^L%mtR4W}iRV6a zxXkNbd;W|03%^#h736ooWB^DXd}|s89mR8Wdd_&eJk% zJTT<||CI-TEhvI>>ZEX^X|7^u?$XS>E6dpJO*yJxV?*t2%`MM|K>CkdrR2vNcek0e z>O|Slu_P0T7`Nm|5Xmpx5B4@BEK8Wy$oe{!uEV$u^xh^!%${^SVy9Don=ve*G=VUD zmV!p0)TdDeLM|gHn$J%N#p)j9R(!I$4{=0ReKJ0XSnQKW5KDdf7%G=F?8mt=fuwxy zEYF)|IpamL*NuG`W*JVIf?%o?S&Fe2CfS+DNqFY(ME@lZ7BgoJ*JTy>GKho~3F|=w zToC^@t|#6bv3e6M+&94zm|PSuizIN`3YL#gzO3@ zWnnAR=*^|9cUj1Kiy3'; -SELECT * FROM out_persons_00825 ORDER BY name FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:AltPerson'; -SELECT 'STRINGS->'; -SELECT * FROM out_persons_00825 ORDER BY name FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:StrPerson'; -SELECT 'SYNTAX2->'; -SELECT * FROM out_persons_00825 ORDER BY name FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format_syntax2:Syntax2Person'; -SELECT 'SQUARES->'; -SELECT * FROM out_squares_00825 ORDER BY number FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:NumberAndSquare'; - -SELECT '\n\n** ProtobufSingle **\n\n'; - -SELECT * FROM out_persons_00825 ORDER BY name LIMIT 1 FORMAT ProtobufSingle SETTINGS format_schema = '$CURDIR/00825_protobuf_format:Person'; -SELECT 'ALTERNATIVE->'; -SELECT * FROM out_persons_00825 ORDER BY name LIMIT 1 FORMAT ProtobufSingle SETTINGS format_schema = '$CURDIR/00825_protobuf_format:AltPerson'; -SELECT 'STRINGS->'; -SELECT * FROM out_persons_00825 ORDER BY name LIMIT 1 FORMAT ProtobufSingle SETTINGS format_schema = '$CURDIR/00825_protobuf_format:StrPerson'; -SELECT 'SYNTAX2->'; -SELECT * FROM out_persons_00825 ORDER BY name LIMIT 1 FORMAT ProtobufSingle SETTINGS format_schema = '$CURDIR/00825_protobuf_format_syntax2:Syntax2Person'; -SELECT 'SQUARES->'; -SELECT * FROM out_squares_00825 ORDER BY number LIMIT 1 FORMAT ProtobufSingle SETTINGS format_schema = '$CURDIR/00825_protobuf_format:NumberAndSquare'; - --- Code: 546, e.displayText() = DB::Exception: The ProtobufSingle format can't be used to write multiple rows because this format doesn't have any row delimiter. -SELECT * FROM out_persons_00825 ORDER BY name FORMAT ProtobufSingle SETTINGS format_schema = '$CURDIR/00825_protobuf_format:Person'; -- { clientError 546 } - -DROP TABLE IF EXISTS out_persons_00825; -DROP TABLE IF EXISTS out_squares_00825; -EOF diff --git a/tests/queries/0_stateless/00825_protobuf_format_persons.reference b/tests/queries/0_stateless/00825_protobuf_format_persons.reference new file mode 100644 index 00000000000..711980b3592 --- /dev/null +++ b/tests/queries/0_stateless/00825_protobuf_format_persons.reference @@ -0,0 +1,569 @@ +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753215,37.622504] 3.14 214.10 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 jpg \N 0 \N 26 pisces [] [100,200,50] Plymouth [50.403724,-4.142123] 3.14159 \N 0.007 5.4 -20000000000000 [] [] \N [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 bmp +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.970682,33.074981] 3.14159265358979 100000000000.00 800 -3.2 154400000 ['pound'] [16] 503 [] + +Schema 00825_protobuf_format_persons:Person + +Binary representation: +00000000 f4 01 0a 24 61 37 35 32 32 31 35 38 2d 33 64 34 |...$a7522158-3d4| +00000010 31 2d 34 62 37 37 2d 61 64 36 39 2d 36 63 35 39 |1-4b77-ad69-6c59| +00000020 38 65 65 35 35 63 34 39 12 04 49 76 61 6e 1a 06 |8ee55c49..Ivan..| +00000030 50 65 74 72 6f 76 20 01 28 af 1f 32 03 70 6e 67 |Petrov .(..2.png| +00000040 3a 0d 2b 37 34 39 35 31 32 33 34 35 36 37 00 40 |:.+74951234567.@| +00000050 01 4d fc d0 30 5c 50 26 58 09 62 09 59 65 73 74 |.M..0\P&X.b.Yest| +00000060 65 72 64 61 79 62 07 46 6c 6f 77 65 72 73 6a 04 |erdayb.Flowersj.| +00000070 ff 01 00 00 72 06 4d 6f 73 63 6f 77 7a 08 4b 03 |....r.Moscowz.K.| +00000080 5f 42 72 7d 16 42 81 01 1f 85 eb 51 b8 1e 09 40 |_Br}.B.....Q...@| +00000090 89 01 33 33 33 33 33 c3 6a 40 95 01 cd cc cc 3d |..33333.j@.....=| +000000a0 9d 01 9a 99 b9 40 a0 01 80 c4 d7 8d 7f aa 01 0c |.....@..........| +000000b0 0a 05 6d 65 74 65 72 15 00 00 80 3f aa 01 11 0a |..meter....?....| +000000c0 0a 63 65 6e 74 69 6d 65 74 65 72 15 0a d7 23 3c |.centimeter...#<| +000000d0 aa 01 10 0a 09 6b 69 6c 6f 6d 65 74 65 72 15 00 |.....kilometer..| +000000e0 00 7a 44 b2 01 10 0a 0e a2 06 0b 0a 09 08 f4 03 |.zD.............| +000000f0 12 04 f5 03 f6 03 7e 0a 24 63 36 39 34 61 64 38 |......~.$c694ad8| +00000100 61 2d 66 37 31 34 2d 34 65 61 33 2d 39 30 37 64 |a-f714-4ea3-907d| +00000110 2d 66 64 35 34 66 62 32 35 64 39 62 35 12 07 4e |-fd54fb25d9b5..N| +00000120 61 74 61 6c 69 61 1a 08 53 6f 6b 6f 6c 6f 76 61 |atalia..Sokolova| +00000130 28 a6 3f 32 03 6a 70 67 50 1a 58 0b 6a 04 64 c8 |(.?2.jpgP.X.j.d.| +00000140 01 32 72 08 50 6c 79 6d 6f 75 74 68 7a 08 6a 9d |.2r.Plymouthz.j.| +00000150 49 42 46 8c 84 c0 81 01 6e 86 1b f0 f9 21 09 40 |IBF.....n....!.@| +00000160 95 01 42 60 e5 3b 9d 01 cd cc ac 40 a0 01 ff ff |..B`.;.....@....| +00000170 a9 ce 93 8c 09 c0 01 0a 24 61 37 64 61 31 61 61 |........$a7da1aa| +00000180 36 2d 66 34 32 35 2d 34 37 38 39 2d 38 39 34 37 |6-f425-4789-8947| +00000190 2d 62 30 33 34 37 38 36 65 64 33 37 34 12 06 56 |-b034786ed374..V| +000001a0 61 73 69 6c 79 1a 07 53 69 64 6f 72 6f 76 20 01 |asily..Sidorov .| +000001b0 28 fb 48 32 03 62 6d 70 3a 0d 2b 34 34 32 30 31 |(.H2.bmp:.+44201| +000001c0 32 33 34 35 36 37 38 40 01 4d 50 e0 27 5c 50 17 |2345678@.MP.'\P.| +000001d0 58 04 62 05 53 75 6e 6e 79 6a 05 fa 01 f4 01 0a |X.b.Sunnyj......| +000001e0 72 08 4d 75 72 6d 61 6e 73 6b 7a 08 fd f0 89 42 |r.Murmanskz....B| +000001f0 c8 4c 04 42 81 01 11 2d 44 54 fb 21 09 40 89 01 |.L.B...-DT.!.@..| +00000200 00 00 00 e8 76 48 37 42 95 01 00 00 48 44 9d 01 |....vH7B....HD..| +00000210 cd cc 4c c0 a0 01 80 d4 9f 93 01 aa 01 0c 0a 05 |..L.............| +00000220 70 6f 75 6e 64 15 00 00 80 41 b2 01 0a 0a 08 a2 |pound....A......| +00000230 06 05 0a 03 08 f7 03 |.......| +00000237 + +MESSAGE #1 AT 0x00000002 +uuid: "a7522158-3d41-4b77-ad69-6c598ee55c49" +name: "Ivan" +surname: "Petrov" +gender: male +birthDate: 4015 +photo: "png" +phoneNumber: "+74951234567\000" +isOnline: true +visitTime: 1546703100 +age: 38 +zodiacSign: capricorn +songs: "Yesterday" +songs: "Flowers" +color: 255 +color: 0 +color: 0 +hometown: "Moscow" +location: 55.7532158 +location: 37.6225052 +pi: 3.14 +lotteryWin: 214.1 +someRatio: 0.1 +temperature: 5.8 +randomBigNumber: 17060000000 +measureUnits { + unit: "meter" + coef: 1 +} +measureUnits { + unit: "centimeter" + coef: 0.01 +} +measureUnits { + unit: "kilometer" + coef: 1000 +} +nestiness { + a { + b { + c { + d: 500 + e: 501 + e: 502 + } + } + } +} +MESSAGE #2 AT 0x000000F7 +uuid: "c694ad8a-f714-4ea3-907d-fd54fb25d9b5" +name: "Natalia" +surname: "Sokolova" +birthDate: 8102 +photo: "jpg" +age: 26 +zodiacSign: pisces +color: 100 +color: 200 +color: 50 +hometown: "Plymouth" +location: 50.4037247 +location: -4.14212322 +pi: 3.14159 +someRatio: 0.007 +temperature: 5.4 +randomBigNumber: -20000000000000 +MESSAGE #3 AT 0x00000177 +uuid: "a7da1aa6-f425-4789-8947-b034786ed374" +name: "Vasily" +surname: "Sidorov" +gender: male +birthDate: 9339 +photo: "bmp" +phoneNumber: "+442012345678" +isOnline: true +visitTime: 1546117200 +age: 23 +zodiacSign: leo +songs: "Sunny" +color: 250 +color: 244 +color: 10 +hometown: "Murmansk" +location: 68.9706802 +location: 33.0749817 +pi: 3.14159265358979 +lotteryWin: 100000000000 +someRatio: 800 +temperature: -3.2 +randomBigNumber: 154400000 +measureUnits { + unit: "pound" + coef: 16 +} +nestiness { + a { + b { + c { + d: 503 + } + } + } +} + +Binary representation is as expected + +Roundtrip: +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753216,37.622504] 3.14 214.10 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 jpg \N 0 \N 26 pisces [] [100,200,50] Plymouth [50.403724,-4.142123] 3.14159 \N 0.007 5.4 -20000000000000 [] [] \N [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 bmp +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.970680,33.074982] 3.14159265358979 100000000000.00 800 -3.2 154400000 ['pound'] [16] 503 [] + +Schema 00825_protobuf_format_persons:AltPerson + +Binary representation: +00000000 c4 01 08 01 12 04 49 76 61 6e 28 87 a8 c4 9b 97 |......Ivan(.....| +00000010 02 52 06 50 65 74 72 6f 76 72 0c 00 00 7f 43 00 |.R.Petrovr....C.| +00000020 00 00 00 00 00 00 00 79 fc d0 30 5c 00 00 00 00 |.......y..0\....| +00000030 c8 02 0a c2 05 0c 00 00 80 3f 0a d7 23 3c 00 00 |.........?..#<..| +00000040 7a 44 9a 06 05 6d 65 74 65 72 9a 06 0a 63 65 6e |zD...meter...cen| +00000050 74 69 6d 65 74 65 72 9a 06 09 6b 69 6c 6f 6d 65 |timeter...kilome| +00000060 74 65 72 a1 06 00 00 00 a0 99 99 b9 3f a8 06 37 |ter.........?..7| +00000070 a8 06 25 bd 06 c3 f5 48 40 fa 06 02 33 38 90 08 |..%....H@...38..| +00000080 c6 09 e1 08 00 f1 da f8 03 00 00 00 b0 09 af 1f |................| +00000090 d0 0c d6 01 e2 12 24 61 37 35 32 32 31 35 38 2d |......$a7522158-| +000000a0 33 64 34 31 2d 34 62 37 37 2d 61 64 36 39 2d 36 |3d41-4b77-ad69-6| +000000b0 63 35 39 38 65 65 35 35 63 34 39 a0 38 f4 03 aa |c598ee55c49.8...| +000000c0 38 04 f5 03 f6 03 84 01 12 07 4e 61 74 61 6c 69 |8.........Natali| +000000d0 61 52 08 53 6f 6b 6f 6c 6f 76 61 72 0c 00 00 c8 |aR.Sokolovar....| +000000e0 42 00 00 48 43 00 00 48 42 c8 02 0a a1 06 00 00 |B..HC..HB.......| +000000f0 00 40 08 ac 7c 3f a8 06 32 a8 06 fc ff ff ff ff |.@..|?..2.......| +00000100 ff ff ff ff 01 b0 06 01 bd 06 d0 0f 49 40 fa 06 |............I@..| +00000110 02 32 36 90 08 db 01 e1 08 00 c0 1a 63 cf ed ff |.26.........c...| +00000120 ff b0 09 a6 3f e2 12 24 63 36 39 34 61 64 38 61 |....?..$c694ad8a| +00000130 2d 66 37 31 34 2d 34 65 61 33 2d 39 30 37 64 2d |-f714-4ea3-907d-| +00000140 66 64 35 34 66 62 32 35 64 39 62 35 a3 01 08 01 |fd54fb25d9b5....| +00000150 12 06 56 61 73 69 6c 79 28 ce ca f4 cf ee 0c 52 |..Vasily(......R| +00000160 07 53 69 64 6f 72 6f 76 72 0c 00 00 7a 43 00 00 |.Sidorovr...zC..| +00000170 74 43 00 00 20 41 79 50 e0 27 5c 00 00 00 00 c8 |tC.. AyP.'\.....| +00000180 02 05 c2 05 04 00 00 80 41 9a 06 05 70 6f 75 6e |........A...poun| +00000190 64 a1 06 00 00 00 00 00 00 89 40 a8 06 44 a8 06 |d.........@..D..| +000001a0 21 bd 06 db 0f 49 40 fa 06 02 32 33 90 08 d3 05 |!....I@...23....| +000001b0 e1 08 00 f5 33 09 00 00 00 00 b0 09 fb 48 d0 0c |....3........H..| +000001c0 80 d0 db c3 f4 02 e2 12 24 61 37 64 61 31 61 61 |........$a7da1aa| +000001d0 36 2d 66 34 32 35 2d 34 37 38 39 2d 38 39 34 37 |6-f425-4789-8947| +000001e0 2d 62 30 33 34 37 38 36 65 64 33 37 34 a0 38 f7 |-b034786ed374.8.| +000001f0 03 |.| +000001f1 + +MESSAGE #1 AT 0x00000002 +isOnline: online +name: "Ivan" +phoneNumber: 74951234567 +surname: "Petrov" +color: 255 +color: 0 +color: 0 +visitTime: 1546703100 +temperature: 5 +measureUnits_coef: 1 +measureUnits_coef: 0.01 +measureUnits_coef: 1000 +measureUnits_unit: "meter" +measureUnits_unit: "centimeter" +measureUnits_unit: "kilometer" +someRatio: 0.10000000149011612 +location: 55 +location: 37 +pi: 3.14 +age: "38" +zodiacSign: 1222 +randomBigNumber: 17060000000 +birthDate: 4015 +lotteryWin: 214 +uuid: "a7522158-3d41-4b77-ad69-6c598ee55c49" +nestiness_a_b_c_d: 500 +nestiness_a_b_c_e: 501 +nestiness_a_b_c_e: 502 +MESSAGE #2 AT 0x000000C8 +name: "Natalia" +surname: "Sokolova" +color: 100 +color: 200 +color: 50 +temperature: 5 +someRatio: 0.0070000002160668373 +location: 50 +location: -4 +gender: female +pi: 3.14159 +age: "26" +zodiacSign: 219 +randomBigNumber: -20000000000000 +birthDate: 8102 +uuid: "c694ad8a-f714-4ea3-907d-fd54fb25d9b5" +MESSAGE #3 AT 0x0000014E +isOnline: online +name: "Vasily" +phoneNumber: 442012345678 +surname: "Sidorov" +color: 250 +color: 244 +color: 10 +visitTime: 1546117200 +temperature: -3 +measureUnits_coef: 16 +measureUnits_unit: "pound" +someRatio: 800 +location: 68 +location: 33 +pi: 3.14159274 +age: "23" +zodiacSign: 723 +randomBigNumber: 154400000 +birthDate: 9339 +lotteryWin: 100000000000 +uuid: "a7da1aa6-f425-4789-8947-b034786ed374" +nestiness_a_b_c_d: 503 + +Binary representation is as expected + +Roundtrip: +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 \N 74951234567\0\0 1 2019-01-05 18:45:00 38 capricorn [] [255,0,0] [55.000000,37.000000] 3.140000104904175 214.00 0.1 5.0 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 \N \N 0 \N 26 pisces [] [100,200,50] [50.000000,-4.000000] 3.141590118408203 \N 0.007 5.0 -20000000000000 [] [] \N [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 \N 442012345678\0 1 2018-12-30 00:00:00 23 leo [] [250,244,10] [68.000000,33.000000] 3.1415927410125732 100000000000.00 800 -3.0 154400000 ['pound'] [16] 503 [] + +Schema 00825_protobuf_format_persons:StrPerson + +Binary representation: +00000000 a7 02 0a 24 61 37 35 32 32 31 35 38 2d 33 64 34 |...$a7522158-3d4| +00000010 31 2d 34 62 37 37 2d 61 64 36 39 2d 36 63 35 39 |1-4b77-ad69-6c59| +00000020 38 65 65 35 35 63 34 39 12 04 49 76 61 6e 1a 06 |8ee55c49..Ivan..| +00000030 50 65 74 72 6f 76 22 04 6d 61 6c 65 2a 0a 31 39 |Petrov".male*.19| +00000040 38 30 2d 31 32 2d 32 39 3a 0d 2b 37 34 39 35 31 |80-12-29:.+74951| +00000050 32 33 34 35 36 37 00 42 01 31 4a 13 32 30 31 39 |234567.B.1J.2019| +00000060 2d 30 31 2d 30 35 20 31 38 3a 34 35 3a 30 30 52 |-01-05 18:45:00R| +00000070 02 33 38 5a 09 63 61 70 72 69 63 6f 72 6e 62 09 |.38Z.capricornb.| +00000080 59 65 73 74 65 72 64 61 79 62 07 46 6c 6f 77 65 |Yesterdayb.Flowe| +00000090 72 73 6a 03 32 35 35 6a 01 30 6a 01 30 72 06 4d |rsj.255j.0j.0r.M| +000000a0 6f 73 63 6f 77 7a 09 35 35 2e 37 35 33 32 31 35 |oscowz.55.753215| +000000b0 7a 09 33 37 2e 36 32 32 35 30 34 82 01 04 33 2e |z.37.622504...3.| +000000c0 31 34 8a 01 06 32 31 34 2e 31 30 92 01 03 30 2e |14...214.10...0.| +000000d0 31 9a 01 03 35 2e 38 a2 01 0b 31 37 30 36 30 30 |1...5.8...170600| +000000e0 30 30 30 30 30 aa 01 2d 0a 05 6d 65 74 65 72 0a |00000..-..meter.| +000000f0 0a 63 65 6e 74 69 6d 65 74 65 72 0a 09 6b 69 6c |.centimeter..kil| +00000100 6f 6d 65 74 65 72 12 01 31 12 04 30 2e 30 31 12 |ometer..1..0.01.| +00000110 04 31 30 30 30 b2 01 11 0a 0f 0a 03 35 30 30 12 |.1000.......500.| +00000120 03 35 30 31 12 03 35 30 32 b4 01 0a 24 63 36 39 |.501..502...$c69| +00000130 34 61 64 38 61 2d 66 37 31 34 2d 34 65 61 33 2d |4ad8a-f714-4ea3-| +00000140 39 30 37 64 2d 66 64 35 34 66 62 32 35 64 39 62 |907d-fd54fb25d9b| +00000150 35 12 07 4e 61 74 61 6c 69 61 1a 08 53 6f 6b 6f |5..Natalia..Soko| +00000160 6c 6f 76 61 22 06 66 65 6d 61 6c 65 2a 0a 31 39 |lova".female*.19| +00000170 39 32 2d 30 33 2d 30 38 42 01 30 52 02 32 36 5a |92-03-08B.0R.26Z| +00000180 06 70 69 73 63 65 73 6a 03 31 30 30 6a 03 32 30 |.piscesj.100j.20| +00000190 30 6a 02 35 30 72 08 50 6c 79 6d 6f 75 74 68 7a |0j.50r.Plymouthz| +000001a0 09 35 30 2e 34 30 33 37 32 34 7a 09 2d 34 2e 31 |.50.403724z.-4.1| +000001b0 34 32 31 32 33 82 01 07 33 2e 31 34 31 35 39 92 |42123...3.14159.| +000001c0 01 05 30 2e 30 30 37 9a 01 03 35 2e 34 a2 01 0f |..0.007...5.4...| +000001d0 2d 32 30 30 30 30 30 30 30 30 30 30 30 30 30 84 |-20000000000000.| +000001e0 02 0a 24 61 37 64 61 31 61 61 36 2d 66 34 32 35 |..$a7da1aa6-f425| +000001f0 2d 34 37 38 39 2d 38 39 34 37 2d 62 30 33 34 37 |-4789-8947-b0347| +00000200 38 36 65 64 33 37 34 12 06 56 61 73 69 6c 79 1a |86ed374..Vasily.| +00000210 07 53 69 64 6f 72 6f 76 22 04 6d 61 6c 65 2a 0a |.Sidorov".male*.| +00000220 31 39 39 35 2d 30 37 2d 32 38 3a 0d 2b 34 34 32 |1995-07-28:.+442| +00000230 30 31 32 33 34 35 36 37 38 42 01 31 4a 13 32 30 |012345678B.1J.20| +00000240 31 38 2d 31 32 2d 33 30 20 30 30 3a 30 30 3a 30 |18-12-30 00:00:0| +00000250 30 52 02 32 33 5a 03 6c 65 6f 62 05 53 75 6e 6e |0R.23Z.leob.Sunn| +00000260 79 6a 03 32 35 30 6a 03 32 34 34 6a 02 31 30 72 |yj.250j.244j.10r| +00000270 08 4d 75 72 6d 61 6e 73 6b 7a 09 36 38 2e 39 37 |.Murmanskz.68.97| +00000280 30 36 38 32 7a 09 33 33 2e 30 37 34 39 38 31 82 |0682z.33.074981.| +00000290 01 10 33 2e 31 34 31 35 39 32 36 35 33 35 38 39 |..3.141592653589| +000002a0 37 39 8a 01 0f 31 30 30 30 30 30 30 30 30 30 30 |79...10000000000| +000002b0 30 2e 30 30 92 01 03 38 30 30 9a 01 04 2d 33 2e |0.00...800...-3.| +000002c0 32 a2 01 09 31 35 34 34 30 30 30 30 30 aa 01 0b |2...154400000...| +000002d0 0a 05 70 6f 75 6e 64 12 02 31 36 b2 01 07 0a 05 |..pound..16.....| +000002e0 0a 03 35 30 33 |..503| +000002e5 + +MESSAGE #1 AT 0x00000002 +uuid: "a7522158-3d41-4b77-ad69-6c598ee55c49" +name: "Ivan" +surname: "Petrov" +gender: "male" +birthDate: "1980-12-29" +phoneNumber: "+74951234567\000" +isOnline: "1" +visitTime: "2019-01-05 18:45:00" +age: "38" +zodiacSign: "capricorn" +songs: "Yesterday" +songs: "Flowers" +color: "255" +color: "0" +color: "0" +hometown: "Moscow" +location: "55.753215" +location: "37.622504" +pi: "3.14" +lotteryWin: "214.10" +someRatio: "0.1" +temperature: "5.8" +randomBigNumber: "17060000000" +measureUnits { + unit: "meter" + unit: "centimeter" + unit: "kilometer" + coef: "1" + coef: "0.01" + coef: "1000" +} +nestiness_a { + b_c { + d: "500" + e: "501" + e: "502" + } +} +MESSAGE #2 AT 0x0000012B +uuid: "c694ad8a-f714-4ea3-907d-fd54fb25d9b5" +name: "Natalia" +surname: "Sokolova" +gender: "female" +birthDate: "1992-03-08" +isOnline: "0" +age: "26" +zodiacSign: "pisces" +color: "100" +color: "200" +color: "50" +hometown: "Plymouth" +location: "50.403724" +location: "-4.142123" +pi: "3.14159" +someRatio: "0.007" +temperature: "5.4" +randomBigNumber: "-20000000000000" +MESSAGE #3 AT 0x000001E1 +uuid: "a7da1aa6-f425-4789-8947-b034786ed374" +name: "Vasily" +surname: "Sidorov" +gender: "male" +birthDate: "1995-07-28" +phoneNumber: "+442012345678" +isOnline: "1" +visitTime: "2018-12-30 00:00:00" +age: "23" +zodiacSign: "leo" +songs: "Sunny" +color: "250" +color: "244" +color: "10" +hometown: "Murmansk" +location: "68.970682" +location: "33.074981" +pi: "3.14159265358979" +lotteryWin: "100000000000.00" +someRatio: "800" +temperature: "-3.2" +randomBigNumber: "154400000" +measureUnits { + unit: "pound" + coef: "16" +} +nestiness_a { + b_c { + d: "503" + } +} + +Binary representation is as expected + +Roundtrip: +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 \N +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753215,37.622504] 3.14 214.10 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 \N \N 0 \N 26 pisces [] [100,200,50] Plymouth [50.403724,-4.142123] 3.14159 \N 0.007 5.4 -20000000000000 [] [] \N [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 \N +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.970682,33.074981] 3.14159265358979 100000000000.00 800 -3.2 154400000 ['pound'] [16] 503 [] + +Schema 00825_protobuf_format_syntax2:Syntax2Person + +Binary representation: +00000000 f1 01 0a 24 61 37 35 32 32 31 35 38 2d 33 64 34 |...$a7522158-3d4| +00000010 31 2d 34 62 37 37 2d 61 64 36 39 2d 36 63 35 39 |1-4b77-ad69-6c59| +00000020 38 65 65 35 35 63 34 39 12 04 49 76 61 6e 1a 06 |8ee55c49..Ivan..| +00000030 50 65 74 72 6f 76 20 01 28 af 1f 32 03 70 6e 67 |Petrov .(..2.png| +00000040 3a 0d 2b 37 34 39 35 31 32 33 34 35 36 37 00 40 |:.+74951234567.@| +00000050 01 4d fc d0 30 5c 50 26 58 09 62 09 59 65 73 74 |.M..0\P&X.b.Yest| +00000060 65 72 64 61 79 62 07 46 6c 6f 77 65 72 73 68 ff |erdayb.Flowersh.| +00000070 01 68 00 68 00 72 06 4d 6f 73 63 6f 77 7a 08 4b |.h.h.r.Moscowz.K| +00000080 03 5f 42 72 7d 16 42 81 01 1f 85 eb 51 b8 1e 09 |._Br}.B.....Q...| +00000090 40 89 01 33 33 33 33 33 c3 6a 40 95 01 cd cc cc |@..33333.j@.....| +000000a0 3d 9d 01 9a 99 b9 40 a0 01 80 c4 d7 8d 7f ab 01 |=.....@.........| +000000b0 0d 00 00 80 3f 0d 0a d7 23 3c 0d 00 00 7a 44 12 |....?...#<...zD.| +000000c0 05 6d 65 74 65 72 12 0a 63 65 6e 74 69 6d 65 74 |.meter..centimet| +000000d0 65 72 12 09 6b 69 6c 6f 6d 65 74 65 72 ac 01 b3 |er..kilometer...| +000000e0 01 0b a2 06 0b 0b 08 f4 03 10 f5 03 10 f6 03 0c |................| +000000f0 0c b4 01 83 01 0a 24 63 36 39 34 61 64 38 61 2d |......$c694ad8a-| +00000100 66 37 31 34 2d 34 65 61 33 2d 39 30 37 64 2d 66 |f714-4ea3-907d-f| +00000110 64 35 34 66 62 32 35 64 39 62 35 12 07 4e 61 74 |d54fb25d9b5..Nat| +00000120 61 6c 69 61 1a 08 53 6f 6b 6f 6c 6f 76 61 20 00 |alia..Sokolova .| +00000130 28 a6 3f 32 03 6a 70 67 40 00 50 1a 58 0b 68 64 |(.?2.jpg@.P.X.hd| +00000140 68 c8 01 68 32 72 08 50 6c 79 6d 6f 75 74 68 7a |h..h2r.Plymouthz| +00000150 08 6a 9d 49 42 46 8c 84 c0 81 01 6e 86 1b f0 f9 |.j.IBF.....n....| +00000160 21 09 40 95 01 42 60 e5 3b 9d 01 cd cc ac 40 a0 |!.@..B`.;.....@.| +00000170 01 ff ff a9 ce 93 8c 09 c3 01 0a 24 61 37 64 61 |...........$a7da| +00000180 31 61 61 36 2d 66 34 32 35 2d 34 37 38 39 2d 38 |1aa6-f425-4789-8| +00000190 39 34 37 2d 62 30 33 34 37 38 36 65 64 33 37 34 |947-b034786ed374| +000001a0 12 06 56 61 73 69 6c 79 1a 07 53 69 64 6f 72 6f |..Vasily..Sidoro| +000001b0 76 20 01 28 fb 48 32 03 62 6d 70 3a 0d 2b 34 34 |v .(.H2.bmp:.+44| +000001c0 32 30 31 32 33 34 35 36 37 38 40 01 4d 50 e0 27 |2012345678@.MP.'| +000001d0 5c 50 17 58 04 62 05 53 75 6e 6e 79 68 fa 01 68 |\P.X.b.Sunnyh..h| +000001e0 f4 01 68 0a 72 08 4d 75 72 6d 61 6e 73 6b 7a 08 |..h.r.Murmanskz.| +000001f0 fd f0 89 42 c8 4c 04 42 81 01 11 2d 44 54 fb 21 |...B.L.B...-DT.!| +00000200 09 40 89 01 00 00 00 e8 76 48 37 42 95 01 00 00 |.@......vH7B....| +00000210 48 44 9d 01 cd cc 4c c0 a0 01 80 d4 9f 93 01 ab |HD....L.........| +00000220 01 0d 00 00 80 41 12 05 70 6f 75 6e 64 ac 01 b3 |.....A..pound...| +00000230 01 0b a2 06 05 0b 08 f7 03 0c 0c b4 01 |.............| +0000023d + +MESSAGE #1 AT 0x00000002 +uuid: "a7522158-3d41-4b77-ad69-6c598ee55c49" +name: "Ivan" +surname: "Petrov" +gender: male +birthDate: 4015 +photo: "png" +phoneNumber: "+74951234567\000" +isOnline: true +visitTime: 1546703100 +age: 38 +zodiacSign: capricorn +songs: "Yesterday" +songs: "Flowers" +color: 255 +color: 0 +color: 0 +hometown: "Moscow" +location: 55.7532158 +location: 37.6225052 +pi: 3.14 +lotteryWin: 214.1 +someRatio: 0.1 +temperature: 5.8 +randomBigNumber: 17060000000 +MeasureUnits { + coef: 1 + coef: 0.01 + coef: 1000 + unit: "meter" + unit: "centimeter" + unit: "kilometer" +} +Nestiness { + A { + b { + C { + d: 500 + e: 501 + e: 502 + } + } + } +} +MESSAGE #2 AT 0x000000F5 +uuid: "c694ad8a-f714-4ea3-907d-fd54fb25d9b5" +name: "Natalia" +surname: "Sokolova" +gender: female +birthDate: 8102 +photo: "jpg" +isOnline: false +age: 26 +zodiacSign: pisces +color: 100 +color: 200 +color: 50 +hometown: "Plymouth" +location: 50.4037247 +location: -4.14212322 +pi: 3.14159 +someRatio: 0.007 +temperature: 5.4 +randomBigNumber: -20000000000000 +MESSAGE #3 AT 0x0000017A +uuid: "a7da1aa6-f425-4789-8947-b034786ed374" +name: "Vasily" +surname: "Sidorov" +gender: male +birthDate: 9339 +photo: "bmp" +phoneNumber: "+442012345678" +isOnline: true +visitTime: 1546117200 +age: 23 +zodiacSign: leo +songs: "Sunny" +color: 250 +color: 244 +color: 10 +hometown: "Murmansk" +location: 68.9706802 +location: 33.0749817 +pi: 3.14159265358979 +lotteryWin: 100000000000 +someRatio: 800 +temperature: -3.2 +randomBigNumber: 154400000 +MeasureUnits { + coef: 16 + unit: "pound" +} +Nestiness { + A { + b { + C { + d: 503 + } + } + } +} + +Binary representation is as expected + +Roundtrip: +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753216,37.622504] 3.14 214.10 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 jpg \N 0 \N 26 pisces [] [100,200,50] Plymouth [50.403724,-4.142123] 3.14159 \N 0.007 5.4 -20000000000000 [] [] \N [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 bmp +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.970680,33.074982] 3.14159265358979 100000000000.00 800 -3.2 154400000 ['pound'] [16] 503 [] diff --git a/tests/queries/0_stateless/00825_protobuf_format_persons.sh b/tests/queries/0_stateless/00825_protobuf_format_persons.sh new file mode 100755 index 00000000000..957ed738d99 --- /dev/null +++ b/tests/queries/0_stateless/00825_protobuf_format_persons.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash + +# To generate reference file for this test use the following commands: +# ninja ProtobufDelimitedMessagesSerializer +# build/utils/test-data-generator/ProtobufDelimitedMessagesSerializer + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SCHEMADIR=$CURDIR/format_schemas +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -eo pipefail + +# Run the client. +$CLICKHOUSE_CLIENT --multiquery < $BINARY_FILE_PATH +echo +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/00825_protobuf_format_persons:Person" --input "$BINARY_FILE_PATH" +echo +echo "Roundtrip:" +$CLICKHOUSE_CLIENT --query "CREATE TABLE roundtrip_persons_00825 AS persons_00825" +$CLICKHOUSE_CLIENT --query "INSERT INTO roundtrip_persons_00825 FORMAT Protobuf SETTINGS format_schema='$SCHEMADIR/00825_protobuf_format_persons:Person'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM roundtrip_persons_00825 ORDER BY name" +rm "$BINARY_FILE_PATH" + +# Use schema 00825_protobuf_format_persons:AltPerson +echo +echo "Schema 00825_protobuf_format_persons:AltPerson" +BINARY_FILE_PATH=$(mktemp "$CURDIR/00825_protobuf_format_persons.XXXXXX.binary") +$CLICKHOUSE_CLIENT --query "SELECT * FROM persons_00825 ORDER BY name FORMAT Protobuf SETTINGS format_schema = '$SCHEMADIR/00825_protobuf_format_persons:AltPerson'" > $BINARY_FILE_PATH +echo +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/00825_protobuf_format_persons:AltPerson" --input "$BINARY_FILE_PATH" +echo +echo "Roundtrip:" +$CLICKHOUSE_CLIENT --query "CREATE TABLE alt_persons_00825 AS persons_00825" +$CLICKHOUSE_CLIENT --query "INSERT INTO alt_persons_00825 FORMAT Protobuf SETTINGS format_schema='$SCHEMADIR/00825_protobuf_format_persons:AltPerson'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM alt_persons_00825 ORDER BY name" +rm "$BINARY_FILE_PATH" + +# Use schema 00825_protobuf_format_persons:StrPerson +echo +echo "Schema 00825_protobuf_format_persons:StrPerson" +BINARY_FILE_PATH=$(mktemp "$CURDIR/00825_protobuf_format_persons.XXXXXX.binary") +$CLICKHOUSE_CLIENT --query "SELECT * FROM persons_00825 ORDER BY name FORMAT Protobuf SETTINGS format_schema = '$SCHEMADIR/00825_protobuf_format_persons:StrPerson'" > $BINARY_FILE_PATH +echo +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/00825_protobuf_format_persons:StrPerson" --input "$BINARY_FILE_PATH" +echo +echo "Roundtrip:" +$CLICKHOUSE_CLIENT --query "CREATE TABLE str_persons_00825 AS persons_00825" +$CLICKHOUSE_CLIENT --query "INSERT INTO str_persons_00825 FORMAT Protobuf SETTINGS format_schema='$SCHEMADIR/00825_protobuf_format_persons:StrPerson'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM str_persons_00825 ORDER BY name" +rm "$BINARY_FILE_PATH" + +# Use schema 00825_protobuf_format_syntax2:Syntax2Person +echo +echo "Schema 00825_protobuf_format_syntax2:Syntax2Person" +BINARY_FILE_PATH=$(mktemp "$CURDIR/00825_protobuf_format_persons.XXXXXX.binary") +$CLICKHOUSE_CLIENT --query "SELECT * FROM persons_00825 ORDER BY name FORMAT Protobuf SETTINGS format_schema = '$SCHEMADIR/00825_protobuf_format_persons_syntax2:Syntax2Person'" > $BINARY_FILE_PATH +echo +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/00825_protobuf_format_persons_syntax2:Syntax2Person" --input "$BINARY_FILE_PATH" +echo +echo "Roundtrip:" +$CLICKHOUSE_CLIENT --query "CREATE TABLE syntax2_persons_00825 AS persons_00825" +$CLICKHOUSE_CLIENT --query "INSERT INTO syntax2_persons_00825 FORMAT Protobuf SETTINGS format_schema='$SCHEMADIR/00825_protobuf_format_persons_syntax2:Syntax2Person'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM syntax2_persons_00825 ORDER BY name" +rm "$BINARY_FILE_PATH" + +$CLICKHOUSE_CLIENT --multiquery < "$BINARY_FILE_PATH" + +# Check the output in the protobuf format +echo +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/00825_protobuf_format_squares:NumberAndSquare" --input "$BINARY_FILE_PATH" + +# Check the input in the protobuf format (now the table contains the same data twice). +echo +$CLICKHOUSE_CLIENT --query "INSERT INTO squares_protobuf_00825 FORMAT Protobuf SETTINGS format_schema='$SCHEMADIR/00825_protobuf_format_squares:NumberAndSquare'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM squares_protobuf_00825" + +rm "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "DROP TABLE squares_protobuf_00825" diff --git a/tests/queries/0_stateless/00825_protobuf_format_table_default.sh b/tests/queries/0_stateless/00825_protobuf_format_table_default.sh index 97f7769269a..96664ca7f20 100755 --- a/tests/queries/0_stateless/00825_protobuf_format_table_default.sh +++ b/tests/queries/0_stateless/00825_protobuf_format_table_default.sh @@ -1,13 +1,14 @@ #!/usr/bin/env bash CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SCHEMADIR=$CURDIR/format_schemas # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh set -eo pipefail # Run the client. -$CLICKHOUSE_CLIENT --multiquery <<'EOF' +$CLICKHOUSE_CLIENT --multiquery < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM table_default_protobuf_00825 ORDER BY x,y,z FORMAT Protobuf SETTINGS format_schema = '$SCHEMADIR/00825_protobuf_format_table_default:Message'" > "$BINARY_FILE_PATH" # Check the output in the protobuf format echo -$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$CURDIR/00825_protobuf_format_table_default:Message" --input "$BINARY_FILE_PATH" +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/00825_protobuf_format_table_default:Message" --input "$BINARY_FILE_PATH" # Check the input in the protobuf format (now the table contains the same data twice). echo -$CLICKHOUSE_CLIENT --query "INSERT INTO table_default_protobuf_00825 FORMAT Protobuf SETTINGS format_schema='$CURDIR/00825_protobuf_format_table_default:Message'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "INSERT INTO table_default_protobuf_00825 FORMAT Protobuf SETTINGS format_schema='$SCHEMADIR/00825_protobuf_format_table_default:Message'" < "$BINARY_FILE_PATH" $CLICKHOUSE_CLIENT --query "SELECT * FROM table_default_protobuf_00825 ORDER BY x,y,z" rm "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "DROP TABLE table_default_protobuf_00825" diff --git a/tests/queries/0_stateless/00825_protobuf_format_array_3dim.proto b/tests/queries/0_stateless/format_schemas/00825_protobuf_format_array_3dim.proto similarity index 100% rename from tests/queries/0_stateless/00825_protobuf_format_array_3dim.proto rename to tests/queries/0_stateless/format_schemas/00825_protobuf_format_array_3dim.proto diff --git a/tests/queries/0_stateless/00825_protobuf_format_array_of_arrays.proto b/tests/queries/0_stateless/format_schemas/00825_protobuf_format_array_of_arrays.proto similarity index 100% rename from tests/queries/0_stateless/00825_protobuf_format_array_of_arrays.proto rename to tests/queries/0_stateless/format_schemas/00825_protobuf_format_array_of_arrays.proto diff --git a/tests/queries/0_stateless/00825_protobuf_format_enum_mapping.proto b/tests/queries/0_stateless/format_schemas/00825_protobuf_format_enum_mapping.proto similarity index 100% rename from tests/queries/0_stateless/00825_protobuf_format_enum_mapping.proto rename to tests/queries/0_stateless/format_schemas/00825_protobuf_format_enum_mapping.proto diff --git a/tests/queries/0_stateless/00825_protobuf_format_map.proto b/tests/queries/0_stateless/format_schemas/00825_protobuf_format_map.proto similarity index 100% rename from tests/queries/0_stateless/00825_protobuf_format_map.proto rename to tests/queries/0_stateless/format_schemas/00825_protobuf_format_map.proto diff --git a/tests/queries/0_stateless/00825_protobuf_format_nested_optional.proto b/tests/queries/0_stateless/format_schemas/00825_protobuf_format_nested_optional.proto similarity index 100% rename from tests/queries/0_stateless/00825_protobuf_format_nested_optional.proto rename to tests/queries/0_stateless/format_schemas/00825_protobuf_format_nested_optional.proto diff --git a/tests/queries/0_stateless/format_schemas/00825_protobuf_format_no_length_delimiter.proto b/tests/queries/0_stateless/format_schemas/00825_protobuf_format_no_length_delimiter.proto new file mode 100644 index 00000000000..9726d0dede1 --- /dev/null +++ b/tests/queries/0_stateless/format_schemas/00825_protobuf_format_no_length_delimiter.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; + +message Message { + int32 x = 1; + string str = 2; +}; diff --git a/tests/queries/0_stateless/00825_protobuf_format.proto b/tests/queries/0_stateless/format_schemas/00825_protobuf_format_persons.proto similarity index 97% rename from tests/queries/0_stateless/00825_protobuf_format.proto rename to tests/queries/0_stateless/format_schemas/00825_protobuf_format_persons.proto index 0d9bdd83ccd..b588619f488 100644 --- a/tests/queries/0_stateless/00825_protobuf_format.proto +++ b/tests/queries/0_stateless/format_schemas/00825_protobuf_format_persons.proto @@ -143,9 +143,3 @@ message StrPerson { MeasureUnits measureUnits = 21; NestinessA nestiness_a = 22; }; - -message NumberAndSquare -{ - uint32 number = 1; - uint64 square = 2; -}; diff --git a/tests/queries/0_stateless/00825_protobuf_format_syntax2.proto b/tests/queries/0_stateless/format_schemas/00825_protobuf_format_persons_syntax2.proto similarity index 100% rename from tests/queries/0_stateless/00825_protobuf_format_syntax2.proto rename to tests/queries/0_stateless/format_schemas/00825_protobuf_format_persons_syntax2.proto diff --git a/tests/queries/0_stateless/format_schemas/00825_protobuf_format_squares.proto b/tests/queries/0_stateless/format_schemas/00825_protobuf_format_squares.proto new file mode 100644 index 00000000000..bca321568f3 --- /dev/null +++ b/tests/queries/0_stateless/format_schemas/00825_protobuf_format_squares.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; + +message NumberAndSquare { + uint32 number = 1; + uint64 square = 2; +}; diff --git a/tests/queries/0_stateless/format_schemas/00825_protobuf_format_syntax2.proto b/tests/queries/0_stateless/format_schemas/00825_protobuf_format_syntax2.proto new file mode 100644 index 00000000000..d4abea38d4e --- /dev/null +++ b/tests/queries/0_stateless/format_schemas/00825_protobuf_format_syntax2.proto @@ -0,0 +1,63 @@ +syntax = "proto2"; + +message Syntax2Person { + enum Gender { + female = 0; + male = 1; + }; + + enum ZodiacSign { + aries = 0; + taurus = 1; + gemini = 2; + cancer = 3; + leo = 4; + virgo = 5; + libra = 6; + scorpius = 7; + sagittarius = 8; + capricorn = 9; + aquarius = 10; + pisces = 11; + }; + + required string uuid = 1; + required string name = 2; + required string surname = 3; + required Gender gender = 4; + required uint32 birthDate = 5; + optional bytes photo = 6; + optional string phoneNumber = 7; + optional bool isOnline = 8; + optional fixed32 visitTime = 9; + optional uint32 age = 10; + optional ZodiacSign zodiacSign = 11; + repeated string songs = 12; + repeated uint32 color = 13; + optional string hometown = 14 [default='Moscow']; + repeated float location = 15 [packed=true]; + optional double pi = 16; + optional double lotteryWin = 17; + optional float someRatio = 18; + optional float temperature = 19; + optional sint64 randomBigNumber = 20; + optional group MeasureUnits = 21 { + repeated float coef = 1; + repeated string unit = 2; + }; + optional group Nestiness = 22 + { + optional group A = 1 { + message SubB { + optional group C = 1 { + optional uint32 d = 1; + repeated uint32 e = 2; + }; + }; + optional SubB b = 100; + }; + }; + optional string newFieldStr = 23 [default='abc']; + optional int32 newFieldInt = 24 [default=-11]; + optional bool newBool = 25 [default=true]; +}; diff --git a/tests/queries/0_stateless/00825_protobuf_format_table_default.proto b/tests/queries/0_stateless/format_schemas/00825_protobuf_format_table_default.proto similarity index 100% rename from tests/queries/0_stateless/00825_protobuf_format_table_default.proto rename to tests/queries/0_stateless/format_schemas/00825_protobuf_format_table_default.proto diff --git a/tests/queries/0_stateless/helpers/protobuf_length_delimited_encoder.py b/tests/queries/0_stateless/helpers/protobuf_length_delimited_encoder.py index 3ed42f1c820..86c5048c8a3 100755 --- a/tests/queries/0_stateless/helpers/protobuf_length_delimited_encoder.py +++ b/tests/queries/0_stateless/helpers/protobuf_length_delimited_encoder.py @@ -12,18 +12,18 @@ import tempfile def read_varint(input): res = 0 - shift = 0 + multiplier = 1 while True: c = input.read(1) if len(c) == 0: return None b = c[0] if b < 0x80: - res += b << shift + res += b * multiplier break b -= 0x80 - res += b << shift - shift = shift << 7 + res += b * multiplier + multiplier *= 0x80 return res def write_varint(output, value):