diff --git a/dbms/tests/queries/0_stateless/00825_protobuf_format.proto b/dbms/tests/queries/0_stateless/00825_protobuf_format.proto new file mode 100644 index 00000000000..47c4e74ff23 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00825_protobuf_format.proto @@ -0,0 +1,98 @@ +syntax = "proto3"; + +enum Gender { + female = 0; + male = 1; +}; + +enum ZodiacSign { + aries = 0; + taurus = 1; + gemini = 2; + cancer = 3; + leo = 4; + virgo = 5; + libra = 6; + scorpius = 7; + sagittarius = 8; + capricorn = 9; + aquarius = 10; + pisces = 11; +}; + +message Person { + string uuid = 1; + string name = 2; + string surname = 3; + Gender gender = 4; + uint32 birthDate = 5; + bytes photo = 6; + string phoneNumber = 7; + bool isOnline = 8; + fixed32 visitTime = 9; + uint32 age = 10; + ZodiacSign zodiacSign = 11; + repeated string songs = 12; + repeated uint32 color = 13; + string hometown = 14; + repeated float location = 15; + double pi = 16; + double lotteryWin = 17; + float someRatio = 18; + float temperature = 19; + sint64 randomBigNumber = 20; +}; + +enum OnlineStatus { + offline = 0; + online = 1; +}; + +message AltPerson { + enum Gender { + male = 0; + female = 1; + }; + repeated int32 location = 101 [packed=false]; + float pi = 103; + bytes uuid = 300; + bool newFieldBool = 299; + string name = 2; + Gender gender = 102; + int32 zodiacSign = 130; + int64 birthDate = 150; + bytes age = 111; + OnlineStatus isOnline = 1; + double someRatio = 100; + fixed64 visitTime = 15; + sfixed64 randomBigNumber = 140; + repeated int32 newFieldInt = 104; + repeated float color = 14; + uint64 lotteryWin = 202; + bytes surname = 10; + uint64 phoneNumber = 5; + sint32 temperature = 41; + string newFieldStr = 21; +}; + +message StrPerson { + string uuid = 1; + string name = 2; + string surname = 3; + string gender = 4; + string birthDate = 5; + string phoneNumber = 7; + string isOnline = 8; + string visitTime = 9; + string age = 10; + string zodiacSign = 11; + repeated string songs = 12; + repeated string color = 13; + string hometown = 14; + repeated string location = 15; + string pi = 16; + string lotteryWin = 17; + string someRatio = 18; + string temperature = 19; + string randomBigNumber = 20; +}; diff --git a/dbms/tests/queries/0_stateless/00825_protobuf_format_output.reference b/dbms/tests/queries/0_stateless/00825_protobuf_format_output.reference new file mode 100644 index 00000000000..ed3e22f41e3 Binary files /dev/null and b/dbms/tests/queries/0_stateless/00825_protobuf_format_output.reference differ diff --git a/dbms/tests/queries/0_stateless/00825_protobuf_format_output.sh b/dbms/tests/queries/0_stateless/00825_protobuf_format_output.sh new file mode 100755 index 00000000000..a89126a3af8 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00825_protobuf_format_output.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# To generate reference file for this test use the following commands: +# ninja ProtobufDelimitedMessagesSerializer +# build/utils/test-data-generator/ProtobufDelimitedMessagesSerializer > dbms/tests/queries/0_stateless/00825_protobuf_format_output.reference + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +set -e -o pipefail + +# Copy schema files to the current directory because the client can open schemas from the current directory only. +cp "$CURDIR/00825_protobuf_format.proto" 00825_protobuf_format_copy.proto +cp "$CURDIR/00825_protobuf_format_syntax2.proto" 00825_protobuf_format_syntax2_copy.proto + +# Run the client. +$CLICKHOUSE_CLIENT --multiquery <<'EOF' +SET allow_experimental_low_cardinality_type = 1; +CREATE DATABASE IF NOT EXISTS test; +DROP TABLE IF EXISTS test.table; + +CREATE TABLE test.table (uuid UUID, + name String, + surname String, + gender Enum8('male'=1, 'female'=0), + birthDate Date, + photo Nullable(String), + phoneNumber Nullable(FixedString(13)), + isOnline UInt8, + visitTime Nullable(DateTime), + age UInt8, + zodiacSign Enum16('aries'=321, 'taurus'=420, 'gemini'=521, 'cancer'=621, 'leo'=723, 'virgo'=823, + 'libra'=923, 'scorpius'=1023, 'sagittarius'=1122, 'capricorn'=1222, 'aquarius'=120, + 'pisces'=219), + songs Array(String), + color Array(UInt8), + hometown LowCardinality(String), + location Array(Decimal32(6)), + pi Nullable(Float64), + lotteryWin Nullable(Decimal64(2)), + someRatio Float32, + temperature Decimal32(1), + randomBigNumber Int64 + ) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO test.table VALUES (toUUID('a7522158-3d41-4b77-ad69-6c598ee55c49'), 'Ivan', 'Petrov', 'male', toDate('1980-12-29'), 'png', '+74951234567', 1, toDateTime('2019-01-05 18:45:00'), 38, 'capricorn', ['Yesterday', 'Flowers'], [255, 0, 0], 'Moscow', [55.753215, 37.622504], 3.14, 214.10, 0.1, 5.8, 17060000000); +INSERT INTO test.table VALUES (toUUID('c694ad8a-f714-4ea3-907d-fd54fb25d9b5'), 'Natalia', 'Sokolova', 'female', toDate('1992-03-08'), 'jpg', NULL, 0, NULL, 26, 'pisces', [], [100, 200, 50], 'Tver', [54.782635, 32.045251], 3.14159, NULL, 0.007, 5.4, -20000000000000); +INSERT INTO test.table VALUES (toUUID('a7da1aa6-f425-4789-8947-b034786ed374'), 'Vasily', 'Sidorov', 'male', toDate('1995-07-28'), 'bmp', '+442012345678', 1, toDateTime('2018-12-30 00:00:00'), 23, 'leo', ['Sunny'], [250, 244, 10], 'Murmansk', [68.970682, 33.074981], 3.14159265358979, 100000000000, 800, -3.2, 154400000); + +SELECT * FROM test.table ORDER BY name FORMAT Protobuf SETTINGS format_schema = '00825_protobuf_format_copy:Person'; +SELECT 'ALTERNATIVE->'; +SELECT * FROM test.table ORDER BY name FORMAT Protobuf SETTINGS format_schema = '00825_protobuf_format_copy:AltPerson'; +SELECT 'STRINGS->'; +SELECT * FROM test.table ORDER BY name FORMAT Protobuf SETTINGS format_schema = '00825_protobuf_format_copy:StrPerson'; +SELECT 'SYNTAX2->'; +SELECT * FROM test.table ORDER BY name FORMAT Protobuf SETTINGS format_schema = '00825_protobuf_format_syntax2_copy:Syntax2Person'; +EOF + +# Remove copies of the schema files. +rm "00825_protobuf_format_copy.proto" "00825_protobuf_format_syntax2_copy.proto" diff --git a/dbms/tests/queries/0_stateless/00825_protobuf_format_syntax2.proto b/dbms/tests/queries/0_stateless/00825_protobuf_format_syntax2.proto new file mode 100644 index 00000000000..dad1b596824 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00825_protobuf_format_syntax2.proto @@ -0,0 +1,47 @@ +syntax = "proto2"; + +message Syntax2Person { + enum Gender { + female = 0; + male = 1; + }; + + enum ZodiacSign { + aries = 0; + taurus = 1; + gemini = 2; + cancer = 3; + leo = 4; + virgo = 5; + libra = 6; + scorpius = 7; + sagittarius = 8; + capricorn = 9; + aquarius = 10; + pisces = 11; + }; + + required string uuid = 1; + required string name = 2; + required string surname = 3; + required Gender gender = 4; + required uint32 birthDate = 5; + optional bytes photo = 6; + optional string phoneNumber = 7; + optional bool isOnline = 8; + optional fixed32 visitTime = 9; + optional uint32 age = 10; + optional ZodiacSign zodiacSign = 11; + repeated string songs = 12; + repeated uint32 color = 13; + optional string hometown = 14 [default='Moscow']; + repeated float location = 15 [packed=true]; + optional double pi = 16; + optional double lotteryWin = 17; + optional float someRatio = 18; + optional float temperature = 19; + optional sint64 randomBigNumber = 20; + optional string newFieldStr = 21 [default='abc']; + optional int32 newFieldInt = 22 [default=-11]; + optional bool newBool = 23 [default=true]; +}; diff --git a/utils/test-data-generator/CMakeLists.txt b/utils/test-data-generator/CMakeLists.txt index 60f17894704..bf3cfa1bf7f 100644 --- a/utils/test-data-generator/CMakeLists.txt +++ b/utils/test-data-generator/CMakeLists.txt @@ -3,3 +3,9 @@ target_link_libraries(test-data-generator PRIVATE clickhouse_common_io ${Boost_P add_executable (markov-model markov-model.cpp) target_link_libraries(markov-model PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) + +protobuf_generate_cpp(ProtobufDelimitedMessagesSerializer_Srcs ProtobufDelimitedMessagesSerializer_Hdrs ${CMAKE_CURRENT_SOURCE_DIR}/../../dbms/tests/queries/0_stateless/00825_protobuf_format.proto) +protobuf_generate_cpp(ProtobufDelimitedMessagesSerializer_Srcs2 ProtobufDelimitedMessagesSerializer_Hdrs2 ${CMAKE_CURRENT_SOURCE_DIR}/../../dbms/tests/queries/0_stateless/00825_protobuf_format_syntax2.proto) +add_executable (ProtobufDelimitedMessagesSerializer ProtobufDelimitedMessagesSerializer.cpp ${ProtobufDelimitedMessagesSerializer_Srcs} ${ProtobufDelimitedMessagesSerializer_Hdrs} ${ProtobufDelimitedMessagesSerializer_Srcs2} ${ProtobufDelimitedMessagesSerializer_Hdrs2}) +target_include_directories (ProtobufDelimitedMessagesSerializer SYSTEM BEFORE PRIVATE ${Protobuf_INCLUDE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) +target_link_libraries (ProtobufDelimitedMessagesSerializer PRIVATE ${Protobuf_LIBRARY}) \ No newline at end of file diff --git a/utils/test-data-generator/ProtobufDelimitedMessagesSerializer.cpp b/utils/test-data-generator/ProtobufDelimitedMessagesSerializer.cpp new file mode 100644 index 00000000000..4a5596cb7f2 --- /dev/null +++ b/utils/test-data-generator/ProtobufDelimitedMessagesSerializer.cpp @@ -0,0 +1,332 @@ +// Reference file generator for the test dbms/tests/queries/0_stateless/00825_protobuf_format_output.sh + +#include +#include +#include "00825_protobuf_format.pb.h" +#include "00825_protobuf_format_syntax2.pb.h" + + +int main(int, char **) +{ + std::ostream* out = &std::cout; + + { + Person person; + person.set_uuid("a7522158-3d41-4b77-ad69-6c598ee55c49"); + person.set_name("Ivan"); + person.set_surname("Petrov"); + person.set_gender(Gender::male); + person.set_birthdate(4015); // 1980-12-29 + person.set_photo("png"); + person.set_phonenumber(std::string("+74951234567\0", 13)); // Converted from FixedString(13) + person.set_isonline(true); + person.set_visittime(1546703100); // 2019-01-05 18:45:00 + person.set_age(38); + person.set_zodiacsign(ZodiacSign::capricorn); + person.add_songs("Yesterday"); + person.add_songs("Flowers"); + person.add_color(255); + person.add_color(0); + person.add_color(0); + person.set_hometown("Moscow"); + person.add_location(55.753215); + person.add_location(37.622504); + person.set_pi(3.14); + person.set_lotterywin(214.10); + person.set_someratio(0.1); + person.set_temperature(5.8); + person.set_randombignumber(17060000000); + google::protobuf::util::SerializeDelimitedToOstream(person, out); + } + + { + Person person; + person.set_uuid("c694ad8a-f714-4ea3-907d-fd54fb25d9b5"); + person.set_name("Natalia"); + person.set_surname("Sokolova"); + person.set_gender(Gender::female); + person.set_birthdate(8102); // 1992-03-08 + person.set_photo("jpg"); + person.set_isonline(false); + person.set_age(26); + person.set_zodiacsign(ZodiacSign::pisces); + person.add_color(100); + person.add_color(200); + person.add_color(50); + person.set_hometown("Tver"); + person.add_location(54.782635); + person.add_location(32.045251); + person.set_pi(3.14159); + person.set_someratio(0.007); + person.set_temperature(5.4); + person.set_randombignumber(-20000000000000); + google::protobuf::util::SerializeDelimitedToOstream(person, out); + } + + { + Person person; + person.set_uuid("a7da1aa6-f425-4789-8947-b034786ed374"); + person.set_name("Vasily"); + person.set_surname("Sidorov"); + person.set_gender(Gender::male); + person.set_birthdate(9339); // 1995-07-28 + person.set_photo("bmp"); + person.set_phonenumber("+442012345678"); + person.set_isonline(true); + person.set_visittime(1546117200); // 2018-12-30 00:00:00 + person.set_age(23); + person.set_zodiacsign(ZodiacSign::leo); + person.add_songs("Sunny"); + person.add_color(250); + person.add_color(244); + person.add_color(10); + person.set_hometown("Murmansk"); + person.add_location(68.970682); + person.add_location(33.074981); + person.set_pi(3.14159265358979); + person.set_lotterywin(100000000000); + person.set_someratio(800); + person.set_temperature(-3.2); + person.set_randombignumber(154400000); + google::protobuf::util::SerializeDelimitedToOstream(person, out); + } + + *out << "ALTERNATIVE->" << std::endl; + + { + AltPerson person; + person.add_location(55); + person.add_location(37); + person.set_pi(3.14); + person.set_uuid("a7522158-3d41-4b77-ad69-6c598ee55c49"); + person.set_name("Ivan"); + person.set_gender(AltPerson::male); + person.set_zodiacsign(1222); // capricorn + person.set_birthdate(4015); // 1980-12-29 + person.set_age("38"); + person.set_isonline(OnlineStatus::online); + person.set_someratio(0.100000001490116119384765625); // 0.1 converted from float to double + person.set_visittime(1546703100); // 2019-01-05 18:45:00 + person.set_randombignumber(17060000000); + person.add_color(255); + person.add_color(0); + person.add_color(0); + person.set_lotterywin(214); + person.set_surname("Petrov"); + person.set_phonenumber(+74951234567); + person.set_temperature(5); + google::protobuf::util::SerializeDelimitedToOstream(person, out); + } + + { + AltPerson person; + person.add_location(54); + person.add_location(32); + person.set_pi(3.14159); + person.set_uuid("c694ad8a-f714-4ea3-907d-fd54fb25d9b5"); + person.set_name("Natalia"); + person.set_gender(AltPerson::female); + person.set_zodiacsign(219); // pisces + person.set_birthdate(8102); // 1992-03-08 + person.set_age("26"); + person.set_isonline(OnlineStatus::offline); + person.set_someratio(0.007000000216066837310791015625); // 0.007 converted from float to double + person.set_randombignumber(-20000000000000); + person.add_color(100); + person.add_color(200); + person.add_color(50); + person.set_surname("Sokolova"); + person.set_temperature(5); + google::protobuf::util::SerializeDelimitedToOstream(person, out); + } + + { + AltPerson person; + person.add_location(68); + person.add_location(33); + person.set_pi(3.1415926535897); + person.set_uuid("a7da1aa6-f425-4789-8947-b034786ed374"); + person.set_name("Vasily"); + person.set_gender(AltPerson::male); + person.set_zodiacsign(723); // leo + person.set_birthdate(9339); // 1995-07-28 + person.set_age("23"); + person.set_isonline(OnlineStatus::online); + person.set_someratio(800); + person.set_visittime(1546117200); // 2018-12-30 00:00:00 + person.set_randombignumber(154400000); + person.add_color(250); + person.add_color(244); + person.add_color(10); + person.set_lotterywin(100000000000); + person.set_surname("Sidorov"); + person.set_phonenumber(+442012345678); + person.set_temperature(-3); + google::protobuf::util::SerializeDelimitedToOstream(person, out); + } + + *out << "STRINGS->" << std::endl; + + { + StrPerson person; + person.set_uuid("a7522158-3d41-4b77-ad69-6c598ee55c49"); + person.set_name("Ivan"); + person.set_surname("Petrov"); + person.set_gender("male"); + person.set_birthdate("1980-12-29"); + person.set_phonenumber(std::string("+74951234567\0", 13)); // Converted from FixedString(13) + person.set_isonline("1"); + person.set_visittime("2019-01-05 18:45:00"); + person.set_age("38"); + person.set_zodiacsign("capricorn"); + person.add_songs("Yesterday"); + person.add_songs("Flowers"); + person.add_color("255"); + person.add_color("0"); + person.add_color("0"); + person.set_hometown("Moscow"); + person.add_location("55.753215"); + person.add_location("37.622504"); + person.set_pi("3.14"); + person.set_lotterywin("214.10"); + person.set_someratio("0.1"); + person.set_temperature("5.8"); + person.set_randombignumber("17060000000"); + google::protobuf::util::SerializeDelimitedToOstream(person, out); + } + + { + StrPerson person; + person.set_uuid("c694ad8a-f714-4ea3-907d-fd54fb25d9b5"); + person.set_name("Natalia"); + person.set_surname("Sokolova"); + person.set_gender("female"); + person.set_birthdate("1992-03-08"); + person.set_isonline("0"); + person.set_age("26"); + person.set_zodiacsign("pisces"); + person.add_color("100"); + person.add_color("200"); + person.add_color("50"); + person.set_hometown("Tver"); + person.add_location("54.782635"); + person.add_location("32.045251"); + person.set_pi("3.14159"); + person.set_someratio("0.007"); + person.set_temperature("5.4"); + person.set_randombignumber("-20000000000000"); + google::protobuf::util::SerializeDelimitedToOstream(person, out); + } + + { + StrPerson person; + person.set_uuid("a7da1aa6-f425-4789-8947-b034786ed374"); + person.set_name("Vasily"); + person.set_surname("Sidorov"); + person.set_gender("male"); + person.set_birthdate("1995-07-28"); + person.set_phonenumber("+442012345678"); + person.set_isonline("1"); + person.set_visittime("2018-12-30 00:00:00"); + person.set_age("23"); + person.set_zodiacsign("leo"); + person.add_songs("Sunny"); + person.add_color("250"); + person.add_color("244"); + person.add_color("10"); + person.set_hometown("Murmansk"); + person.add_location("68.970682"); + person.add_location("33.074981"); + person.set_pi("3.14159265358979"); + person.set_lotterywin("100000000000.00"); + person.set_someratio("800"); + person.set_temperature("-3.2"); + person.set_randombignumber("154400000"); + google::protobuf::util::SerializeDelimitedToOstream(person, out); + } + + *out << "SYNTAX2->" << std::endl; + + { + Syntax2Person person; + person.set_uuid("a7522158-3d41-4b77-ad69-6c598ee55c49"); + person.set_name("Ivan"); + person.set_surname("Petrov"); + person.set_gender(Syntax2Person::male); + person.set_birthdate(4015); // 1980-12-29 + person.set_photo("png"); + person.set_phonenumber(std::string("+74951234567\0", 13)); // Converted from FixedString(13) + person.set_isonline(true); + person.set_visittime(1546703100); // 2019-01-05 18:45:00 + person.set_age(38); + person.set_zodiacsign(Syntax2Person::capricorn); + person.add_songs("Yesterday"); + person.add_songs("Flowers"); + person.add_color(255); + person.add_color(0); + person.add_color(0); + person.set_hometown("Moscow"); + person.add_location(55.753215); + person.add_location(37.622504); + person.set_pi(3.14); + person.set_lotterywin(214.10); + person.set_someratio(0.1); + person.set_temperature(5.8); + person.set_randombignumber(17060000000); + google::protobuf::util::SerializeDelimitedToOstream(person, out); + } + + { + Syntax2Person person; + person.set_uuid("c694ad8a-f714-4ea3-907d-fd54fb25d9b5"); + person.set_name("Natalia"); + person.set_surname("Sokolova"); + person.set_gender(Syntax2Person::female); + person.set_birthdate(8102); // 1992-03-08 + person.set_photo("jpg"); + person.set_isonline(false); + person.set_age(26); + person.set_zodiacsign(Syntax2Person::pisces); + person.add_color(100); + person.add_color(200); + person.add_color(50); + person.set_hometown("Tver"); + person.add_location(54.782635); + person.add_location(32.045251); + person.set_pi(3.14159); + person.set_someratio(0.007); + person.set_temperature(5.4); + person.set_randombignumber(-20000000000000); + google::protobuf::util::SerializeDelimitedToOstream(person, out); + } + + { + Syntax2Person person; + person.set_uuid("a7da1aa6-f425-4789-8947-b034786ed374"); + person.set_name("Vasily"); + person.set_surname("Sidorov"); + person.set_gender(Syntax2Person::male); + person.set_birthdate(9339); // 1995-07-28 + person.set_photo("bmp"); + person.set_phonenumber("+442012345678"); + person.set_isonline(true); + person.set_visittime(1546117200); // 2018-12-30 00:00:00 + person.set_age(23); + person.set_zodiacsign(Syntax2Person::leo); + person.add_songs("Sunny"); + person.add_color(250); + person.add_color(244); + person.add_color(10); + person.set_hometown("Murmansk"); + person.add_location(68.970682); + person.add_location(33.074981); + person.set_pi(3.14159265358979); + person.set_lotterywin(100000000000); + person.set_someratio(800); + person.set_temperature(-3.2); + person.set_randombignumber(154400000); + google::protobuf::util::SerializeDelimitedToOstream(person, out); + } + + return 0; +}