Added tests for writing protobufs.

This commit is contained in:
Vitaly Baranov 2019-01-23 22:43:23 +03:00
parent 2886f42c0e
commit 2e383bfdeb
6 changed files with 543 additions and 0 deletions

View File

@ -0,0 +1,98 @@
syntax = "proto3";
enum Gender {
female = 0;
male = 1;
};
enum ZodiacSign {
aries = 0;
taurus = 1;
gemini = 2;
cancer = 3;
leo = 4;
virgo = 5;
libra = 6;
scorpius = 7;
sagittarius = 8;
capricorn = 9;
aquarius = 10;
pisces = 11;
};
message Person {
string uuid = 1;
string name = 2;
string surname = 3;
Gender gender = 4;
uint32 birthDate = 5;
bytes photo = 6;
string phoneNumber = 7;
bool isOnline = 8;
fixed32 visitTime = 9;
uint32 age = 10;
ZodiacSign zodiacSign = 11;
repeated string songs = 12;
repeated uint32 color = 13;
string hometown = 14;
repeated float location = 15;
double pi = 16;
double lotteryWin = 17;
float someRatio = 18;
float temperature = 19;
sint64 randomBigNumber = 20;
};
enum OnlineStatus {
offline = 0;
online = 1;
};
message AltPerson {
enum Gender {
male = 0;
female = 1;
};
repeated int32 location = 101 [packed=false];
float pi = 103;
bytes uuid = 300;
bool newFieldBool = 299;
string name = 2;
Gender gender = 102;
int32 zodiacSign = 130;
int64 birthDate = 150;
bytes age = 111;
OnlineStatus isOnline = 1;
double someRatio = 100;
fixed64 visitTime = 15;
sfixed64 randomBigNumber = 140;
repeated int32 newFieldInt = 104;
repeated float color = 14;
uint64 lotteryWin = 202;
bytes surname = 10;
uint64 phoneNumber = 5;
sint32 temperature = 41;
string newFieldStr = 21;
};
message StrPerson {
string uuid = 1;
string name = 2;
string surname = 3;
string gender = 4;
string birthDate = 5;
string phoneNumber = 7;
string isOnline = 8;
string visitTime = 9;
string age = 10;
string zodiacSign = 11;
repeated string songs = 12;
repeated string color = 13;
string hometown = 14;
repeated string location = 15;
string pi = 16;
string lotteryWin = 17;
string someRatio = 18;
string temperature = 19;
string randomBigNumber = 20;
};

View File

@ -0,0 +1,60 @@
#!/usr/bin/env bash
# To generate reference file for this test use the following commands:
# ninja ProtobufDelimitedMessagesSerializer
# build/utils/test-data-generator/ProtobufDelimitedMessagesSerializer > dbms/tests/queries/0_stateless/00825_protobuf_format_output.reference
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. $CURDIR/../shell_config.sh
set -e -o pipefail
# Copy schema files to the current directory because the client can open schemas from the current directory only.
cp "$CURDIR/00825_protobuf_format.proto" 00825_protobuf_format_copy.proto
cp "$CURDIR/00825_protobuf_format_syntax2.proto" 00825_protobuf_format_syntax2_copy.proto
# Run the client.
$CLICKHOUSE_CLIENT --multiquery <<'EOF'
SET allow_experimental_low_cardinality_type = 1;
CREATE DATABASE IF NOT EXISTS test;
DROP TABLE IF EXISTS test.table;
CREATE TABLE test.table (uuid UUID,
name String,
surname String,
gender Enum8('male'=1, 'female'=0),
birthDate Date,
photo Nullable(String),
phoneNumber Nullable(FixedString(13)),
isOnline UInt8,
visitTime Nullable(DateTime),
age UInt8,
zodiacSign Enum16('aries'=321, 'taurus'=420, 'gemini'=521, 'cancer'=621, 'leo'=723, 'virgo'=823,
'libra'=923, 'scorpius'=1023, 'sagittarius'=1122, 'capricorn'=1222, 'aquarius'=120,
'pisces'=219),
songs Array(String),
color Array(UInt8),
hometown LowCardinality(String),
location Array(Decimal32(6)),
pi Nullable(Float64),
lotteryWin Nullable(Decimal64(2)),
someRatio Float32,
temperature Decimal32(1),
randomBigNumber Int64
) ENGINE = MergeTree ORDER BY tuple();
INSERT INTO test.table VALUES (toUUID('a7522158-3d41-4b77-ad69-6c598ee55c49'), 'Ivan', 'Petrov', 'male', toDate('1980-12-29'), 'png', '+74951234567', 1, toDateTime('2019-01-05 18:45:00'), 38, 'capricorn', ['Yesterday', 'Flowers'], [255, 0, 0], 'Moscow', [55.753215, 37.622504], 3.14, 214.10, 0.1, 5.8, 17060000000);
INSERT INTO test.table VALUES (toUUID('c694ad8a-f714-4ea3-907d-fd54fb25d9b5'), 'Natalia', 'Sokolova', 'female', toDate('1992-03-08'), 'jpg', NULL, 0, NULL, 26, 'pisces', [], [100, 200, 50], 'Tver', [54.782635, 32.045251], 3.14159, NULL, 0.007, 5.4, -20000000000000);
INSERT INTO test.table VALUES (toUUID('a7da1aa6-f425-4789-8947-b034786ed374'), 'Vasily', 'Sidorov', 'male', toDate('1995-07-28'), 'bmp', '+442012345678', 1, toDateTime('2018-12-30 00:00:00'), 23, 'leo', ['Sunny'], [250, 244, 10], 'Murmansk', [68.970682, 33.074981], 3.14159265358979, 100000000000, 800, -3.2, 154400000);
SELECT * FROM test.table ORDER BY name FORMAT Protobuf SETTINGS format_schema = '00825_protobuf_format_copy:Person';
SELECT 'ALTERNATIVE->';
SELECT * FROM test.table ORDER BY name FORMAT Protobuf SETTINGS format_schema = '00825_protobuf_format_copy:AltPerson';
SELECT 'STRINGS->';
SELECT * FROM test.table ORDER BY name FORMAT Protobuf SETTINGS format_schema = '00825_protobuf_format_copy:StrPerson';
SELECT 'SYNTAX2->';
SELECT * FROM test.table ORDER BY name FORMAT Protobuf SETTINGS format_schema = '00825_protobuf_format_syntax2_copy:Syntax2Person';
EOF
# Remove copies of the schema files.
rm "00825_protobuf_format_copy.proto" "00825_protobuf_format_syntax2_copy.proto"

View File

@ -0,0 +1,47 @@
syntax = "proto2";
message Syntax2Person {
enum Gender {
female = 0;
male = 1;
};
enum ZodiacSign {
aries = 0;
taurus = 1;
gemini = 2;
cancer = 3;
leo = 4;
virgo = 5;
libra = 6;
scorpius = 7;
sagittarius = 8;
capricorn = 9;
aquarius = 10;
pisces = 11;
};
required string uuid = 1;
required string name = 2;
required string surname = 3;
required Gender gender = 4;
required uint32 birthDate = 5;
optional bytes photo = 6;
optional string phoneNumber = 7;
optional bool isOnline = 8;
optional fixed32 visitTime = 9;
optional uint32 age = 10;
optional ZodiacSign zodiacSign = 11;
repeated string songs = 12;
repeated uint32 color = 13;
optional string hometown = 14 [default='Moscow'];
repeated float location = 15 [packed=true];
optional double pi = 16;
optional double lotteryWin = 17;
optional float someRatio = 18;
optional float temperature = 19;
optional sint64 randomBigNumber = 20;
optional string newFieldStr = 21 [default='abc'];
optional int32 newFieldInt = 22 [default=-11];
optional bool newBool = 23 [default=true];
};

View File

@ -3,3 +3,9 @@ target_link_libraries(test-data-generator PRIVATE clickhouse_common_io ${Boost_P
add_executable (markov-model markov-model.cpp)
target_link_libraries(markov-model PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
protobuf_generate_cpp(ProtobufDelimitedMessagesSerializer_Srcs ProtobufDelimitedMessagesSerializer_Hdrs ${CMAKE_CURRENT_SOURCE_DIR}/../../dbms/tests/queries/0_stateless/00825_protobuf_format.proto)
protobuf_generate_cpp(ProtobufDelimitedMessagesSerializer_Srcs2 ProtobufDelimitedMessagesSerializer_Hdrs2 ${CMAKE_CURRENT_SOURCE_DIR}/../../dbms/tests/queries/0_stateless/00825_protobuf_format_syntax2.proto)
add_executable (ProtobufDelimitedMessagesSerializer ProtobufDelimitedMessagesSerializer.cpp ${ProtobufDelimitedMessagesSerializer_Srcs} ${ProtobufDelimitedMessagesSerializer_Hdrs} ${ProtobufDelimitedMessagesSerializer_Srcs2} ${ProtobufDelimitedMessagesSerializer_Hdrs2})
target_include_directories (ProtobufDelimitedMessagesSerializer SYSTEM BEFORE PRIVATE ${Protobuf_INCLUDE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
target_link_libraries (ProtobufDelimitedMessagesSerializer PRIVATE ${Protobuf_LIBRARY})

View File

@ -0,0 +1,332 @@
// Reference file generator for the test dbms/tests/queries/0_stateless/00825_protobuf_format_output.sh
#include <iostream>
#include <google/protobuf/util/delimited_message_util.h>
#include "00825_protobuf_format.pb.h"
#include "00825_protobuf_format_syntax2.pb.h"
int main(int, char **)
{
std::ostream* out = &std::cout;
{
Person person;
person.set_uuid("a7522158-3d41-4b77-ad69-6c598ee55c49");
person.set_name("Ivan");
person.set_surname("Petrov");
person.set_gender(Gender::male);
person.set_birthdate(4015); // 1980-12-29
person.set_photo("png");
person.set_phonenumber(std::string("+74951234567\0", 13)); // Converted from FixedString(13)
person.set_isonline(true);
person.set_visittime(1546703100); // 2019-01-05 18:45:00
person.set_age(38);
person.set_zodiacsign(ZodiacSign::capricorn);
person.add_songs("Yesterday");
person.add_songs("Flowers");
person.add_color(255);
person.add_color(0);
person.add_color(0);
person.set_hometown("Moscow");
person.add_location(55.753215);
person.add_location(37.622504);
person.set_pi(3.14);
person.set_lotterywin(214.10);
person.set_someratio(0.1);
person.set_temperature(5.8);
person.set_randombignumber(17060000000);
google::protobuf::util::SerializeDelimitedToOstream(person, out);
}
{
Person person;
person.set_uuid("c694ad8a-f714-4ea3-907d-fd54fb25d9b5");
person.set_name("Natalia");
person.set_surname("Sokolova");
person.set_gender(Gender::female);
person.set_birthdate(8102); // 1992-03-08
person.set_photo("jpg");
person.set_isonline(false);
person.set_age(26);
person.set_zodiacsign(ZodiacSign::pisces);
person.add_color(100);
person.add_color(200);
person.add_color(50);
person.set_hometown("Tver");
person.add_location(54.782635);
person.add_location(32.045251);
person.set_pi(3.14159);
person.set_someratio(0.007);
person.set_temperature(5.4);
person.set_randombignumber(-20000000000000);
google::protobuf::util::SerializeDelimitedToOstream(person, out);
}
{
Person person;
person.set_uuid("a7da1aa6-f425-4789-8947-b034786ed374");
person.set_name("Vasily");
person.set_surname("Sidorov");
person.set_gender(Gender::male);
person.set_birthdate(9339); // 1995-07-28
person.set_photo("bmp");
person.set_phonenumber("+442012345678");
person.set_isonline(true);
person.set_visittime(1546117200); // 2018-12-30 00:00:00
person.set_age(23);
person.set_zodiacsign(ZodiacSign::leo);
person.add_songs("Sunny");
person.add_color(250);
person.add_color(244);
person.add_color(10);
person.set_hometown("Murmansk");
person.add_location(68.970682);
person.add_location(33.074981);
person.set_pi(3.14159265358979);
person.set_lotterywin(100000000000);
person.set_someratio(800);
person.set_temperature(-3.2);
person.set_randombignumber(154400000);
google::protobuf::util::SerializeDelimitedToOstream(person, out);
}
*out << "ALTERNATIVE->" << std::endl;
{
AltPerson person;
person.add_location(55);
person.add_location(37);
person.set_pi(3.14);
person.set_uuid("a7522158-3d41-4b77-ad69-6c598ee55c49");
person.set_name("Ivan");
person.set_gender(AltPerson::male);
person.set_zodiacsign(1222); // capricorn
person.set_birthdate(4015); // 1980-12-29
person.set_age("38");
person.set_isonline(OnlineStatus::online);
person.set_someratio(0.100000001490116119384765625); // 0.1 converted from float to double
person.set_visittime(1546703100); // 2019-01-05 18:45:00
person.set_randombignumber(17060000000);
person.add_color(255);
person.add_color(0);
person.add_color(0);
person.set_lotterywin(214);
person.set_surname("Petrov");
person.set_phonenumber(+74951234567);
person.set_temperature(5);
google::protobuf::util::SerializeDelimitedToOstream(person, out);
}
{
AltPerson person;
person.add_location(54);
person.add_location(32);
person.set_pi(3.14159);
person.set_uuid("c694ad8a-f714-4ea3-907d-fd54fb25d9b5");
person.set_name("Natalia");
person.set_gender(AltPerson::female);
person.set_zodiacsign(219); // pisces
person.set_birthdate(8102); // 1992-03-08
person.set_age("26");
person.set_isonline(OnlineStatus::offline);
person.set_someratio(0.007000000216066837310791015625); // 0.007 converted from float to double
person.set_randombignumber(-20000000000000);
person.add_color(100);
person.add_color(200);
person.add_color(50);
person.set_surname("Sokolova");
person.set_temperature(5);
google::protobuf::util::SerializeDelimitedToOstream(person, out);
}
{
AltPerson person;
person.add_location(68);
person.add_location(33);
person.set_pi(3.1415926535897);
person.set_uuid("a7da1aa6-f425-4789-8947-b034786ed374");
person.set_name("Vasily");
person.set_gender(AltPerson::male);
person.set_zodiacsign(723); // leo
person.set_birthdate(9339); // 1995-07-28
person.set_age("23");
person.set_isonline(OnlineStatus::online);
person.set_someratio(800);
person.set_visittime(1546117200); // 2018-12-30 00:00:00
person.set_randombignumber(154400000);
person.add_color(250);
person.add_color(244);
person.add_color(10);
person.set_lotterywin(100000000000);
person.set_surname("Sidorov");
person.set_phonenumber(+442012345678);
person.set_temperature(-3);
google::protobuf::util::SerializeDelimitedToOstream(person, out);
}
*out << "STRINGS->" << std::endl;
{
StrPerson person;
person.set_uuid("a7522158-3d41-4b77-ad69-6c598ee55c49");
person.set_name("Ivan");
person.set_surname("Petrov");
person.set_gender("male");
person.set_birthdate("1980-12-29");
person.set_phonenumber(std::string("+74951234567\0", 13)); // Converted from FixedString(13)
person.set_isonline("1");
person.set_visittime("2019-01-05 18:45:00");
person.set_age("38");
person.set_zodiacsign("capricorn");
person.add_songs("Yesterday");
person.add_songs("Flowers");
person.add_color("255");
person.add_color("0");
person.add_color("0");
person.set_hometown("Moscow");
person.add_location("55.753215");
person.add_location("37.622504");
person.set_pi("3.14");
person.set_lotterywin("214.10");
person.set_someratio("0.1");
person.set_temperature("5.8");
person.set_randombignumber("17060000000");
google::protobuf::util::SerializeDelimitedToOstream(person, out);
}
{
StrPerson person;
person.set_uuid("c694ad8a-f714-4ea3-907d-fd54fb25d9b5");
person.set_name("Natalia");
person.set_surname("Sokolova");
person.set_gender("female");
person.set_birthdate("1992-03-08");
person.set_isonline("0");
person.set_age("26");
person.set_zodiacsign("pisces");
person.add_color("100");
person.add_color("200");
person.add_color("50");
person.set_hometown("Tver");
person.add_location("54.782635");
person.add_location("32.045251");
person.set_pi("3.14159");
person.set_someratio("0.007");
person.set_temperature("5.4");
person.set_randombignumber("-20000000000000");
google::protobuf::util::SerializeDelimitedToOstream(person, out);
}
{
StrPerson person;
person.set_uuid("a7da1aa6-f425-4789-8947-b034786ed374");
person.set_name("Vasily");
person.set_surname("Sidorov");
person.set_gender("male");
person.set_birthdate("1995-07-28");
person.set_phonenumber("+442012345678");
person.set_isonline("1");
person.set_visittime("2018-12-30 00:00:00");
person.set_age("23");
person.set_zodiacsign("leo");
person.add_songs("Sunny");
person.add_color("250");
person.add_color("244");
person.add_color("10");
person.set_hometown("Murmansk");
person.add_location("68.970682");
person.add_location("33.074981");
person.set_pi("3.14159265358979");
person.set_lotterywin("100000000000.00");
person.set_someratio("800");
person.set_temperature("-3.2");
person.set_randombignumber("154400000");
google::protobuf::util::SerializeDelimitedToOstream(person, out);
}
*out << "SYNTAX2->" << std::endl;
{
Syntax2Person person;
person.set_uuid("a7522158-3d41-4b77-ad69-6c598ee55c49");
person.set_name("Ivan");
person.set_surname("Petrov");
person.set_gender(Syntax2Person::male);
person.set_birthdate(4015); // 1980-12-29
person.set_photo("png");
person.set_phonenumber(std::string("+74951234567\0", 13)); // Converted from FixedString(13)
person.set_isonline(true);
person.set_visittime(1546703100); // 2019-01-05 18:45:00
person.set_age(38);
person.set_zodiacsign(Syntax2Person::capricorn);
person.add_songs("Yesterday");
person.add_songs("Flowers");
person.add_color(255);
person.add_color(0);
person.add_color(0);
person.set_hometown("Moscow");
person.add_location(55.753215);
person.add_location(37.622504);
person.set_pi(3.14);
person.set_lotterywin(214.10);
person.set_someratio(0.1);
person.set_temperature(5.8);
person.set_randombignumber(17060000000);
google::protobuf::util::SerializeDelimitedToOstream(person, out);
}
{
Syntax2Person person;
person.set_uuid("c694ad8a-f714-4ea3-907d-fd54fb25d9b5");
person.set_name("Natalia");
person.set_surname("Sokolova");
person.set_gender(Syntax2Person::female);
person.set_birthdate(8102); // 1992-03-08
person.set_photo("jpg");
person.set_isonline(false);
person.set_age(26);
person.set_zodiacsign(Syntax2Person::pisces);
person.add_color(100);
person.add_color(200);
person.add_color(50);
person.set_hometown("Tver");
person.add_location(54.782635);
person.add_location(32.045251);
person.set_pi(3.14159);
person.set_someratio(0.007);
person.set_temperature(5.4);
person.set_randombignumber(-20000000000000);
google::protobuf::util::SerializeDelimitedToOstream(person, out);
}
{
Syntax2Person person;
person.set_uuid("a7da1aa6-f425-4789-8947-b034786ed374");
person.set_name("Vasily");
person.set_surname("Sidorov");
person.set_gender(Syntax2Person::male);
person.set_birthdate(9339); // 1995-07-28
person.set_photo("bmp");
person.set_phonenumber("+442012345678");
person.set_isonline(true);
person.set_visittime(1546117200); // 2018-12-30 00:00:00
person.set_age(23);
person.set_zodiacsign(Syntax2Person::leo);
person.add_songs("Sunny");
person.add_color(250);
person.add_color(244);
person.add_color(10);
person.set_hometown("Murmansk");
person.add_location(68.970682);
person.add_location(33.074981);
person.set_pi(3.14159265358979);
person.set_lotterywin(100000000000);
person.set_someratio(800);
person.set_temperature(-3.2);
person.set_randombignumber(154400000);
google::protobuf::util::SerializeDelimitedToOstream(person, out);
}
return 0;
}