diff --git a/tests/queries/0_stateless/00825_protobuf_format_persons.sh b/tests/queries/0_stateless/00825_protobuf_format_persons.sh index bb376e6ed70..465b27aa683 100755 --- a/tests/queries/0_stateless/00825_protobuf_format_persons.sh +++ b/tests/queries/0_stateless/00825_protobuf_format_persons.sh @@ -1,6 +1,10 @@ #!/usr/bin/env bash # Tags: no-fasttest +# End-to-end test of serialization/deserialization of a table with different +# data types to/from Protobuf format. +# Cf. 02240_protobuflist_format_persons.sh + # To generate reference file for this test use the following commands: # ninja ProtobufDelimitedMessagesSerializer # build/utils/test-data-generator/ProtobufDelimitedMessagesSerializer diff --git a/tests/queries/0_stateless/02187_async_inserts_all_formats.python b/tests/queries/0_stateless/02187_async_inserts_all_formats.python index 0a909451259..65a323ef9db 100644 --- a/tests/queries/0_stateless/02187_async_inserts_all_formats.python +++ b/tests/queries/0_stateless/02187_async_inserts_all_formats.python @@ -28,7 +28,7 @@ def run_test(data_format, gen_data_template, settings): exit(1) formats = client.query("SELECT name FROM system.formats WHERE is_input AND is_output \ - AND name NOT IN ('CapnProto', 'RawBLOB', 'Template', 'ProtobufSingle', 'LineAsString', 'Protobuf') ORDER BY name").strip().split('\n') + AND name NOT IN ('CapnProto', 'RawBLOB', 'Template', 'ProtobufSingle', 'LineAsString', 'Protobuf', 'ProtobufList') ORDER BY name").strip().split('\n') # Generic formats client.query("DROP TABLE IF EXISTS t_async_insert") diff --git a/tests/queries/0_stateless/02240_protobuflist_format_persons.reference b/tests/queries/0_stateless/02240_protobuflist_format_persons.reference new file mode 100644 index 00000000000..3ff4a6e57df --- /dev/null +++ b/tests/queries/0_stateless/02240_protobuflist_format_persons.reference @@ -0,0 +1,569 @@ +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753215,37.622504] 3.14 214.1 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 jpg \N 0 \N 26 pisces [] [100,200,50] Plymouth [50.403724,-4.142123] 3.14159 \N 0.007 5.4 -20000000000000 [] [] \N [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 bmp +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.970682,33.074981] 3.14159265358979 100000000000 800 -3.2 154400000 ['pound'] [16] 503 [] + +Schema 02240_protobuflist1_format_persons:Person + +Binary representation: +00000000 ba 04 0a f4 01 0a 24 61 37 35 32 32 31 35 38 2d |......$a7522158-| +00000010 33 64 34 31 2d 34 62 37 37 2d 61 64 36 39 2d 36 |3d41-4b77-ad69-6| +00000020 63 35 39 38 65 65 35 35 63 34 39 12 04 49 76 61 |c598ee55c49..Iva| +00000030 6e 1a 06 50 65 74 72 6f 76 20 01 28 af 1f 32 03 |n..Petrov .(..2.| +00000040 70 6e 67 3a 0d 2b 37 34 39 35 31 32 33 34 35 36 |png:.+7495123456| +00000050 37 00 40 01 4d fc d0 30 5c 50 26 58 09 62 09 59 |7.@.M..0\P&X.b.Y| +00000060 65 73 74 65 72 64 61 79 62 07 46 6c 6f 77 65 72 |esterdayb.Flower| +00000070 73 6a 04 ff 01 00 00 72 06 4d 6f 73 63 6f 77 7a |sj.....r.Moscowz| +00000080 08 4b 03 5f 42 72 7d 16 42 81 01 1f 85 eb 51 b8 |.K._Br}.B.....Q.| +00000090 1e 09 40 89 01 33 33 33 33 33 c3 6a 40 95 01 cd |..@..33333.j@...| +000000a0 cc cc 3d 9d 01 9a 99 b9 40 a0 01 80 c4 d7 8d 7f |..=.....@.......| +000000b0 aa 01 0c 0a 05 6d 65 74 65 72 15 00 00 80 3f aa |.....meter....?.| +000000c0 01 11 0a 0a 63 65 6e 74 69 6d 65 74 65 72 15 0a |....centimeter..| +000000d0 d7 23 3c aa 01 10 0a 09 6b 69 6c 6f 6d 65 74 65 |.#<.....kilomete| +000000e0 72 15 00 00 7a 44 b2 01 10 0a 0e a2 06 0b 0a 09 |r...zD..........| +000000f0 08 f4 03 12 04 f5 03 f6 03 0a 7e 0a 24 63 36 39 |..........~.$c69| +00000100 34 61 64 38 61 2d 66 37 31 34 2d 34 65 61 33 2d |4ad8a-f714-4ea3-| +00000110 39 30 37 64 2d 66 64 35 34 66 62 32 35 64 39 62 |907d-fd54fb25d9b| +00000120 35 12 07 4e 61 74 61 6c 69 61 1a 08 53 6f 6b 6f |5..Natalia..Soko| +00000130 6c 6f 76 61 28 a6 3f 32 03 6a 70 67 50 1a 58 0b |lova(.?2.jpgP.X.| +00000140 6a 04 64 c8 01 32 72 08 50 6c 79 6d 6f 75 74 68 |j.d..2r.Plymouth| +00000150 7a 08 6a 9d 49 42 46 8c 84 c0 81 01 6e 86 1b f0 |z.j.IBF.....n...| +00000160 f9 21 09 40 95 01 42 60 e5 3b 9d 01 cd cc ac 40 |.!.@..B`.;.....@| +00000170 a0 01 ff ff a9 ce 93 8c 09 0a c0 01 0a 24 61 37 |.............$a7| +00000180 64 61 31 61 61 36 2d 66 34 32 35 2d 34 37 38 39 |da1aa6-f425-4789| +00000190 2d 38 39 34 37 2d 62 30 33 34 37 38 36 65 64 33 |-8947-b034786ed3| +000001a0 37 34 12 06 56 61 73 69 6c 79 1a 07 53 69 64 6f |74..Vasily..Sido| +000001b0 72 6f 76 20 01 28 fb 48 32 03 62 6d 70 3a 0d 2b |rov .(.H2.bmp:.+| +000001c0 34 34 32 30 31 32 33 34 35 36 37 38 40 01 4d 50 |442012345678@.MP| +000001d0 e0 27 5c 50 17 58 04 62 05 53 75 6e 6e 79 6a 05 |.'\P.X.b.Sunnyj.| +000001e0 fa 01 f4 01 0a 72 08 4d 75 72 6d 61 6e 73 6b 7a |.....r.Murmanskz| +000001f0 08 fd f0 89 42 c8 4c 04 42 81 01 11 2d 44 54 fb |....B.L.B...-DT.| +00000200 21 09 40 89 01 00 00 00 e8 76 48 37 42 95 01 00 |!.@......vH7B...| +00000210 00 48 44 9d 01 cd cc 4c c0 a0 01 80 d4 9f 93 01 |.HD....L........| +00000220 aa 01 0c 0a 05 70 6f 75 6e 64 15 00 00 80 41 b2 |.....pound....A.| +00000230 01 0a 0a 08 a2 06 05 0a 03 08 f7 03 |............| +0000023c + +MESSAGE #1 AT 0x00000005 +uuid: "a7522158-3d41-4b77-ad69-6c598ee55c49" +name: "Ivan" +surname: "Petrov" +gender: male +birthDate: 4015 +photo: "png" +phoneNumber: "+74951234567\000" +isOnline: true +visitTime: 1546703100 +age: 38 +zodiacSign: capricorn +songs: "Yesterday" +songs: "Flowers" +color: 255 +color: 0 +color: 0 +hometown: "Moscow" +location: 55.7532158 +location: 37.6225052 +pi: 3.14 +lotteryWin: 214.1 +someRatio: 0.1 +temperature: 5.8 +randomBigNumber: 17060000000 +measureUnits { + unit: "meter" + coef: 1 +} +measureUnits { + unit: "centimeter" + coef: 0.01 +} +measureUnits { + unit: "kilometer" + coef: 1000 +} +nestiness { + a { + b { + c { + d: 500 + e: 501 + e: 502 + } + } + } +} +MESSAGE #2 AT 0x000000FB +uuid: "c694ad8a-f714-4ea3-907d-fd54fb25d9b5" +name: "Natalia" +surname: "Sokolova" +birthDate: 8102 +photo: "jpg" +age: 26 +zodiacSign: pisces +color: 100 +color: 200 +color: 50 +hometown: "Plymouth" +location: 50.4037247 +location: -4.14212322 +pi: 3.14159 +someRatio: 0.007 +temperature: 5.4 +randomBigNumber: -20000000000000 +MESSAGE #3 AT 0x0000017C +uuid: "a7da1aa6-f425-4789-8947-b034786ed374" +name: "Vasily" +surname: "Sidorov" +gender: male +birthDate: 9339 +photo: "bmp" +phoneNumber: "+442012345678" +isOnline: true +visitTime: 1546117200 +age: 23 +zodiacSign: leo +songs: "Sunny" +color: 250 +color: 244 +color: 10 +hometown: "Murmansk" +location: 68.9706802 +location: 33.0749817 +pi: 3.14159265358979 +lotteryWin: 100000000000 +someRatio: 800 +temperature: -3.2 +randomBigNumber: 154400000 +measureUnits { + unit: "pound" + coef: 16 +} +nestiness { + a { + b { + c { + d: 503 + } + } + } +} + +Binary representation is as expected + +Roundtrip: +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753216,37.622504] 3.14 214.1 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 jpg \N 0 \N 26 pisces [] [100,200,50] Plymouth [50.403724,-4.142123] 3.14159 \N 0.007 5.4 -20000000000000 [] [] \N [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 bmp +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.97068,33.074982] 3.14159265358979 100000000000 800 -3.2 154400000 ['pound'] [16] 503 [] + +Schema 02240_protobuflist2_format_persons:AltPerson + +Binary representation: +00000000 f4 03 0a c4 01 08 01 12 04 49 76 61 6e 28 87 a8 |.........Ivan(..| +00000010 c4 9b 97 02 52 06 50 65 74 72 6f 76 72 0c 00 00 |....R.Petrovr...| +00000020 7f 43 00 00 00 00 00 00 00 00 79 fc d0 30 5c 00 |.C........y..0\.| +00000030 00 00 00 c8 02 0a c2 05 0c 00 00 80 3f 0a d7 23 |............?..#| +00000040 3c 00 00 7a 44 9a 06 05 6d 65 74 65 72 9a 06 0a |<..zD...meter...| +00000050 63 65 6e 74 69 6d 65 74 65 72 9a 06 09 6b 69 6c |centimeter...kil| +00000060 6f 6d 65 74 65 72 a1 06 00 00 00 a0 99 99 b9 3f |ometer.........?| +00000070 a8 06 37 a8 06 25 bd 06 c3 f5 48 40 fa 06 02 33 |..7..%....H@...3| +00000080 38 90 08 c6 09 e1 08 00 f1 da f8 03 00 00 00 b0 |8...............| +00000090 09 af 1f d0 0c d6 01 e2 12 24 61 37 35 32 32 31 |.........$a75221| +000000a0 35 38 2d 33 64 34 31 2d 34 62 37 37 2d 61 64 36 |58-3d41-4b77-ad6| +000000b0 39 2d 36 63 35 39 38 65 65 35 35 63 34 39 a0 38 |9-6c598ee55c49.8| +000000c0 f4 03 aa 38 04 f5 03 f6 03 0a 84 01 12 07 4e 61 |...8..........Na| +000000d0 74 61 6c 69 61 52 08 53 6f 6b 6f 6c 6f 76 61 72 |taliaR.Sokolovar| +000000e0 0c 00 00 c8 42 00 00 48 43 00 00 48 42 c8 02 0a |....B..HC..HB...| +000000f0 a1 06 00 00 00 40 08 ac 7c 3f a8 06 32 a8 06 fc |.....@..|?..2...| +00000100 ff ff ff ff ff ff ff ff 01 b0 06 01 bd 06 d0 0f |................| +00000110 49 40 fa 06 02 32 36 90 08 db 01 e1 08 00 c0 1a |I@...26.........| +00000120 63 cf ed ff ff b0 09 a6 3f e2 12 24 63 36 39 34 |c.......?..$c694| +00000130 61 64 38 61 2d 66 37 31 34 2d 34 65 61 33 2d 39 |ad8a-f714-4ea3-9| +00000140 30 37 64 2d 66 64 35 34 66 62 32 35 64 39 62 35 |07d-fd54fb25d9b5| +00000150 0a a3 01 08 01 12 06 56 61 73 69 6c 79 28 ce ca |.......Vasily(..| +00000160 f4 cf ee 0c 52 07 53 69 64 6f 72 6f 76 72 0c 00 |....R.Sidorovr..| +00000170 00 7a 43 00 00 74 43 00 00 20 41 79 50 e0 27 5c |.zC..tC.. AyP.'\| +00000180 00 00 00 00 c8 02 05 c2 05 04 00 00 80 41 9a 06 |.............A..| +00000190 05 70 6f 75 6e 64 a1 06 00 00 00 00 00 00 89 40 |.pound.........@| +000001a0 a8 06 44 a8 06 21 bd 06 db 0f 49 40 fa 06 02 32 |..D..!....I@...2| +000001b0 33 90 08 d3 05 e1 08 00 f5 33 09 00 00 00 00 b0 |3........3......| +000001c0 09 fb 48 d0 0c 80 d0 db c3 f4 02 e2 12 24 61 37 |..H..........$a7| +000001d0 64 61 31 61 61 36 2d 66 34 32 35 2d 34 37 38 39 |da1aa6-f425-4789| +000001e0 2d 38 39 34 37 2d 62 30 33 34 37 38 36 65 64 33 |-8947-b034786ed3| +000001f0 37 34 a0 38 f7 03 |74.8..| +000001f6 + +MESSAGE #1 AT 0x00000005 +isOnline: online +name: "Ivan" +phoneNumber: 74951234567 +surname: "Petrov" +color: 255 +color: 0 +color: 0 +visitTime: 1546703100 +temperature: 5 +measureUnits_coef: 1 +measureUnits_coef: 0.01 +measureUnits_coef: 1000 +measureUnits_unit: "meter" +measureUnits_unit: "centimeter" +measureUnits_unit: "kilometer" +someRatio: 0.10000000149011612 +location: 55 +location: 37 +pi: 3.14 +age: "38" +zodiacSign: 1222 +randomBigNumber: 17060000000 +birthDate: 4015 +lotteryWin: 214 +uuid: "a7522158-3d41-4b77-ad69-6c598ee55c49" +nestiness_a_b_c_d: 500 +nestiness_a_b_c_e: 501 +nestiness_a_b_c_e: 502 +MESSAGE #2 AT 0x000000CC +name: "Natalia" +surname: "Sokolova" +color: 100 +color: 200 +color: 50 +temperature: 5 +someRatio: 0.0070000002160668373 +location: 50 +location: -4 +gender: female +pi: 3.14159 +age: "26" +zodiacSign: 219 +randomBigNumber: -20000000000000 +birthDate: 8102 +uuid: "c694ad8a-f714-4ea3-907d-fd54fb25d9b5" +MESSAGE #3 AT 0x00000153 +isOnline: online +name: "Vasily" +phoneNumber: 442012345678 +surname: "Sidorov" +color: 250 +color: 244 +color: 10 +visitTime: 1546117200 +temperature: -3 +measureUnits_coef: 16 +measureUnits_unit: "pound" +someRatio: 800 +location: 68 +location: 33 +pi: 3.14159274 +age: "23" +zodiacSign: 723 +randomBigNumber: 154400000 +birthDate: 9339 +lotteryWin: 100000000000 +uuid: "a7da1aa6-f425-4789-8947-b034786ed374" +nestiness_a_b_c_d: 503 + +Binary representation is as expected + +Roundtrip: +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 \N 74951234567\0\0 1 2019-01-05 18:45:00 38 capricorn [] [255,0,0] [55,37] 3.140000104904175 214 0.1 5 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 \N \N 0 \N 26 pisces [] [100,200,50] [50,-4] 3.141590118408203 \N 0.007 5 -20000000000000 [] [] \N [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 \N 442012345678\0 1 2018-12-30 00:00:00 23 leo [] [250,244,10] [68,33] 3.1415927410125732 100000000000 800 -3 154400000 ['pound'] [16] 503 [] + +Schema 02240_protobuflist3_format_persons:StrPerson as ProtobufList + +Binary representation: +00000000 e4 05 0a a6 02 0a 24 61 37 35 32 32 31 35 38 2d |......$a7522158-| +00000010 33 64 34 31 2d 34 62 37 37 2d 61 64 36 39 2d 36 |3d41-4b77-ad69-6| +00000020 63 35 39 38 65 65 35 35 63 34 39 12 04 49 76 61 |c598ee55c49..Iva| +00000030 6e 1a 06 50 65 74 72 6f 76 22 04 6d 61 6c 65 2a |n..Petrov".male*| +00000040 0a 31 39 38 30 2d 31 32 2d 32 39 3a 0d 2b 37 34 |.1980-12-29:.+74| +00000050 39 35 31 32 33 34 35 36 37 00 42 01 31 4a 13 32 |951234567.B.1J.2| +00000060 30 31 39 2d 30 31 2d 30 35 20 31 38 3a 34 35 3a |019-01-05 18:45:| +00000070 30 30 52 02 33 38 5a 09 63 61 70 72 69 63 6f 72 |00R.38Z.capricor| +00000080 6e 62 09 59 65 73 74 65 72 64 61 79 62 07 46 6c |nb.Yesterdayb.Fl| +00000090 6f 77 65 72 73 6a 03 32 35 35 6a 01 30 6a 01 30 |owersj.255j.0j.0| +000000a0 72 06 4d 6f 73 63 6f 77 7a 09 35 35 2e 37 35 33 |r.Moscowz.55.753| +000000b0 32 31 35 7a 09 33 37 2e 36 32 32 35 30 34 82 01 |215z.37.622504..| +000000c0 04 33 2e 31 34 8a 01 05 32 31 34 2e 31 92 01 03 |.3.14...214.1...| +000000d0 30 2e 31 9a 01 03 35 2e 38 a2 01 0b 31 37 30 36 |0.1...5.8...1706| +000000e0 30 30 30 30 30 30 30 aa 01 2d 0a 05 6d 65 74 65 |0000000..-..mete| +000000f0 72 0a 0a 63 65 6e 74 69 6d 65 74 65 72 0a 09 6b |r..centimeter..k| +00000100 69 6c 6f 6d 65 74 65 72 12 01 31 12 04 30 2e 30 |ilometer..1..0.0| +00000110 31 12 04 31 30 30 30 b2 01 11 0a 0f 0a 03 35 30 |1..1000.......50| +00000120 30 12 03 35 30 31 12 03 35 30 32 0a b4 01 0a 24 |0..501..502....$| +00000130 63 36 39 34 61 64 38 61 2d 66 37 31 34 2d 34 65 |c694ad8a-f714-4e| +00000140 61 33 2d 39 30 37 64 2d 66 64 35 34 66 62 32 35 |a3-907d-fd54fb25| +00000150 64 39 62 35 12 07 4e 61 74 61 6c 69 61 1a 08 53 |d9b5..Natalia..S| +00000160 6f 6b 6f 6c 6f 76 61 22 06 66 65 6d 61 6c 65 2a |okolova".female*| +00000170 0a 31 39 39 32 2d 30 33 2d 30 38 42 01 30 52 02 |.1992-03-08B.0R.| +00000180 32 36 5a 06 70 69 73 63 65 73 6a 03 31 30 30 6a |26Z.piscesj.100j| +00000190 03 32 30 30 6a 02 35 30 72 08 50 6c 79 6d 6f 75 |.200j.50r.Plymou| +000001a0 74 68 7a 09 35 30 2e 34 30 33 37 32 34 7a 09 2d |thz.50.403724z.-| +000001b0 34 2e 31 34 32 31 32 33 82 01 07 33 2e 31 34 31 |4.142123...3.141| +000001c0 35 39 92 01 05 30 2e 30 30 37 9a 01 03 35 2e 34 |59...0.007...5.4| +000001d0 a2 01 0f 2d 32 30 30 30 30 30 30 30 30 30 30 30 |...-200000000000| +000001e0 30 30 0a 81 02 0a 24 61 37 64 61 31 61 61 36 2d |00....$a7da1aa6-| +000001f0 66 34 32 35 2d 34 37 38 39 2d 38 39 34 37 2d 62 |f425-4789-8947-b| +00000200 30 33 34 37 38 36 65 64 33 37 34 12 06 56 61 73 |034786ed374..Vas| +00000210 69 6c 79 1a 07 53 69 64 6f 72 6f 76 22 04 6d 61 |ily..Sidorov".ma| +00000220 6c 65 2a 0a 31 39 39 35 2d 30 37 2d 32 38 3a 0d |le*.1995-07-28:.| +00000230 2b 34 34 32 30 31 32 33 34 35 36 37 38 42 01 31 |+442012345678B.1| +00000240 4a 13 32 30 31 38 2d 31 32 2d 33 30 20 30 30 3a |J.2018-12-30 00:| +00000250 30 30 3a 30 30 52 02 32 33 5a 03 6c 65 6f 62 05 |00:00R.23Z.leob.| +00000260 53 75 6e 6e 79 6a 03 32 35 30 6a 03 32 34 34 6a |Sunnyj.250j.244j| +00000270 02 31 30 72 08 4d 75 72 6d 61 6e 73 6b 7a 09 36 |.10r.Murmanskz.6| +00000280 38 2e 39 37 30 36 38 32 7a 09 33 33 2e 30 37 34 |8.970682z.33.074| +00000290 39 38 31 82 01 10 33 2e 31 34 31 35 39 32 36 35 |981...3.14159265| +000002a0 33 35 38 39 37 39 8a 01 0c 31 30 30 30 30 30 30 |358979...1000000| +000002b0 30 30 30 30 30 92 01 03 38 30 30 9a 01 04 2d 33 |00000...800...-3| +000002c0 2e 32 a2 01 09 31 35 34 34 30 30 30 30 30 aa 01 |.2...154400000..| +000002d0 0b 0a 05 70 6f 75 6e 64 12 02 31 36 b2 01 07 0a |...pound..16....| +000002e0 05 0a 03 35 30 33 |...503| +000002e6 + +MESSAGE #1 AT 0x00000005 +uuid: "a7522158-3d41-4b77-ad69-6c598ee55c49" +name: "Ivan" +surname: "Petrov" +gender: "male" +birthDate: "1980-12-29" +phoneNumber: "+74951234567\000" +isOnline: "1" +visitTime: "2019-01-05 18:45:00" +age: "38" +zodiacSign: "capricorn" +songs: "Yesterday" +songs: "Flowers" +color: "255" +color: "0" +color: "0" +hometown: "Moscow" +location: "55.753215" +location: "37.622504" +pi: "3.14" +lotteryWin: "214.1" +someRatio: "0.1" +temperature: "5.8" +randomBigNumber: "17060000000" +measureUnits { + unit: "meter" + unit: "centimeter" + unit: "kilometer" + coef: "1" + coef: "0.01" + coef: "1000" +} +nestiness_a { + b_c { + d: "500" + e: "501" + e: "502" + } +} +MESSAGE #2 AT 0x0000012E +uuid: "c694ad8a-f714-4ea3-907d-fd54fb25d9b5" +name: "Natalia" +surname: "Sokolova" +gender: "female" +birthDate: "1992-03-08" +isOnline: "0" +age: "26" +zodiacSign: "pisces" +color: "100" +color: "200" +color: "50" +hometown: "Plymouth" +location: "50.403724" +location: "-4.142123" +pi: "3.14159" +someRatio: "0.007" +temperature: "5.4" +randomBigNumber: "-20000000000000" +MESSAGE #3 AT 0x000001E5 +uuid: "a7da1aa6-f425-4789-8947-b034786ed374" +name: "Vasily" +surname: "Sidorov" +gender: "male" +birthDate: "1995-07-28" +phoneNumber: "+442012345678" +isOnline: "1" +visitTime: "2018-12-30 00:00:00" +age: "23" +zodiacSign: "leo" +songs: "Sunny" +color: "250" +color: "244" +color: "10" +hometown: "Murmansk" +location: "68.970682" +location: "33.074981" +pi: "3.14159265358979" +lotteryWin: "100000000000" +someRatio: "800" +temperature: "-3.2" +randomBigNumber: "154400000" +measureUnits { + unit: "pound" + coef: "16" +} +nestiness_a { + b_c { + d: "503" + } +} + +Binary representation is as expected +Roundtrip: +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 \N +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753215,37.622504] 3.14 214.1 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 \N \N 0 \N 26 pisces [] [100,200,50] Plymouth [50.403724,-4.142123] 3.14159 \N 0.007 5.4 -20000000000000 [] [] \N [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 \N +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.970682,33.074981] 3.14159265358979 100000000000 800 -3.2 154400000 ['pound'] [16] 503 [] + +Schema 02240_protobuf_format_syntax2:Syntax2Person + +Binary representation: +00000000 c0 04 0a f1 01 0a 24 61 37 35 32 32 31 35 38 2d |......$a7522158-| +00000010 33 64 34 31 2d 34 62 37 37 2d 61 64 36 39 2d 36 |3d41-4b77-ad69-6| +00000020 63 35 39 38 65 65 35 35 63 34 39 12 04 49 76 61 |c598ee55c49..Iva| +00000030 6e 1a 06 50 65 74 72 6f 76 20 01 28 af 1f 32 03 |n..Petrov .(..2.| +00000040 70 6e 67 3a 0d 2b 37 34 39 35 31 32 33 34 35 36 |png:.+7495123456| +00000050 37 00 40 01 4d fc d0 30 5c 50 26 58 09 62 09 59 |7.@.M..0\P&X.b.Y| +00000060 65 73 74 65 72 64 61 79 62 07 46 6c 6f 77 65 72 |esterdayb.Flower| +00000070 73 68 ff 01 68 00 68 00 72 06 4d 6f 73 63 6f 77 |sh..h.h.r.Moscow| +00000080 7a 08 4b 03 5f 42 72 7d 16 42 81 01 1f 85 eb 51 |z.K._Br}.B.....Q| +00000090 b8 1e 09 40 89 01 33 33 33 33 33 c3 6a 40 95 01 |...@..33333.j@..| +000000a0 cd cc cc 3d 9d 01 9a 99 b9 40 a0 01 80 c4 d7 8d |...=.....@......| +000000b0 7f ab 01 0d 00 00 80 3f 0d 0a d7 23 3c 0d 00 00 |.......?...#<...| +000000c0 7a 44 12 05 6d 65 74 65 72 12 0a 63 65 6e 74 69 |zD..meter..centi| +000000d0 6d 65 74 65 72 12 09 6b 69 6c 6f 6d 65 74 65 72 |meter..kilometer| +000000e0 ac 01 b3 01 0b a2 06 0b 0b 08 f4 03 10 f5 03 10 |................| +000000f0 f6 03 0c 0c b4 01 0a 83 01 0a 24 63 36 39 34 61 |..........$c694a| +00000100 64 38 61 2d 66 37 31 34 2d 34 65 61 33 2d 39 30 |d8a-f714-4ea3-90| +00000110 37 64 2d 66 64 35 34 66 62 32 35 64 39 62 35 12 |7d-fd54fb25d9b5.| +00000120 07 4e 61 74 61 6c 69 61 1a 08 53 6f 6b 6f 6c 6f |.Natalia..Sokolo| +00000130 76 61 20 00 28 a6 3f 32 03 6a 70 67 40 00 50 1a |va .(.?2.jpg@.P.| +00000140 58 0b 68 64 68 c8 01 68 32 72 08 50 6c 79 6d 6f |X.hdh..h2r.Plymo| +00000150 75 74 68 7a 08 6a 9d 49 42 46 8c 84 c0 81 01 6e |uthz.j.IBF.....n| +00000160 86 1b f0 f9 21 09 40 95 01 42 60 e5 3b 9d 01 cd |....!.@..B`.;...| +00000170 cc ac 40 a0 01 ff ff a9 ce 93 8c 09 0a c3 01 0a |..@.............| +00000180 24 61 37 64 61 31 61 61 36 2d 66 34 32 35 2d 34 |$a7da1aa6-f425-4| +00000190 37 38 39 2d 38 39 34 37 2d 62 30 33 34 37 38 36 |789-8947-b034786| +000001a0 65 64 33 37 34 12 06 56 61 73 69 6c 79 1a 07 53 |ed374..Vasily..S| +000001b0 69 64 6f 72 6f 76 20 01 28 fb 48 32 03 62 6d 70 |idorov .(.H2.bmp| +000001c0 3a 0d 2b 34 34 32 30 31 32 33 34 35 36 37 38 40 |:.+442012345678@| +000001d0 01 4d 50 e0 27 5c 50 17 58 04 62 05 53 75 6e 6e |.MP.'\P.X.b.Sunn| +000001e0 79 68 fa 01 68 f4 01 68 0a 72 08 4d 75 72 6d 61 |yh..h..h.r.Murma| +000001f0 6e 73 6b 7a 08 fd f0 89 42 c8 4c 04 42 81 01 11 |nskz....B.L.B...| +00000200 2d 44 54 fb 21 09 40 89 01 00 00 00 e8 76 48 37 |-DT.!.@......vH7| +00000210 42 95 01 00 00 48 44 9d 01 cd cc 4c c0 a0 01 80 |B....HD....L....| +00000220 d4 9f 93 01 ab 01 0d 00 00 80 41 12 05 70 6f 75 |..........A..pou| +00000230 6e 64 ac 01 b3 01 0b a2 06 05 0b 08 f7 03 0c 0c |nd..............| +00000240 b4 01 |..| +00000242 + +MESSAGE #1 AT 0x00000005 +uuid: "a7522158-3d41-4b77-ad69-6c598ee55c49" +name: "Ivan" +surname: "Petrov" +gender: male +birthDate: 4015 +photo: "png" +phoneNumber: "+74951234567\000" +isOnline: true +visitTime: 1546703100 +age: 38 +zodiacSign: capricorn +songs: "Yesterday" +songs: "Flowers" +color: 255 +color: 0 +color: 0 +hometown: "Moscow" +location: 55.7532158 +location: 37.6225052 +pi: 3.14 +lotteryWin: 214.1 +someRatio: 0.1 +temperature: 5.8 +randomBigNumber: 17060000000 +MeasureUnits { + coef: 1 + coef: 0.01 + coef: 1000 + unit: "meter" + unit: "centimeter" + unit: "kilometer" +} +Nestiness { + A { + b { + C { + d: 500 + e: 501 + e: 502 + } + } + } +} +MESSAGE #2 AT 0x000000F9 +uuid: "c694ad8a-f714-4ea3-907d-fd54fb25d9b5" +name: "Natalia" +surname: "Sokolova" +gender: female +birthDate: 8102 +photo: "jpg" +isOnline: false +age: 26 +zodiacSign: pisces +color: 100 +color: 200 +color: 50 +hometown: "Plymouth" +location: 50.4037247 +location: -4.14212322 +pi: 3.14159 +someRatio: 0.007 +temperature: 5.4 +randomBigNumber: -20000000000000 +MESSAGE #3 AT 0x0000017F +uuid: "a7da1aa6-f425-4789-8947-b034786ed374" +name: "Vasily" +surname: "Sidorov" +gender: male +birthDate: 9339 +photo: "bmp" +phoneNumber: "+442012345678" +isOnline: true +visitTime: 1546117200 +age: 23 +zodiacSign: leo +songs: "Sunny" +color: 250 +color: 244 +color: 10 +hometown: "Murmansk" +location: 68.9706802 +location: 33.0749817 +pi: 3.14159265358979 +lotteryWin: 100000000000 +someRatio: 800 +temperature: -3.2 +randomBigNumber: 154400000 +MeasureUnits { + coef: 16 + unit: "pound" +} +Nestiness { + A { + b { + C { + d: 503 + } + } + } +} + +Binary representation is as expected + +Roundtrip: +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753216,37.622504] 3.14 214.1 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 jpg \N 0 \N 26 pisces [] [100,200,50] Plymouth [50.403724,-4.142123] 3.14159 \N 0.007 5.4 -20000000000000 [] [] \N [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 bmp +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.97068,33.074982] 3.14159265358979 100000000000 800 -3.2 154400000 ['pound'] [16] 503 [] diff --git a/tests/queries/0_stateless/02240_protobuflist_format_persons.sh b/tests/queries/0_stateless/02240_protobuflist_format_persons.sh new file mode 100755 index 00000000000..dec14b54eb2 --- /dev/null +++ b/tests/queries/0_stateless/02240_protobuflist_format_persons.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +# End-to-end test of serialization/deserialization of a table with different +# data types to/from ProtobufList format. +# Cf. 00825_protobuf_format_persons.sh + +# To generate reference file for this test use the following commands: +# ninja ProtobufDelimitedMessagesSerializer +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +SCHEMADIR=$CURDIR/format_schemas +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -eo pipefail + +# Run the client. +$CLICKHOUSE_CLIENT --multiquery < $BINARY_FILE_PATH +echo +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/02240_protobuflist1_format_persons:Person" --input "$BINARY_FILE_PATH" --format "protobuflist" +echo +echo "Roundtrip:" +$CLICKHOUSE_CLIENT --query "CREATE TABLE roundtrip_persons_02240 AS persons_02240" +$CLICKHOUSE_CLIENT --query "INSERT INTO roundtrip_persons_02240 FORMAT ProtobufList SETTINGS format_schema='$SCHEMADIR/02240_protobuflist1_format_persons:Person'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM roundtrip_persons_02240 ORDER BY name" +rm "$BINARY_FILE_PATH" + +# Use schema 02240_protobuflist2_format_persons:AltPerson +echo +echo "Schema 02240_protobuflist2_format_persons:AltPerson" +BINARY_FILE_PATH=$(mktemp "$CURDIR/02240_protobuflist2_format_persons.XXXXXX.binary") +$CLICKHOUSE_CLIENT --query "SELECT * FROM persons_02240 ORDER BY name FORMAT ProtobufList SETTINGS format_schema = '$SCHEMADIR/02240_protobuflist2_format_persons:AltPerson'" > $BINARY_FILE_PATH +echo +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/02240_protobuflist2_format_persons:AltPerson" --input "$BINARY_FILE_PATH" --format="protobuflist" +echo +echo "Roundtrip:" +$CLICKHOUSE_CLIENT --query "CREATE TABLE alt_persons_02240 AS persons_02240" +$CLICKHOUSE_CLIENT --query "INSERT INTO alt_persons_02240 FORMAT ProtobufList SETTINGS format_schema='$SCHEMADIR/02240_protobuflist2_format_persons:AltPerson'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM alt_persons_02240 ORDER BY name" +rm "$BINARY_FILE_PATH" + +# Use schema 02240_protobuflist3_format_persons:StrPerson +echo +echo "Schema 02240_protobuflist3_format_persons:StrPerson as ProtobufList" +BINARY_FILE_PATH=$(mktemp "$CURDIR/02240_protobuflist3_format_persons.XXXXXX.binary") +$CLICKHOUSE_CLIENT --query "SELECT * FROM persons_02240 ORDER BY name FORMAT ProtobufList SETTINGS format_schema = '$SCHEMADIR/02240_protobuflist3_format_persons:StrPerson'" > $BINARY_FILE_PATH +echo +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/02240_protobuflist3_format_persons:StrPerson" --input "$BINARY_FILE_PATH" --format="protobuflist" +# echo +echo "Roundtrip:" +$CLICKHOUSE_CLIENT --query "CREATE TABLE str_persons_02240 AS persons_02240" +$CLICKHOUSE_CLIENT --query "INSERT INTO str_persons_02240 FORMAT ProtobufList SETTINGS format_schema='$SCHEMADIR/02240_protobuflist3_format_persons:StrPerson'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM str_persons_02240 ORDER BY name" +rm "$BINARY_FILE_PATH" + +# Use schema 02240_protobuflist_format_syntax2:Syntax2Person +echo +echo "Schema 02240_protobuf_format_syntax2:Syntax2Person" +BINARY_FILE_PATH=$(mktemp "$CURDIR/02240_protobuflist_format_persons.XXXXXX.binary") +$CLICKHOUSE_CLIENT --query "SELECT * FROM persons_02240 ORDER BY name FORMAT ProtobufList SETTINGS format_schema = '$SCHEMADIR/02240_protobuflist_format_persons_syntax2:Syntax2Person'" > $BINARY_FILE_PATH +echo +$CURDIR/helpers/protobuf_length_delimited_encoder.py --decode_and_check --format_schema "$SCHEMADIR/02240_protobuflist_format_persons_syntax2:Syntax2Person" --input "$BINARY_FILE_PATH" --format="protobuflist" +echo +echo "Roundtrip:" +$CLICKHOUSE_CLIENT --query "CREATE TABLE syntax2_persons_02240 AS persons_02240" +$CLICKHOUSE_CLIENT --query "INSERT INTO syntax2_persons_02240 FORMAT ProtobufList SETTINGS format_schema='$SCHEMADIR/02240_protobuflist_format_persons_syntax2:Syntax2Person'" < "$BINARY_FILE_PATH" +$CLICKHOUSE_CLIENT --query "SELECT * FROM syntax2_persons_02240 ORDER BY name" +rm "$BINARY_FILE_PATH" + +$CLICKHOUSE_CLIENT --multiquery <