Add Avro tests

dbms/tests/queries/0_stateless/01060_avro.reference

@@ -0,0 +1,37 @@
+=== input
+= primitive
+1,1,2,3.4,5.6,"b1","s1"
+0,-1,9223372036854775807,3.00004,0.00001,"",""
+1,2,"s1"
+0,9223372036854775807,""
+"s1",2,1
+"",9223372036854775807,0
+"s1"
+""
+= complex
+"A","t","['s1','s2']","[['a1'],['a2']]","s1",\N,"79cd909892d7e7ade1987cc7422628ba"
+"C","f","[]","[]",\N,123,"79cd909892d7e7ade1987cc7422628ba"
+"79cd909892d7e7ade1987cc7422628ba"
+"79cd909892d7e7ade1987cc7422628ba"
+= logical_types
+"2019-12-20","2020-01-10 01:31:56.227","2020-01-10 01:31:56.227000"
+18250,1578641516227,1578641516227000
+= compression
+1000
+1000
+1000
+= other
+0
+not compatible
+not found
+=== output
+= primitive
+1,1,2,3.4,5.6,"b1","s1"
+= complex
+"A","t","['s1','s2']","[['a1'],['a2']]","s1",\N,"79cd909892d7e7ade1987cc7422628ba"
+= logical_types
+"2019-12-20","2020-01-10 01:31:56.227","2020-01-10 01:31:56.227000"
+= other
+0
+1000
+not supported

dbms/tests/queries/0_stateless/01060_avro.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+set -e
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. $CUR_DIR/../shell_config.sh
+
+DATA_DIR=$CUR_DIR/data_avro
+
+# input
+echo === input
+echo = primitive
+
+cat $DATA_DIR/primitive.avro | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S 'a_bool UInt8, b_int Int32, c_long Int64, d_float Float32, e_double Float64, f_bytes String, g_string String' -q 'select * from table'
+cat $DATA_DIR/primitive.avro | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S 'a_bool UInt8, c_long Int64, g_string String' -q 'select * from table'
+cat $DATA_DIR/primitive.avro | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S 'g_string String, c_long Int64, a_bool UInt8' -q 'select * from table'
+cat $DATA_DIR/primitive.avro | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S 'g_string String' -q 'select * from table'
+
+echo = complex
+cat $DATA_DIR/complex.avro | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S "a_enum_to_string String, b_enum_to_enum Enum('t' = 1, 'f' = 0), c_array_string Array(String), d_array_array_string Array(Array(String)), e_union_null_string Nullable(String), f_union_long_null Nullable(Int64), g_fixed FixedString(32)" -q 'select * from table'
+cat $DATA_DIR/complex.avro | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S "g_fixed FixedString(32)" -q 'select * from table'
+
+echo = logical_types
+cat $DATA_DIR/logical_types.avro | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S 'a_date Date, b_timestamp_millis DateTime64, c_timestamp_micros DateTime64(6)' -q 'select * from table'
+cat $DATA_DIR/logical_types.avro | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S 'a_date Int32, b_timestamp_millis Int64, c_timestamp_micros Int64' -q 'select * from table'
+
+
+
+echo = compression
+cat $DATA_DIR/simple.null.avro | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table'
+cat $DATA_DIR/simple.deflate.avro | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table'
+cat $DATA_DIR/simple.snappy.avro | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table'
+
+echo = other
+#no data
+cat $DATA_DIR/empty.avro | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S 'a Int64' -q 'select count() from table'
+# type mismatch
+cat $DATA_DIR/simple.null.avro | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S 'a Int32' -q 'select count() from table' 2>&1 | grep -i 'not compatible' -o
+# field not found
+cat $DATA_DIR/simple.null.avro | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S 'b Int64' -q 'select count() from table' 2>&1 | grep -i 'not found' -o
+
+
+
+
+
+
+# output
+echo === output
+
+echo = primitive
+S1="a_bool UInt8, b_int Int32, c_long Int64, d_float Float32, e_double Float64, f_bytes String, g_string String"
+echo '1,1,2,3.4,5.6,"b1","s1"' | ${CLICKHOUSE_LOCAL} --input-format CSV -S "$S1" -q "select * from table  format Avro" | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S "$S1" -q 'select * from table'
+
+echo = complex
+S2="a_enum_to_string String, b_enum_to_enum Enum('t' = 1, 'f' = 0), c_array_string Array(String), d_array_array_string Array(Array(String)), e_union_null_string Nullable(String), f_union_long_null Nullable(Int64), g_fixed FixedString(32)"
+echo "\"A\",\"t\",\"['s1','s2']\",\"[['a1'],['a2']]\",\"s1\",\N,\"79cd909892d7e7ade1987cc7422628ba\"" | ${CLICKHOUSE_LOCAL} --input-format CSV -S "$S2" -q "select * from table  format Avro" | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S "$S2" -q 'select * from table'
+
+echo = logical_types
+S3="a_date Date, b_timestamp_millis DateTime64, c_timestamp_micros DateTime64(6)"
+echo '"2019-12-20","2020-01-10 01:31:56.227","2020-01-10 01:31:56.227000"' | ${CLICKHOUSE_LOCAL} --input-format CSV -S "$S3" -q "select * from table  format Avro" | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S "$S3" -q 'select * from table'
+
+echo = other
+S4="a Int64"
+${CLICKHOUSE_LOCAL} -q "select toInt64(number) as a from numbers(0)  format Avro" | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S "$S4" -q 'select count() from table'
+${CLICKHOUSE_LOCAL} -q "select toInt64(number) as a from numbers(1000)  format Avro" | ${CLICKHOUSE_LOCAL}  --input-format Avro --output-format CSV -S "$S4" -q 'select count() from table'
+
+# type not supported
+${CLICKHOUSE_LOCAL}  -q "select toInt16(123) as a format Avro" 2>&1 | grep -i 'not supported' -o

dbms/tests/queries/0_stateless/data_avro/complex.avsc

@@ -0,0 +1,20 @@
+{
+    "type": "record",
+    "name": "row",
+    "fields": [
+        {"name": "a_enum_to_string", "type": { "type": "enum", "name": "enum_1", "symbols" : ["A", "B", "C"]}},
+        {"name": "b_enum_to_enum", "type": { "type": "enum", "name": "enum_2", "symbols" : ["t", "f"]}},
+        {"name": "c_array_string", "type": { "type": "array", "items": "string"}},
+        {"name": "d_array_array_string", "type": { "type": "array", "items": {"type": "array", "items": "string"}}},
+        {"name": "e_union_null_string", "type": ["null", "string"]},
+        {"name": "f_union_long_null", "type": ["long", "null"]},
+        {"name": "g_fixed", "type": {"type":"fixed", "size": 32, "name": "fixed_1"}},
+        {"name": "h_record_skip", "type": {
+            "type": "record",
+            "name": "subrecord",
+            "fields": [
+                {"name": "a", "type": "string"}
+            ]
+        }}
+    ]
+  }

dbms/tests/queries/0_stateless/data_avro/complex.json

@@ -0,0 +1,2 @@
+{"a_enum_to_string":"A","b_enum_to_enum":"t","c_array_string":["s1", "s2"],"d_array_array_string":[["a1"], ["a2"]],"e_union_null_string":{"string": "s1"},"f_union_long_null":null,"g_fixed":"79cd909892d7e7ade1987cc7422628ba","h_record_skip":{"a": "a"}}
+{"a_enum_to_string":"C","b_enum_to_enum":"f","c_array_string":[],"d_array_array_string":[],"e_union_null_string":null,"f_union_long_null":{"long": 123},"g_fixed":"79cd909892d7e7ade1987cc7422628ba","h_record_skip":{"a": "a"}}

dbms/tests/queries/0_stateless/data_avro/empty.avsc

@@ -0,0 +1,7 @@
+{
+    "type": "record",
+    "name": "row",
+    "fields": [
+        {"name": "a", "type": "long"}
+    ]
+  }

dbms/tests/queries/0_stateless/data_avro/generate_avro.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+#avro tools: https://www.apache.org/dyn/closer.cgi?path=avro/avro-1.9.1/java/avro-tools-1.9.1.jar
+
+
+avro-tools fromjson  --schema-file primitive.avsc primitive.json > primitive.avro
+avro-tools fromjson  --schema-file complex.avsc complex.json > complex.avro
+avro-tools fromjson  --schema-file logical_types.avsc logical_types.json > logical_types.avro
+avro-tools fromjson  --schema-file empty.avsc empty.json > empty.avro
+
+#compression
+avro-tools fromjson --codec null  --schema-file simple.avsc simple.json > simple.null.avro
+avro-tools fromjson --codec deflate  --schema-file simple.avsc simple.json > simple.deflate.avro
+avro-tools fromjson --codec snappy  --schema-file simple.avsc simple.json > simple.snappy.avro

dbms/tests/queries/0_stateless/data_avro/logical_types.avsc

@@ -0,0 +1,9 @@
+{
+    "type": "record",
+    "name": "row",
+    "fields": [
+        {"name": "a_date", "type": { "type": "int", "logicalType": "date"}},
+        {"name": "b_timestamp_millis", "type": { "type": "long", "logicalType": "timestamp-millis"}},
+        {"name": "c_timestamp_micros", "type": { "type": "long", "logicalType": "timestamp-micros"}}
+    ]
+  }

dbms/tests/queries/0_stateless/data_avro/logical_types.json

@@ -0,0 +1 @@
+{"a_date":18250,"b_timestamp_millis":1578641516227,"c_timestamp_micros":1578641516227000}

dbms/tests/queries/0_stateless/data_avro/primitive.avsc

@@ -0,0 +1,14 @@
+{
+    "type": "record",
+    "name": "row",
+    "fields": [
+        {"name": "a_bool", "type": "boolean"},
+        {"name": "b_int", "type": "int"},
+        {"name": "c_long", "type": "long"},
+        {"name": "d_float", "type": "float"},
+        {"name": "e_double", "type": "double"},
+        {"name": "f_bytes", "type": "bytes"},
+        {"name": "g_string", "type": "string"},
+        {"name": "h_null", "type": "null"}
+    ]
+  }

dbms/tests/queries/0_stateless/data_avro/primitive.json

@@ -0,0 +1,2 @@
+{"a_bool":true,"b_int":1,"c_long":2,"d_float":3.4,"e_double":5.6,"f_bytes":"b1","g_string":"s1","h_null": null}
+{"a_bool":false,"b_int":-1,"c_long":9223372036854775807,"d_float":3.00004,"e_double":0.00001,"f_bytes":"","g_string":"","h_null": null}

dbms/tests/queries/0_stateless/data_avro/simple.avsc

@@ -0,0 +1,7 @@
+{
+    "type": "record",
+    "name": "row",
+    "fields": [
+        {"name": "a", "type": "long"}
+    ]
+  }