From aba86b010b9cc51064987086691cbc70f25f13be Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Fri, 17 Sep 2021 17:32:23 +0300 Subject: [PATCH] Update docs, tests and add some fixes --- docs/en/interfaces/formats.md | 26 +++++++++-------- .../Formats/Impl/MsgPackRowInputFormat.cpp | 12 ++++++-- .../Formats/Impl/MsgPackRowInputFormat.h | 1 + .../Formats/Impl/MsgPackRowOutputFormat.cpp | 4 +-- .../01098_msgpack_format.reference | 15 +++++++++- .../0_stateless/01098_msgpack_format.sh | 28 +++++++++++++++++-- 6 files changed, 66 insertions(+), 20 deletions(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index f353f0b1d43..7b9245118ae 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1558,18 +1558,20 @@ ClickHouse supports reading and writing [MessagePack](https://msgpack.org/) data ### Data Types Matching {#data-types-matching-msgpack} -| MsgPack data type | ClickHouse data type | -|---------------------------------|----------------------------------------------------------------------------------| -| `uint N`, `positive fixint` | [UIntN](../sql-reference/data-types/int-uint.md) | -| `int N` | [IntN](../sql-reference/data-types/int-uint.md) | -| `fixstr`, `str 8`, `str 16`, `str 32` | [String](../sql-reference/data-types/string.md), [FixedString](../sql-reference/data-types/fixedstring.md) | -| `float 32` | [Float32](../sql-reference/data-types/float.md) | -| `float 64` | [Float64](../sql-reference/data-types/float.md) | -| `uint 16` | [Date](../sql-reference/data-types/date.md) | -| `uint 32` | [DateTime](../sql-reference/data-types/datetime.md) | -| `uint 64` | [DateTime64](../sql-reference/data-types/datetime.md) | -| `fixarray`, `array 16`, `array 32`| [Array](../sql-reference/data-types/array.md) | -| `nil` | [Nothing](../sql-reference/data-types/special-data-types/nothing.md) | +| MessagePack data type (`INSERT`) | ClickHouse data type | MessagePack data type (`SELECT`) | +|--------------------------------------------------------------------|-----------------------------------------------------------|------------------------------------| +| `uint N`, `positive fixint` | [UIntN](../sql-reference/data-types/int-uint.md) | `uint N` | +| `int N` | [IntN](../sql-reference/data-types/int-uint.md) | `int N` | +| `bool` | [UInt8](../sql-reference/data-types/int-uint.md) | `uint 8` | +| `fixstr`, `str 8`, `str 16`, `str 32`, `bin 8`, `bin 16`, `bin 32` | [String](../sql-reference/data-types/string.md) | `bin 8`, `bin 16`, `bin 32` | +| `fixstr`, `str 8`, `str 16`, `str 32`, `bin 8`, `bin 16`, `bin 32` | [FixedString](../sql-reference/data-types/fixedstring.md) | `bin 8`, `bin 16`, `bin 32` | +| `float 32` | [Float32](../sql-reference/data-types/float.md) | `float 32` | +| `float 64` | [Float64](../sql-reference/data-types/float.md) | `float 64` | +| `uint 16` | [Date](../sql-reference/data-types/date.md) | `uint 16` | +| `uint 32` | [DateTime](../sql-reference/data-types/datetime.md) | `uint 32` | +| `uint 64` | [DateTime64](../sql-reference/data-types/datetime.md) | `uint 64` | +| `fixarray`, `array 16`, `array 32` | [Array](../sql-reference/data-types/array.md) | `fixarray`, `array 16`, `array 32` | +| `fixmap`, `map 16`, `map 32` | [Map](../sql-reference/data-types/map.md) | `fixmap`, `map 16`, `map 32` | Example: diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp index 5a2c92d06f5..1f7d14eacc9 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp @@ -59,7 +59,7 @@ static bool checkAndInsertNullable(IColumn & column, DataTypePtr type, InsertFun { auto & nullable_column = assert_cast(column); auto & nested_column = nullable_column.getNestedColumn(); - auto & nested_type = assert_cast(type.get())->getNestedType(); + const auto & nested_type = assert_cast(type.get())->getNestedType(); insert_func(nested_column, nested_type); nullable_column.getNullMapColumn().insertValue(0); return true; @@ -202,7 +202,7 @@ static void insertNull(IColumn & column, DataTypePtr type) insertNull(column_, type_); }; - /// LowCardinalityNullable(...) + /// LowCardinality(Nullable(...)) if (checkAndInsertLowCardinality(column, type, insert_func)) return; @@ -230,7 +230,7 @@ bool MsgPackVisitor::visit_str(const char * value, size_t size) // NOLINT return true; } -bool MsgPackVisitor::visit_bin(const char * value, size_t size) +bool MsgPackVisitor::visit_bin(const char * value, size_t size) // NOLINT { insertString(info_stack.top().column, info_stack.top().type, value, size); return true; @@ -248,6 +248,12 @@ bool MsgPackVisitor::visit_float64(Float64 value) // NOLINT return true; } +bool MsgPackVisitor::visit_boolean(bool value) +{ + insertInteger(info_stack.top().column, info_stack.top().type, UInt64(value)); + return true; +} + bool MsgPackVisitor::start_array(size_t size) // NOLINT { if (!isArray(info_stack.top().type)) diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.h b/src/Processors/Formats/Impl/MsgPackRowInputFormat.h index 8d7bc78c6f6..1d790b4485b 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.h +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.h @@ -35,6 +35,7 @@ public: bool visit_float64(Float64 value); bool visit_str(const char * value, size_t size); bool visit_bin(const char * value, size_t size); + bool visit_boolean(bool value); bool start_array(size_t size); bool end_array(); bool visit_nil(); diff --git a/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp index 0339ad6841c..f8fd4088fb0 100644 --- a/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp @@ -149,8 +149,8 @@ void MsgPackRowOutputFormat::serializeField(const IColumn & column, DataTypePtr packer.pack_map(size); for (size_t i = 0; i < size; ++i) { - serializeField(*key_column, map_type.getKeyType(), row_num); - serializeField(*value_column, map_type.getValueType(), row_num); + serializeField(*key_column, map_type.getKeyType(), offset + i); + serializeField(*value_column, map_type.getValueType(), offset + i); } return; } diff --git a/tests/queries/0_stateless/01098_msgpack_format.reference b/tests/queries/0_stateless/01098_msgpack_format.reference index e9f2ccf2084..384852f24a7 100644 --- a/tests/queries/0_stateless/01098_msgpack_format.reference +++ b/tests/queries/0_stateless/01098_msgpack_format.reference @@ -12,7 +12,7 @@ 2020-01-02 2020-01-02 {1:2,2:3} [{1:[1,2],2:[3,4]},{3:[5,6],4:[7,8]}] -{1:2,1:2} [{1:[1,2],1:[1,2]},{2:[3,4],2:[3,4]}] +{1:2,2:3} [{1:[1,2],2:[3,4]},{3:[5,6],4:[7,8]}] 42 42 42 ['42','42'] ['42','42'] 42 \N \N [NULL,'42',NULL] [NULL,'42',NULL] 42 42 42 ['42','42'] ['42','42'] @@ -20,3 +20,16 @@ OK OK OK +OK +OK +OK +OK +OK +OK +OK +OK +OK +OK +OK +OK +OK diff --git a/tests/queries/0_stateless/01098_msgpack_format.sh b/tests/queries/0_stateless/01098_msgpack_format.sh index 262b70ae960..49974913ea8 100755 --- a/tests/queries/0_stateless/01098_msgpack_format.sh +++ b/tests/queries/0_stateless/01098_msgpack_format.sh @@ -99,9 +99,33 @@ $CLICKHOUSE_CLIENT --query="SELECT toString(number) FROM numbers(10) FORMAT Msg $CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x UInt64')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; $CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Float32')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; - - $CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Array(UInt32)')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; +$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Map(UInt64, UInt64)')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; + + +$CLICKHOUSE_CLIENT --query="SELECT number FROM numbers(10) FORMAT MsgPack" > $USER_FILES_PATH/data.msgpack + +$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Float32')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; +$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x String')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; +$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Array(UInt64)')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; +$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Map(UInt64, UInt64)')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; + + +$CLICKHOUSE_CLIENT --query="SELECT [number, number + 1] FROM numbers(10) FORMAT MsgPack" > $USER_FILES_PATH/data.msgpack + +$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Float32')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; +$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x String')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; +$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x UInt64')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; +$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Map(UInt64, UInt64)')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; + + +$CLICKHOUSE_CLIENT --query="SELECT map(number, number + 1) FROM numbers(10) FORMAT MsgPack" > $USER_FILES_PATH/data.msgpack + +$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Float32')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; +$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x String')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; +$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x UInt64')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; +$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Array(UInt64)')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; + rm $USER_FILES_PATH/data.msgpack