thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-25 17:12:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update docs, tests and add some fixes

Browse Source
This commit is contained in:
Pavel Kruglov 2021-09-17 17:32:23 +03:00
parent 96dba14213
commit aba86b010b
6 changed files with 66 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
docs/en/interfaces/formats.md
View File

@ -1558,18 +1558,20 @@ ClickHouse supports reading and writing [MessagePack](https://msgpack.org/) data
### Data Types Matching {#data-types-matching-msgpack} ### Data Types Matching {#data-types-matching-msgpack}
| MsgPack data type | ClickHouse data type | | MessagePack data type (`INSERT`) | ClickHouse data type | MessagePack data type (`SELECT`) |
|---------------------------------|----------------------------------------------------------------------------------| |--------------------------------------------------------------------|-----------------------------------------------------------|------------------------------------|
| `uint N`, `positive fixint` | [UIntN](../sql-reference/data-types/int-uint.md) | | `uint N`, `positive fixint` | [UIntN](../sql-reference/data-types/int-uint.md) | `uint N` |
| `int N` | [IntN](../sql-reference/data-types/int-uint.md) | | `int N` | [IntN](../sql-reference/data-types/int-uint.md) | `int N` |
| `fixstr`, `str 8`, `str 16`, `str 32` | [String](../sql-reference/data-types/string.md), [FixedString](../sql-reference/data-types/fixedstring.md) | | `bool` | [UInt8](../sql-reference/data-types/int-uint.md) | `uint 8` |
| `float 32` | [Float32](../sql-reference/data-types/float.md) | | `fixstr`, `str 8`, `str 16`, `str 32`, `bin 8`, `bin 16`, `bin 32` | [String](../sql-reference/data-types/string.md) | `bin 8`, `bin 16`, `bin 32` |
| `float 64` | [Float64](../sql-reference/data-types/float.md) | | `fixstr`, `str 8`, `str 16`, `str 32`, `bin 8`, `bin 16`, `bin 32` | [FixedString](../sql-reference/data-types/fixedstring.md) | `bin 8`, `bin 16`, `bin 32` |
| `uint 16` | [Date](../sql-reference/data-types/date.md) | | `float 32` | [Float32](../sql-reference/data-types/float.md) | `float 32` |
| `uint 32` | [DateTime](../sql-reference/data-types/datetime.md) | | `float 64` | [Float64](../sql-reference/data-types/float.md) | `float 64` |
| `uint 64` | [DateTime64](../sql-reference/data-types/datetime.md) | | `uint 16` | [Date](../sql-reference/data-types/date.md) | `uint 16` |
| `fixarray`, `array 16`, `array 32`| [Array](../sql-reference/data-types/array.md) | | `uint 32` | [DateTime](../sql-reference/data-types/datetime.md) | `uint 32` |
| `nil` | [Nothing](../sql-reference/data-types/special-data-types/nothing.md) | | `uint 64` | [DateTime64](../sql-reference/data-types/datetime.md) | `uint 64` |
| `fixarray`, `array 16`, `array 32` | [Array](../sql-reference/data-types/array.md) | `fixarray`, `array 16`, `array 32` |
| `fixmap`, `map 16`, `map 32` | [Map](../sql-reference/data-types/map.md) | `fixmap`, `map 16`, `map 32` |
Example: Example:

12
src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp
View File

@ -59,7 +59,7 @@ static bool checkAndInsertNullable(IColumn & column, DataTypePtr type, InsertFun
{ {
auto & nullable_column = assert_cast<ColumnNullable &>(column); auto & nullable_column = assert_cast<ColumnNullable &>(column);
auto & nested_column = nullable_column.getNestedColumn(); auto & nested_column = nullable_column.getNestedColumn();
auto & nested_type = assert_cast<const DataTypeNullable *>(type.get())->getNestedType(); const auto & nested_type = assert_cast<const DataTypeNullable *>(type.get())->getNestedType();
insert_func(nested_column, nested_type); insert_func(nested_column, nested_type);
nullable_column.getNullMapColumn().insertValue(0); nullable_column.getNullMapColumn().insertValue(0);
return true; return true;
@ -202,7 +202,7 @@ static void insertNull(IColumn & column, DataTypePtr type)
insertNull(column_, type_); insertNull(column_, type_);
}; };
/// LowCardinalityNullable(...) /// LowCardinality(Nullable(...))
if (checkAndInsertLowCardinality(column, type, insert_func)) if (checkAndInsertLowCardinality(column, type, insert_func))
return; return;
@ -230,7 +230,7 @@ bool MsgPackVisitor::visit_str(const char * value, size_t size) // NOLINT
return true; return true;
} }
bool MsgPackVisitor::visit_bin(const char * value, size_t size) bool MsgPackVisitor::visit_bin(const char * value, size_t size) // NOLINT
{ {
insertString(info_stack.top().column, info_stack.top().type, value, size); insertString(info_stack.top().column, info_stack.top().type, value, size);
return true; return true;
@ -248,6 +248,12 @@ bool MsgPackVisitor::visit_float64(Float64 value) // NOLINT
return true; return true;
} }
bool MsgPackVisitor::visit_boolean(bool value)
{
insertInteger(info_stack.top().column, info_stack.top().type, UInt64(value));
return true;
}
bool MsgPackVisitor::start_array(size_t size) // NOLINT bool MsgPackVisitor::start_array(size_t size) // NOLINT
{ {
if (!isArray(info_stack.top().type)) if (!isArray(info_stack.top().type))

1
src/Processors/Formats/Impl/MsgPackRowInputFormat.h
View File

@ -35,6 +35,7 @@ public:
bool visit_float64(Float64 value); bool visit_float64(Float64 value);
bool visit_str(const char * value, size_t size); bool visit_str(const char * value, size_t size);
bool visit_bin(const char * value, size_t size); bool visit_bin(const char * value, size_t size);
bool visit_boolean(bool value);
bool start_array(size_t size); bool start_array(size_t size);
bool end_array(); bool end_array();
bool visit_nil(); bool visit_nil();

4
src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp
View File

@ -149,8 +149,8 @@ void MsgPackRowOutputFormat::serializeField(const IColumn & column, DataTypePtr
packer.pack_map(size); packer.pack_map(size);
for (size_t i = 0; i < size; ++i) for (size_t i = 0; i < size; ++i)
{ {
serializeField(*key_column, map_type.getKeyType(), row_num); serializeField(*key_column, map_type.getKeyType(), offset + i);
serializeField(*value_column, map_type.getValueType(), row_num); serializeField(*value_column, map_type.getValueType(), offset + i);
} }
return; return;
} }

15
tests/queries/0_stateless/01098_msgpack_format.reference
View File

@ -12,7 +12,7 @@
2020-01-02 2020-01-02
2020-01-02 2020-01-02
{1:2,2:3} [{1:[1,2],2:[3,4]},{3:[5,6],4:[7,8]}] {1:2,2:3} [{1:[1,2],2:[3,4]},{3:[5,6],4:[7,8]}]
{1:2,1:2} [{1:[1,2],1:[1,2]},{2:[3,4],2:[3,4]}] {1:2,2:3} [{1:[1,2],2:[3,4]},{3:[5,6],4:[7,8]}]
42 42 42 ['42','42'] ['42','42'] 42 42 42 ['42','42'] ['42','42']
42 \N \N [NULL,'42',NULL] [NULL,'42',NULL] 42 \N \N [NULL,'42',NULL] [NULL,'42',NULL]
42 42 42 ['42','42'] ['42','42'] 42 42 42 ['42','42'] ['42','42']
@ -20,3 +20,16 @@
OK OK
OK OK
OK OK
OK
OK
OK
OK
OK
OK
OK
OK
OK
OK
OK
OK
OK

28
tests/queries/0_stateless/01098_msgpack_format.sh
View File

@ -99,9 +99,33 @@ $CLICKHOUSE_CLIENT --query="SELECT toString(number) FROM numbers(10) FORMAT Msg
$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x UInt64')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; $CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x UInt64')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL';
$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Float32')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; $CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Float32')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL';
$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Array(UInt32)')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL'; $CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Array(UInt32)')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL';
$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Map(UInt64, UInt64)')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL';
$CLICKHOUSE_CLIENT --query="SELECT number FROM numbers(10) FORMAT MsgPack" > $USER_FILES_PATH/data.msgpack
$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Float32')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL';
$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x String')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL';
$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Array(UInt64)')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL';
$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Map(UInt64, UInt64)')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL';
$CLICKHOUSE_CLIENT --query="SELECT [number, number + 1] FROM numbers(10) FORMAT MsgPack" > $USER_FILES_PATH/data.msgpack
$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Float32')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL';
$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x String')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL';
$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x UInt64')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL';
$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Map(UInt64, UInt64)')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL';
$CLICKHOUSE_CLIENT --query="SELECT map(number, number + 1) FROM numbers(10) FORMAT MsgPack" > $USER_FILES_PATH/data.msgpack
$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Float32')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL';
$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x String')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL';
$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x UInt64')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL';
$CLICKHOUSE_CLIENT --query="SELECT * FROM file('data.msgpack', 'MsgPack', 'x Array(UInt64)')" 2>&1 | grep -F -q "ILLEGAL_COLUMN" && echo 'OK' || echo 'FAIL';
rm $USER_FILES_PATH/data.msgpack rm $USER_FILES_PATH/data.msgpack