thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-24 16:42:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix type map ser/de to json

Browse Source
This commit is contained in:
Anton Popov 2020-12-15 17:34:37 +03:00
parent c78861e86e
commit dd09e1d783
6 changed files with 108 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
src/DataTypes/DataTypeMap.cpp
View File

@ -102,7 +102,9 @@ void DataTypeMap::deserializeBinary(IColumn & column, ReadBuffer & istr) const
nested->deserializeBinary(extractNestedColumn(column), istr);
}
void DataTypeMap::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
template <typename Writer>
void DataTypeMap::serializeTextImpl(const IColumn & column, size_t row_num, WriteBuffer & ostr, Writer && writer) const
{
const auto & column_map = assert_cast<const ColumnMap &>(column);
@ -118,15 +120,15 @@ void DataTypeMap::serializeText(const IColumn & column, size_t row_num, WriteBuf
{
if (i != offset)
writeChar(',', ostr);
key_type->serializeAsTextQuoted(nested_tuple.getColumn(0), i, ostr, settings);
writer(key_type, nested_tuple.getColumn(0), i);
writeChar(':', ostr);
value_type->serializeAsTextQuoted(nested_tuple.getColumn(1), i, ostr, settings);
writer(value_type, nested_tuple.getColumn(1), i);
}
writeChar('}', ostr);
}
template <typename Reader>
static void deserializeTextImpl(IColumn & column, ReadBuffer & istr, bool need_safe_get_int_key, Reader && read_kv)
void DataTypeMap::deserializeTextImpl(IColumn & column, ReadBuffer & istr, bool need_safe_get_int_key, Reader && reader) const
{
auto & column_map = assert_cast<ColumnMap &>(column);
@ -166,18 +168,18 @@ static void deserializeTextImpl(IColumn & column, ReadBuffer & istr, bool need_s
while (*tmp != ':' && *tmp != '}')
++tmp;
*tmp = ' ';
read_kv(key_column, true);
reader(key_type, key_column);
}
else
{
read_kv(key_column, true);
reader(key_type, key_column);
skipWhitespaceIfAny(istr);
assertChar(':', istr);
}
++size;
skipWhitespaceIfAny(istr);
read_kv(value_column, false);
reader(value_type, value_column);
skipWhitespaceIfAny(istr);
}
@ -191,30 +193,47 @@ static void deserializeTextImpl(IColumn & column, ReadBuffer & istr, bool need_s
}
}
void DataTypeMap::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
serializeTextImpl(column, row_num, ostr,
[&](const DataTypePtr & nested_type, const IColumn & nested_column, size_t pos)
{
nested_type->serializeAsTextQuoted(nested_column, pos, ostr, settings);
});
}
void DataTypeMap::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
// need_safe_get_int_key is set for Interger to prevent to readIntTextUnsafe
bool need_safe_get_int_key = isInteger(key_type);
deserializeTextImpl(column, istr, need_safe_get_int_key,
[&](IColumn & nested_column, bool is_key)
[&](const DataTypePtr & nested_type, IColumn & nested_column)
{
if (is_key)
key_type->deserializeAsTextQuoted(nested_column, istr, settings);
else
value_type->deserializeAsTextQuoted(nested_column, istr, settings);
nested_type->deserializeAsTextQuoted(nested_column, istr, settings);
});
}
void DataTypeMap::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
serializeText(column, row_num, ostr, settings);
serializeTextImpl(column, row_num, ostr,
[&](const DataTypePtr & nested_type, const IColumn & nested_column, size_t pos)
{
nested_type->serializeAsTextJSON(nested_column, pos, ostr, settings);
});
}
void DataTypeMap::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{
deserializeText(column, istr, settings);
// need_safe_get_int_key is set for Interger to prevent to readIntTextUnsafe
bool need_safe_get_int_key = isInteger(key_type);
deserializeTextImpl(column, istr, need_safe_get_int_key,
[&](const DataTypePtr & nested_type, IColumn & nested_column)
{
nested_type->deserializeAsTextJSON(nested_column, istr, settings);
});
}
void DataTypeMap::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const

7
src/DataTypes/DataTypeMap.h
View File

@ -88,6 +88,13 @@ public:
const DataTypePtr & getKeyType() const { return key_type; }
const DataTypePtr & getValueType() const { return value_type; }
DataTypes getKeyValueTypes() const { return {key_type, value_type}; }
private:
template <typename Writer>
void serializeTextImpl(const IColumn & column, size_t row_num, WriteBuffer & ostr, Writer && writer) const;
template <typename Reader>
void deserializeTextImpl(IColumn & column, ReadBuffer & istr, bool need_safe_get_int_key, Reader && reader) const;
};
}

2
src/Functions/FunctionsConversion.h
View File

@ -2326,7 +2326,7 @@ private:
const auto * nested_tuple = typeid_cast<const DataTypeTuple *>(from_array->getNestedType().get());
if (!nested_tuple || nested_tuple->getElements().size() != 2)
throw Exception{"CAST AS Map from array requeires nested tuple of 2 elements.\n"
"Left type: " + from_tuple->getName() + ", right type: " + to_type->getName(), ErrorCodes::TYPE_MISMATCH};
"Left type: " + from_array->getName() + ", right type: " + to_type->getName(), ErrorCodes::TYPE_MISMATCH};
return createArrayToMapWrrapper(nested_tuple->getElements(), to_type->getKeyValueTypes());
}

4
src/Functions/map.cpp
View File

@ -56,8 +56,8 @@ public:
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (arguments.empty() || arguments.size() % 2 != 0)
throw Exception("Function " + getName() + " requires at least one argument.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
if (arguments.size() % 2 != 0)
throw Exception("Function " + getName() + " even number of arguments.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
DataTypes keys, values;
for (size_t i = 0; i < arguments.size(); i += 2)

46
tests/queries/0_stateless/01550_type_map_formats.reference Normal file
View File

@ -0,0 +1,46 @@
JSON
{
"meta":
[
{
"name": "m",
"type": "Map(String,UInt32)"
},
{
"name": "m1",
"type": "Map(String,Date)"
},
{
"name": "m2",
"type": "Map(String,Array(UInt32))"
}
],
"data":
[
{
"m": {"k1":1,"k2":2,"k3":3},
"m1": {"k1":"2020-05-05"},
"m2": {"k1":[],"k2":[7,8]}
},
{
"m": {"k1":10,"k3":30},
"m1": {"k2":"2020-06-06"},
"m2": {}
}
],
"rows": 2
}
JSONEachRow
{"m":{"k1":10,"k3":30},"m1":{"k2":"2020-06-06"},"m2":{}}
{"m":{"k1":1,"k2":2,"k3":3},"m1":{"k1":"2020-05-05"},"m2":{"k1":[],"k2":[7,8]}}
CSV
"{'k1':1,'k2':2,'k3':3}","{'k1':'2020-05-05'}","{'k1':[],'k2':[7,8]}"
"{'k1':10,'k3':30}","{'k2':'2020-06-06'}","{}"
TSV
{'k1':1,'k2':2,'k3':3} {'k1':'2020-05-05'} {'k1':[],'k2':[7,8]}
{'k1':10,'k3':30} {'k2':'2020-06-06'} {}
TSKV
m={'k1':10,'k3':30} m1={'k2':'2020-06-06'} m2={}
m={'k1':1,'k2':2,'k3':3} m1={'k1':'2020-05-05'} m2={'k1':[],'k2':[7,8]}

19
tests/queries/0_stateless/01550_type_map_formats.sql Normal file
View File

@ -0,0 +1,19 @@
SET allow_experimental_map_type = 1;
SET output_format_write_statistics = 0;
DROP TABLE IF EXISTS map_formats;
CREATE TABLE map_formats (m Map(String, UInt32), m1 Map(String, Date), m2 Map(String, Array(UInt32))) ENGINE = Log;
INSERT INTO map_formats VALUES(map('k1', 1, 'k2', 2, 'k3', 3), map('k1', toDate('2020-05-05')), map('k1', [], 'k2', [7, 8]));
INSERT INTO map_formats VALUES(map('k1', 10, 'k3', 30), map('k2', toDate('2020-06-06')), map());
SELECT 'JSON';
SELECT * FROM map_formats FORMAT JSON;
SELECT 'JSONEachRow';
SELECT * FROM map_formats FORMAT JSONEachRow;
SELECT 'CSV';
SELECT * FROM map_formats FORMAT CSV;
SELECT 'TSV';
SELECT * FROM map_formats FORMAT TSV;
SELECT 'TSKV';
SELECT * FROM map_formats FORMAT TSKV;