fix type map ser/de to json

This commit is contained in:
Anton Popov 2020-12-15 17:34:37 +03:00
parent c78861e86e
commit dd09e1d783
6 changed files with 108 additions and 17 deletions

View File

@ -102,7 +102,9 @@ void DataTypeMap::deserializeBinary(IColumn & column, ReadBuffer & istr) const
nested->deserializeBinary(extractNestedColumn(column), istr); nested->deserializeBinary(extractNestedColumn(column), istr);
} }
void DataTypeMap::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
template <typename Writer>
void DataTypeMap::serializeTextImpl(const IColumn & column, size_t row_num, WriteBuffer & ostr, Writer && writer) const
{ {
const auto & column_map = assert_cast<const ColumnMap &>(column); const auto & column_map = assert_cast<const ColumnMap &>(column);
@ -118,15 +120,15 @@ void DataTypeMap::serializeText(const IColumn & column, size_t row_num, WriteBuf
{ {
if (i != offset) if (i != offset)
writeChar(',', ostr); writeChar(',', ostr);
key_type->serializeAsTextQuoted(nested_tuple.getColumn(0), i, ostr, settings); writer(key_type, nested_tuple.getColumn(0), i);
writeChar(':', ostr); writeChar(':', ostr);
value_type->serializeAsTextQuoted(nested_tuple.getColumn(1), i, ostr, settings); writer(value_type, nested_tuple.getColumn(1), i);
} }
writeChar('}', ostr); writeChar('}', ostr);
} }
template <typename Reader> template <typename Reader>
static void deserializeTextImpl(IColumn & column, ReadBuffer & istr, bool need_safe_get_int_key, Reader && read_kv) void DataTypeMap::deserializeTextImpl(IColumn & column, ReadBuffer & istr, bool need_safe_get_int_key, Reader && reader) const
{ {
auto & column_map = assert_cast<ColumnMap &>(column); auto & column_map = assert_cast<ColumnMap &>(column);
@ -166,18 +168,18 @@ static void deserializeTextImpl(IColumn & column, ReadBuffer & istr, bool need_s
while (*tmp != ':' && *tmp != '}') while (*tmp != ':' && *tmp != '}')
++tmp; ++tmp;
*tmp = ' '; *tmp = ' ';
read_kv(key_column, true); reader(key_type, key_column);
} }
else else
{ {
read_kv(key_column, true); reader(key_type, key_column);
skipWhitespaceIfAny(istr); skipWhitespaceIfAny(istr);
assertChar(':', istr); assertChar(':', istr);
} }
++size; ++size;
skipWhitespaceIfAny(istr); skipWhitespaceIfAny(istr);
read_kv(value_column, false); reader(value_type, value_column);
skipWhitespaceIfAny(istr); skipWhitespaceIfAny(istr);
} }
@ -191,30 +193,47 @@ static void deserializeTextImpl(IColumn & column, ReadBuffer & istr, bool need_s
} }
} }
void DataTypeMap::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
serializeTextImpl(column, row_num, ostr,
[&](const DataTypePtr & nested_type, const IColumn & nested_column, size_t pos)
{
nested_type->serializeAsTextQuoted(nested_column, pos, ostr, settings);
});
}
void DataTypeMap::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const void DataTypeMap::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{ {
// need_safe_get_int_key is set for Interger to prevent to readIntTextUnsafe // need_safe_get_int_key is set for Interger to prevent to readIntTextUnsafe
bool need_safe_get_int_key = isInteger(key_type); bool need_safe_get_int_key = isInteger(key_type);
deserializeTextImpl(column, istr, need_safe_get_int_key, deserializeTextImpl(column, istr, need_safe_get_int_key,
[&](IColumn & nested_column, bool is_key) [&](const DataTypePtr & nested_type, IColumn & nested_column)
{ {
if (is_key) nested_type->deserializeAsTextQuoted(nested_column, istr, settings);
key_type->deserializeAsTextQuoted(nested_column, istr, settings);
else
value_type->deserializeAsTextQuoted(nested_column, istr, settings);
}); });
} }
void DataTypeMap::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const void DataTypeMap::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{ {
serializeText(column, row_num, ostr, settings); serializeTextImpl(column, row_num, ostr,
[&](const DataTypePtr & nested_type, const IColumn & nested_column, size_t pos)
{
nested_type->serializeAsTextJSON(nested_column, pos, ostr, settings);
});
} }
void DataTypeMap::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const void DataTypeMap::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{ {
deserializeText(column, istr, settings); // need_safe_get_int_key is set for Interger to prevent to readIntTextUnsafe
bool need_safe_get_int_key = isInteger(key_type);
deserializeTextImpl(column, istr, need_safe_get_int_key,
[&](const DataTypePtr & nested_type, IColumn & nested_column)
{
nested_type->deserializeAsTextJSON(nested_column, istr, settings);
});
} }
void DataTypeMap::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const void DataTypeMap::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const

View File

@ -88,6 +88,13 @@ public:
const DataTypePtr & getKeyType() const { return key_type; } const DataTypePtr & getKeyType() const { return key_type; }
const DataTypePtr & getValueType() const { return value_type; } const DataTypePtr & getValueType() const { return value_type; }
DataTypes getKeyValueTypes() const { return {key_type, value_type}; } DataTypes getKeyValueTypes() const { return {key_type, value_type}; }
private:
template <typename Writer>
void serializeTextImpl(const IColumn & column, size_t row_num, WriteBuffer & ostr, Writer && writer) const;
template <typename Reader>
void deserializeTextImpl(IColumn & column, ReadBuffer & istr, bool need_safe_get_int_key, Reader && reader) const;
}; };
} }

View File

@ -2326,7 +2326,7 @@ private:
const auto * nested_tuple = typeid_cast<const DataTypeTuple *>(from_array->getNestedType().get()); const auto * nested_tuple = typeid_cast<const DataTypeTuple *>(from_array->getNestedType().get());
if (!nested_tuple || nested_tuple->getElements().size() != 2) if (!nested_tuple || nested_tuple->getElements().size() != 2)
throw Exception{"CAST AS Map from array requeires nested tuple of 2 elements.\n" throw Exception{"CAST AS Map from array requeires nested tuple of 2 elements.\n"
"Left type: " + from_tuple->getName() + ", right type: " + to_type->getName(), ErrorCodes::TYPE_MISMATCH}; "Left type: " + from_array->getName() + ", right type: " + to_type->getName(), ErrorCodes::TYPE_MISMATCH};
return createArrayToMapWrrapper(nested_tuple->getElements(), to_type->getKeyValueTypes()); return createArrayToMapWrrapper(nested_tuple->getElements(), to_type->getKeyValueTypes());
} }

View File

@ -56,8 +56,8 @@ public:
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{ {
if (arguments.empty() || arguments.size() % 2 != 0) if (arguments.size() % 2 != 0)
throw Exception("Function " + getName() + " requires at least one argument.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); throw Exception("Function " + getName() + " even number of arguments.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
DataTypes keys, values; DataTypes keys, values;
for (size_t i = 0; i < arguments.size(); i += 2) for (size_t i = 0; i < arguments.size(); i += 2)

View File

@ -0,0 +1,46 @@
JSON
{
"meta":
[
{
"name": "m",
"type": "Map(String,UInt32)"
},
{
"name": "m1",
"type": "Map(String,Date)"
},
{
"name": "m2",
"type": "Map(String,Array(UInt32))"
}
],
"data":
[
{
"m": {"k1":1,"k2":2,"k3":3},
"m1": {"k1":"2020-05-05"},
"m2": {"k1":[],"k2":[7,8]}
},
{
"m": {"k1":10,"k3":30},
"m1": {"k2":"2020-06-06"},
"m2": {}
}
],
"rows": 2
}
JSONEachRow
{"m":{"k1":10,"k3":30},"m1":{"k2":"2020-06-06"},"m2":{}}
{"m":{"k1":1,"k2":2,"k3":3},"m1":{"k1":"2020-05-05"},"m2":{"k1":[],"k2":[7,8]}}
CSV
"{'k1':1,'k2':2,'k3':3}","{'k1':'2020-05-05'}","{'k1':[],'k2':[7,8]}"
"{'k1':10,'k3':30}","{'k2':'2020-06-06'}","{}"
TSV
{'k1':1,'k2':2,'k3':3} {'k1':'2020-05-05'} {'k1':[],'k2':[7,8]}
{'k1':10,'k3':30} {'k2':'2020-06-06'} {}
TSKV
m={'k1':10,'k3':30} m1={'k2':'2020-06-06'} m2={}
m={'k1':1,'k2':2,'k3':3} m1={'k1':'2020-05-05'} m2={'k1':[],'k2':[7,8]}

View File

@ -0,0 +1,19 @@
SET allow_experimental_map_type = 1;
SET output_format_write_statistics = 0;
DROP TABLE IF EXISTS map_formats;
CREATE TABLE map_formats (m Map(String, UInt32), m1 Map(String, Date), m2 Map(String, Array(UInt32))) ENGINE = Log;
INSERT INTO map_formats VALUES(map('k1', 1, 'k2', 2, 'k3', 3), map('k1', toDate('2020-05-05')), map('k1', [], 'k2', [7, 8]));
INSERT INTO map_formats VALUES(map('k1', 10, 'k3', 30), map('k2', toDate('2020-06-06')), map());
SELECT 'JSON';
SELECT * FROM map_formats FORMAT JSON;
SELECT 'JSONEachRow';
SELECT * FROM map_formats FORMAT JSONEachRow;
SELECT 'CSV';
SELECT * FROM map_formats FORMAT CSV;
SELECT 'TSV';
SELECT * FROM map_formats FORMAT TSV;
SELECT 'TSKV';
SELECT * FROM map_formats FORMAT TSKV;