thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix serialization of type Map to JSON

Browse Source
This commit is contained in:
Anton Popov 2021-07-07 15:04:07 +03:00
parent 9b01ded17a
commit a24686d300
4 changed files with 46 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
src/DataTypes/Serializations/SerializationMap.cpp
View File

@ -80,8 +80,13 @@ void SerializationMap::deserializeBinary(IColumn & column, ReadBuffer & istr) co
}
template <typename Writer>
void SerializationMap::serializeTextImpl(const IColumn & column, size_t row_num, bool quote_key, WriteBuffer & ostr, Writer && writer) const
template <typename KeyWriter, typename ValueWriter>
void SerializationMap::serializeTextImpl(
const IColumn & column,
size_t row_num,
WriteBuffer & ostr,
KeyWriter && key_writer,
ValueWriter && value_writer) const
{
const auto & column_map = assert_cast<const ColumnMap &>(column);
@ -98,17 +103,9 @@ void SerializationMap::serializeTextImpl(const IColumn & column, size_t row_num,
if (i != offset)
writeChar(',', ostr);
if (quote_key)
{
writeChar('"', ostr);
writer(key, nested_tuple.getColumn(0), i);
writeChar('"', ostr);
}
else
writer(key, nested_tuple.getColumn(0), i);
key_writer(key, nested_tuple.getColumn(0), i);
writeChar(':', ostr);
writer(value, nested_tuple.getColumn(1), i);
value_writer(value, nested_tuple.getColumn(1), i);
}
writeChar('}', ostr);
}
@ -170,11 +167,12 @@ void SerializationMap::deserializeTextImpl(IColumn & column, ReadBuffer & istr,
void SerializationMap::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
serializeTextImpl(column, row_num, /*quote_key=*/ false, ostr,
[&](const SerializationPtr & subcolumn_serialization, const IColumn & subcolumn, size_t pos)
auto writer = [&](const SerializationPtr & subcolumn_serialization, const IColumn & subcolumn, size_t pos)
{
subcolumn_serialization->serializeTextQuoted(subcolumn, pos, ostr, settings);
});
};
serializeTextImpl(column, row_num, ostr, writer, writer);
}
void SerializationMap::deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
@ -188,11 +186,14 @@ void SerializationMap::deserializeText(IColumn & column, ReadBuffer & istr, cons
void SerializationMap::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const
{
/// We need to double-quote integer keys to produce valid JSON.
const auto & column_key = assert_cast<const ColumnMap &>(column).getNestedData().getColumn(0);
bool quote_key = !WhichDataType(column_key.getDataType()).isStringOrFixedString();
serializeTextImpl(column, row_num, quote_key, ostr,
serializeTextImpl(column, row_num, ostr,
[&](const SerializationPtr & subcolumn_serialization, const IColumn & subcolumn, size_t pos)
{
/// We need to double-quote all keys (including integers) to produce valid JSON.
WriteBufferFromOwnString str_buf;
subcolumn_serialization->serializeText(subcolumn, pos, str_buf, settings);
writeJSONString(str_buf.str(), ostr, settings);
},
[&](const SerializationPtr & subcolumn_serialization, const IColumn & subcolumn, size_t pos)
{
subcolumn_serialization->serializeTextJSON(subcolumn, pos, ostr, settings);

4
src/DataTypes/Serializations/SerializationMap.h
View File

@ -60,8 +60,8 @@ public:
SubstreamsCache * cache) const override;
private:
template <typename Writer>
void serializeTextImpl(const IColumn & column, size_t row_num, bool quote_key, WriteBuffer & ostr, Writer && writer) const;
template <typename KeyWriter, typename ValueWriter>
void serializeTextImpl(const IColumn & column, size_t row_num, WriteBuffer & ostr, KeyWriter && key_writer, ValueWriter && value_writer) const;
template <typename Reader>
void deserializeTextImpl(IColumn & column, ReadBuffer & istr, Reader && reader) const;

10
tests/queries/0_stateless/01939_type_map_json.reference
View File

@ -4,5 +4,11 @@
{'key1':1,'key2':2} {"key1":"1","key2":"2"} 1
{"m":{"key1":1,"key2":2}}
{'key1':1,'key2':2} {"key1":1,"key2":2} 1
{"m1":{"k1":"1","k2":"2"},"m2":{"1":2,"2":3}}
{"m1":{"k1":1,"k2":2},"m2":{"1":2,"2":3}}
{"m":{"2020-10-10":"v1","2020-10-11":"v2"}}
{'2020-10-10':'v1','2020-10-11':'v2'} {"2020-10-10":"v1","2020-10-11":"v2"} 1
{"m":{"11":"v1","22":"v2"}}
{11:'v1',22:'v2'} {"11":"v1","22":"v2"} 1
{"m":{"11":"v1","22":"v2"}}
{11:'v1',22:'v2'} {"11":"v1","22":"v2"} 1
{"m1":{"k1":"1","k2":"2"},"m2":{"1":2,"2":3},"m3":{"2020-10-10":"foo"}}
{"m1":{"k1":1,"k2":2},"m2":{"1":2,"2":3},"m3":{"2020-10-10":"foo"}}

17
tests/queries/0_stateless/01939_type_map_json.sql
View File

@ -11,9 +11,18 @@ SELECT map('key1', number, 'key2', number * 2) AS m FROM numbers(1, 1)
SELECT map('key1', number, 'key2', number * 2) AS m, toJSONString(m) AS s, isValidJSON(s) FROM numbers(1, 1)
SETTINGS output_format_json_quote_64bit_integers = 0;
CREATE TEMPORARY TABLE map_json (m1 Map(String, UInt64), m2 Map(UInt32, UInt32));
SELECT map('2020-10-10'::Date, 'v1', '2020-10-11'::Date, 'v2') AS m FORMAT JSONEachRow;
SELECT map('2020-10-10'::Date, 'v1', '2020-10-11'::Date, 'v2') AS m, toJSONString(m) AS s, isValidJSON(s);
INSERT INTO map_json FORMAT JSONEachRow {"m1" : {"k1" : 1, "k2" : 2}, "m2" : {"1" : 2, "2" : 3}};
SELECT map(11::UInt64, 'v1', 22::UInt64, 'v2') AS m FORMAT JSONEachRow;
SELECT map(11::UInt64, 'v1', 22::UInt64, 'v2') AS m, toJSONString(m) AS s, isValidJSON(s);
SELECT m1, m2 FROM map_json FORMAT JSONEachRow;
SELECT m1, m2 FROM map_json FORMAT JSONEachRow SETTINGS output_format_json_quote_64bit_integers = 0;
SELECT map(11::Int128, 'v1', 22::Int128, 'v2') AS m FORMAT JSONEachRow;
SELECT map(11::Int128, 'v1', 22::Int128, 'v2') AS m, toJSONString(m) AS s, isValidJSON(s);
CREATE TEMPORARY TABLE map_json (m1 Map(String, UInt64), m2 Map(UInt32, UInt32), m3 Map(Date, String));
INSERT INTO map_json FORMAT JSONEachRow {"m1" : {"k1" : 1, "k2" : 2}, "m2" : {"1" : 2, "2" : 3}, "m3" : {"2020-10-10" : "foo"}};
SELECT m1, m2, m3 FROM map_json FORMAT JSONEachRow;
SELECT m1, m2, m3 FROM map_json FORMAT JSONEachRow SETTINGS output_format_json_quote_64bit_integers = 0;