thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-24 00:22:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #126 from ludv1x/METR-22950

Browse Source 
output_format_json_quote_64bit_integers new config option [#METR-22950]
This commit is contained in:
alexey-milovidov 2016-10-01 00:10:28 +04:00 committed by GitHub
commit 6f3e69c12d
29 changed files with 355 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dbms/include/DB/DataStreams/JSONCompactRowOutputStream.h
View File

@ -14,7 +14,7 @@ namespace DB
class JSONCompactRowOutputStream : public JSONRowOutputStream
{
public:
JSONCompactRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_);
JSONCompactRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_, bool force_quoting_64bit_integers_);
void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
void writeFieldDelimiter() override;

3
dbms/include/DB/DataStreams/JSONEachRowRowOutputStream.h
View File

@ -14,7 +14,7 @@ namespace DB
class JSONEachRowRowOutputStream : public IRowOutputStream
{
public:
JSONEachRowRowOutputStream(WriteBuffer & ostr_, const Block & sample);
JSONEachRowRowOutputStream(WriteBuffer & ostr_, const Block & sample, bool force_quoting_64bit_integers_);
void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
void writeFieldDelimiter() override;
@ -30,6 +30,7 @@ private:
WriteBuffer & ostr;
size_t field_number = 0;
Names fields;
bool force_quoting_64bit_integers;
};
}

3
dbms/include/DB/DataStreams/JSONRowOutputStream.h
View File

@ -16,7 +16,7 @@ class JSONRowOutputStream : public IRowOutputStream
{
public:
JSONRowOutputStream(WriteBuffer & ostr_, const Block & sample_,
bool write_statistics_);
bool write_statistics_, bool force_quoting_64bit_integers_);
void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
void writeFieldDelimiter() override;
@ -68,6 +68,7 @@ protected:
Progress progress;
Stopwatch watch;
bool write_statistics;
bool force_quoting_64bit_integers;
};
}

2
dbms/include/DB/DataTypes/DataTypeAggregateFunction.h
View File

@ -53,7 +53,7 @@ public:
void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override;
void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override;
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool) const override;
void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override;
void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;

2
dbms/include/DB/DataTypes/DataTypeArray.h
View File

@ -46,7 +46,7 @@ public:
void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override;
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool) const override;
void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override;
void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;

2
dbms/include/DB/DataTypes/DataTypeDate.h
View File

@ -58,7 +58,7 @@ public:
static_cast<ColumnType &>(column).getData().push_back(x); /// Важно делать это в конце - для exception safety.
}
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool) const override
{
writeChar('"', ostr);
serializeText(column, row_num, ostr);

2
dbms/include/DB/DataTypes/DataTypeDateTime.h
View File

@ -58,7 +58,7 @@ public:
static_cast<ColumnType &>(column).getData().push_back(x); /// Важно делать это в конце - для exception safety.
}
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool) const override
{
writeChar('"', ostr);
serializeText(column, row_num, ostr);

2
dbms/include/DB/DataTypes/DataTypeEnum.h
View File

@ -81,7 +81,7 @@ public:
void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override;
void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override;
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool) const override;
void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override;
void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;

2
dbms/include/DB/DataTypes/DataTypeFixedString.h
View File

@ -52,7 +52,7 @@ public:
void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override;
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool) const override;
void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override;
void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;

2
dbms/include/DB/DataTypes/DataTypeString.h
View File

@ -39,7 +39,7 @@ public:
void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override;
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool) const override;
void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override;
void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;

2
dbms/include/DB/DataTypes/DataTypeTuple.h
View File

@ -31,7 +31,7 @@ public:
void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override;
void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override;
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool) const override;
void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override;
void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;

5
dbms/include/DB/DataTypes/IDataType.h
View File

@ -87,9 +87,10 @@ public:
*/
virtual void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr) const = 0;
/** Текстовая сериализация в виде литерала для использования в формате JSON.
/** Text serialization intended for using in JSON format.
* force_quoting_64bit_integers parameter forces to brace UInt64 and Int64 types into quotes.
*/
virtual void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const = 0;
virtual void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool force_quoting_64bit_integers) const = 0;
virtual void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const = 0;
/** Текстовая сериализация для подстановки в формат XML.

2
dbms/include/DB/DataTypes/IDataTypeDummy.h
View File

@ -38,7 +38,7 @@ public:
void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override { throwNoSerialization(); }
void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override { throwNoSerialization(); }
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override { throwNoSerialization(); }
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool) const override { throwNoSerialization(); }
void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override { throwNoSerialization(); }
void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override { throwNoSerialization(); }

26
dbms/include/DB/DataTypes/IDataTypeNumber.h
View File

@ -56,7 +56,7 @@ public:
deserializeText(column, istr);
}
inline void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override;
inline void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool) const override;
void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override
{
@ -121,7 +121,7 @@ public:
void deserializeTextEscaped(IColumn & column, ReadBuffer & istr) const override {}
void serializeTextQuoted(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override {}
void deserializeTextQuoted(IColumn & column, ReadBuffer & istr) const override {}
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override {}
void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool) const override {}
void deserializeTextJSON(IColumn & column, ReadBuffer & istr) const override {}
void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr) const override {}
void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const char delimiter) const override {}
@ -129,26 +129,30 @@ public:
Field getDefault() const override { return {}; }
};
template <typename FType> inline void IDataTypeNumber<FType>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
template <typename FType> inline void IDataTypeNumber<FType>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool) const
{
serializeText(column, row_num, ostr);
}
template <> inline void IDataTypeNumber<Int64>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
template <> inline void IDataTypeNumber<Int64>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool force_quoting_64bit_integers) const
{
writeChar('"', ostr);
if (force_quoting_64bit_integers)
writeChar('"', ostr);
serializeText(column, row_num, ostr);
writeChar('"', ostr);
if (force_quoting_64bit_integers)
writeChar('"', ostr);
}
template <> inline void IDataTypeNumber<UInt64>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
template <> inline void IDataTypeNumber<UInt64>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool force_quoting_64bit_integers) const
{
writeChar('"', ostr);
if (force_quoting_64bit_integers)
writeChar('"', ostr);
serializeText(column, row_num, ostr);
writeChar('"', ostr);
if (force_quoting_64bit_integers)
writeChar('"', ostr);
}
template <> inline void IDataTypeNumber<Float32>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
template <> inline void IDataTypeNumber<Float32>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool) const
{
auto x = static_cast<const ColumnType &>(column).getData()[row_num];
if (likely(std::isfinite(x)))
@ -157,7 +161,7 @@ template <> inline void IDataTypeNumber<Float32>::serializeTextJSON(const IColum
writeCString("null", ostr);
}
template <> inline void IDataTypeNumber<Float64>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
template <> inline void IDataTypeNumber<Float64>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool) const
{
auto x = static_cast<const ColumnType &>(column).getData()[row_num];
if (likely(std::isfinite(x)))

5
dbms/include/DB/Interpreters/Settings.h
View File

@ -225,7 +225,10 @@ struct Settings
M(SettingBool, add_http_cors_header, false) \
\
/** Skip columns with unknown names from input data (it works for JSONEachRow and TSKV formats). */ \
M(SettingBool, input_format_skip_unknown_fields, false)
M(SettingBool, input_format_skip_unknown_fields, false) \
\
/** Controls quoting of 64-bit integers in JSON output format. */ \
M(SettingBool, output_format_json_quote_64bit_integers, true)
/// Всевозможные ограничения на выполнение запроса.
Limits limits;

7
dbms/src/DataStreams/FormatFactory.cpp
View File

@ -135,12 +135,13 @@ static BlockOutputStreamPtr getOutputImpl(const String & name, WriteBuffer & buf
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<ValuesRowOutputStream>(buf));
else if (name == "JSON")
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONRowOutputStream>(buf, sample,
context.getSettingsRef().output_format_write_statistics));
context.getSettingsRef().output_format_write_statistics, context.getSettingsRef().output_format_json_quote_64bit_integers));
else if (name == "JSONCompact")
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONCompactRowOutputStream>(buf, sample,
context.getSettingsRef().output_format_write_statistics));
context.getSettingsRef().output_format_write_statistics, context.getSettingsRef().output_format_json_quote_64bit_integers));
else if (name == "JSONEachRow")
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONEachRowRowOutputStream>(buf, sample));
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONEachRowRowOutputStream>(buf, sample,
context.getSettingsRef().output_format_json_quote_64bit_integers));
else if (name == "XML")
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<XMLRowOutputStream>(buf, sample,
context.getSettingsRef().output_format_write_statistics));

16
dbms/src/DataStreams/JSONCompactRowOutputStream.cpp
View File

@ -6,15 +6,15 @@
namespace DB
{
JSONCompactRowOutputStream::JSONCompactRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_)
: JSONRowOutputStream(ostr_, sample_, write_statistics_)
JSONCompactRowOutputStream::JSONCompactRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_, bool force_quoting_64bit_integers_)
: JSONRowOutputStream(ostr_, sample_, write_statistics_, force_quoting_64bit_integers_)
{
}
void JSONCompactRowOutputStream::writeField(const IColumn & column, const IDataType & type, size_t row_num)
{
type.serializeTextJSON(column, row_num, *ostr);
type.serializeTextJSON(column, row_num, *ostr, force_quoting_64bit_integers);
++field_number;
}
@ -56,7 +56,7 @@ void JSONCompactRowOutputStream::writeTotals()
writeChar(',', *ostr);
const ColumnWithTypeAndName & column = totals.getByPosition(i);
column.type->serializeTextJSON(*column.column.get(), 0, *ostr);
column.type->serializeTextJSON(*column.column.get(), 0, *ostr, force_quoting_64bit_integers);
}
writeChar(']', *ostr);
@ -64,7 +64,7 @@ void JSONCompactRowOutputStream::writeTotals()
}
static void writeExtremesElement(const char * title, const Block & extremes, size_t row_num, WriteBuffer & ostr)
static void writeExtremesElement(const char * title, const Block & extremes, size_t row_num, WriteBuffer & ostr, bool force_quoting_64bit_integers)
{
writeCString("\t\t\"", ostr);
writeCString(title, ostr);
@ -77,7 +77,7 @@ static void writeExtremesElement(const char * title, const Block & extremes, siz
writeChar(',', ostr);
const ColumnWithTypeAndName & column = extremes.getByPosition(i);
column.type->serializeTextJSON(*column.column.get(), row_num, ostr);
column.type->serializeTextJSON(*column.column.get(), row_num, ostr, force_quoting_64bit_integers);
}
writeChar(']', ostr);
@ -92,9 +92,9 @@ void JSONCompactRowOutputStream::writeExtremes()
writeCString("\t\"extremes\":\n", *ostr);
writeCString("\t{\n", *ostr);
writeExtremesElement("min", extremes, 0, *ostr);
writeExtremesElement("min", extremes, 0, *ostr, force_quoting_64bit_integers);
writeCString(",\n", *ostr);
writeExtremesElement("max", extremes, 1, *ostr);
writeExtremesElement("max", extremes, 1, *ostr, force_quoting_64bit_integers);
writeChar('\n', *ostr);
writeCString("\t}", *ostr);

6
dbms/src/DataStreams/JSONEachRowRowOutputStream.cpp
View File

@ -7,8 +7,8 @@ namespace DB
{
JSONEachRowRowOutputStream::JSONEachRowRowOutputStream(WriteBuffer & ostr_, const Block & sample)
: ostr(ostr_)
JSONEachRowRowOutputStream::JSONEachRowRowOutputStream(WriteBuffer & ostr_, const Block & sample, bool force_quoting_)
: ostr(ostr_), force_quoting(force_quoting_)
{
size_t columns = sample.columns();
fields.resize(columns);
@ -25,7 +25,7 @@ void JSONEachRowRowOutputStream::writeField(const IColumn & column, const IDataT
{
writeString(fields[field_number], ostr);
writeChar(':', ostr);
type.serializeTextJSON(column, row_num, ostr);
type.serializeTextJSON(column, row_num, ostr, force_quoting);
++field_number;
}

16
dbms/src/DataStreams/JSONRowOutputStream.cpp
View File

@ -7,8 +7,8 @@ namespace DB
{
JSONRowOutputStream::JSONRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_)
: dst_ostr(ostr_), write_statistics(write_statistics_)
JSONRowOutputStream::JSONRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_, bool force_quoting_)
: dst_ostr(ostr_), write_statistics(write_statistics_), force_quoting(force_quoting_)
{
NamesAndTypesList columns(sample_.getColumnsList());
fields.assign(columns.begin(), columns.end());
@ -72,7 +72,7 @@ void JSONRowOutputStream::writeField(const IColumn & column, const IDataType & t
writeCString("\t\t\t", *ostr);
writeString(fields[field_number].name, *ostr);
writeCString(": ", *ostr);
type.serializeTextJSON(column, row_num, *ostr);
type.serializeTextJSON(column, row_num, *ostr, force_quoting);
++field_number;
}
@ -152,7 +152,7 @@ void JSONRowOutputStream::writeTotals()
writeCString("\t\t", *ostr);
writeJSONString(column.name, *ostr);
writeCString(": ", *ostr);
column.type->serializeTextJSON(*column.column.get(), 0, *ostr);
column.type->serializeTextJSON(*column.column.get(), 0, *ostr, force_quoting);
}
writeChar('\n', *ostr);
@ -161,7 +161,7 @@ void JSONRowOutputStream::writeTotals()
}
static void writeExtremesElement(const char * title, const Block & extremes, size_t row_num, WriteBuffer & ostr)
static void writeExtremesElement(const char * title, const Block & extremes, size_t row_num, WriteBuffer & ostr, bool force_quoting)
{
writeCString("\t\t\"", ostr);
writeCString(title, ostr);
@ -179,7 +179,7 @@ static void writeExtremesElement(const char * title, const Block & extremes, siz
writeCString("\t\t\t", ostr);
writeJSONString(column.name, ostr);
writeCString(": ", ostr);
column.type->serializeTextJSON(*column.column.get(), row_num, ostr);
column.type->serializeTextJSON(*column.column.get(), row_num, ostr, force_quoting);
}
writeChar('\n', ostr);
@ -195,9 +195,9 @@ void JSONRowOutputStream::writeExtremes()
writeCString("\t\"extremes\":\n", *ostr);
writeCString("\t{\n", *ostr);
writeExtremesElement("min", extremes, 0, *ostr);
writeExtremesElement("min", extremes, 0, *ostr, force_quoting);
writeCString(",\n", *ostr);
writeExtremesElement("max", extremes, 1, *ostr);
writeExtremesElement("max", extremes, 1, *ostr, force_quoting);
writeChar('\n', *ostr);
writeCString("\t}", *ostr);

2
dbms/src/DataTypes/DataTypeAggregateFunction.cpp
View File

@ -195,7 +195,7 @@ void DataTypeAggregateFunction::deserializeTextQuoted(IColumn & column, ReadBuff
}
void DataTypeAggregateFunction::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
void DataTypeAggregateFunction::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool) const
{
writeJSONString(serializeToString(function, column, row_num), ostr);
}

4
dbms/src/DataTypes/DataTypeArray.cpp
View File

@ -288,7 +288,7 @@ void DataTypeArray::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) c
}
void DataTypeArray::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
void DataTypeArray::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool force_quoting_64bit_integers) const
{
const ColumnArray & column_array = static_cast<const ColumnArray &>(column);
const ColumnArray::Offsets_t & offsets = column_array.getOffsets();
@ -303,7 +303,7 @@ void DataTypeArray::serializeTextJSON(const IColumn & column, size_t row_num, Wr
{
if (i != offset)
writeChar(',', ostr);
nested->serializeTextJSON(nested_column, i, ostr);
nested->serializeTextJSON(nested_column, i, ostr, force_quoting_64bit_integers);
}
writeChar(']', ostr);
}

2
dbms/src/DataTypes/DataTypeEnum.cpp
View File

@ -170,7 +170,7 @@ void DataTypeEnum<Type>::deserializeTextQuoted(IColumn & column, ReadBuffer & is
}
template <typename Type>
void DataTypeEnum<Type>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
void DataTypeEnum<Type>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool) const
{
writeJSONString(getNameForValue(static_cast<const ColumnType &>(column).getData()[row_num]), ostr);
}

2
dbms/src/DataTypes/DataTypeFixedString.cpp
View File

@ -158,7 +158,7 @@ void DataTypeFixedString::deserializeTextQuoted(IColumn & column, ReadBuffer & i
}
void DataTypeFixedString::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
void DataTypeFixedString::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool) const
{
const char * pos = reinterpret_cast<const char *>(&static_cast<const ColumnFixedString &>(column).getChars()[n * row_num]);
writeJSONString(pos, pos + n, ostr);

2
dbms/src/DataTypes/DataTypeString.cpp
View File

@ -266,7 +266,7 @@ void DataTypeString::deserializeTextQuoted(IColumn & column, ReadBuffer & istr)
}
void DataTypeString::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
void DataTypeString::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool) const
{
writeJSONString(static_cast<const ColumnString &>(column).getDataAt(row_num), ostr);
}

4
dbms/src/DataTypes/DataTypeTuple.cpp
View File

@ -144,14 +144,14 @@ void DataTypeTuple::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) c
deserializeText(column, istr);
}
void DataTypeTuple::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr) const
void DataTypeTuple::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool force_quoting_64bit_integers) const
{
writeChar('[', ostr);
for (const auto i : ext::range(0, ext::size(elems)))
{
if (i != 0)
writeChar(',', ostr);
elems[i]->serializeTextJSON(extractElementColumn(column, i), row_num, ostr);
elems[i]->serializeTextJSON(extractElementColumn(column, i), row_num, ostr, force_quoting_64bit_integers);
}
writeChar(']', ostr);
}

264
dbms/tests/queries/0_stateless/00378_json_quote_64bit_integers.reference Normal file
View File

@ -0,0 +1,264 @@
{
"meta":
[
{
"name": "i0",
"type": "Int64"
},
{
"name": "u0",
"type": "UInt64"
},
{
"name": "ip",
"type": "Int64"
},
{
"name": "in",
"type": "Int64"
},
{
"name": "up",
"type": "UInt64"
},
{
"name": "arr",
"type": "Array(Int64)"
},
{
"name": "tuple",
"type": "Tuple(UInt64, UInt64)"
}
],
"data":
[
{
"i0": "0",
"u0": "0",
"ip": "9223372036854775807",
"in": "-9223372036854775808",
"up": "18446744073709551615",
"arr": ["0"],
"tuple": ["0","0"]
}
],
"totals":
{
"i0": "0",
"u0": "0",
"ip": "0",
"in": "0",
"up": "0",
"arr": [],
"tuple": ["0","0"]
},
"extremes":
{
"min":
{
"i0": "0",
"u0": "0",
"ip": "9223372036854775807",
"in": "-9223372036854775808",
"up": "18446744073709551615",
"arr": [],
"tuple": ["0","0"]
},
"max":
{
"i0": "0",
"u0": "0",
"ip": "9223372036854775807",
"in": "-9223372036854775808",
"up": "18446744073709551615",
"arr": [],
"tuple": ["0","0"]
}
},
"rows": 1
}
{
"meta":
[
{
"name": "i0",
"type": "Int64"
},
{
"name": "u0",
"type": "UInt64"
},
{
"name": "ip",
"type": "Int64"
},
{
"name": "in",
"type": "Int64"
},
{
"name": "up",
"type": "UInt64"
},
{
"name": "arr",
"type": "Array(Int64)"
},
{
"name": "tuple",
"type": "Tuple(UInt64, UInt64)"
}
],
"data":
[
["0", "0", "9223372036854775807", "-9223372036854775808", "18446744073709551615", ["0"], ["0","0"]]
],
"totals": ["0","0","0","0","0",[],["0","0"]],
"extremes":
{
"min": ["0","0","9223372036854775807","-9223372036854775808","18446744073709551615",[],["0","0"]],
"max": ["0","0","9223372036854775807","-9223372036854775808","18446744073709551615",[],["0","0"]]
},
"rows": 1
}
{"i0":"0","u0":"0","ip":"9223372036854775807","in":"-9223372036854775808","up":"18446744073709551615","arr":["0"],"tuple":["0","0"]}
{
"meta":
[
{
"name": "i0",
"type": "Int64"
},
{
"name": "u0",
"type": "UInt64"
},
{
"name": "ip",
"type": "Int64"
},
{
"name": "in",
"type": "Int64"
},
{
"name": "up",
"type": "UInt64"
},
{
"name": "arr",
"type": "Array(Int64)"
},
{
"name": "tuple",
"type": "Tuple(UInt64, UInt64)"
}
],
"data":
[
{
"i0": 0,
"u0": 0,
"ip": 9223372036854775807,
"in": -9223372036854775808,
"up": 18446744073709551615,
"arr": [0],
"tuple": [0,0]
}
],
"totals":
{
"i0": 0,
"u0": 0,
"ip": 0,
"in": 0,
"up": 0,
"arr": [],
"tuple": [0,0]
},
"extremes":
{
"min":
{
"i0": 0,
"u0": 0,
"ip": 9223372036854775807,
"in": -9223372036854775808,
"up": 18446744073709551615,
"arr": [],
"tuple": [0,0]
},
"max":
{
"i0": 0,
"u0": 0,
"ip": 9223372036854775807,
"in": -9223372036854775808,
"up": 18446744073709551615,
"arr": [],
"tuple": [0,0]
}
},
"rows": 1
}
{
"meta":
[
{
"name": "i0",
"type": "Int64"
},
{
"name": "u0",
"type": "UInt64"
},
{
"name": "ip",
"type": "Int64"
},
{
"name": "in",
"type": "Int64"
},
{
"name": "up",
"type": "UInt64"
},
{
"name": "arr",
"type": "Array(Int64)"
},
{
"name": "tuple",
"type": "Tuple(UInt64, UInt64)"
}
],
"data":
[
[0, 0, 9223372036854775807, -9223372036854775808, 18446744073709551615, [0], [0,0]]
],
"totals": [0,0,0,0,0,[],[0,0]],
"extremes":
{
"min": [0,0,9223372036854775807,-9223372036854775808,18446744073709551615,[],[0,0]],
"max": [0,0,9223372036854775807,-9223372036854775808,18446744073709551615,[],[0,0]]
},
"rows": 1
}
{"i0":0,"u0":0,"ip":9223372036854775807,"in":-9223372036854775808,"up":18446744073709551615,"arr":[0],"tuple":[0,0]}

12
dbms/tests/queries/0_stateless/00378_json_quote_64bit_integers.sql Normal file
View File

@ -0,0 +1,12 @@
SET output_format_write_statistics = 0;
SET extremes = 1;
SET output_format_json_quote_64bit_integers = 1;
SELECT toInt64(0) as i0, toUInt64(0) as u0, toInt64(9223372036854775807) as ip, toInt64(-9223372036854775808) as in, toUInt64(18446744073709551615) as up, [toInt64(0)] as arr, (toUInt64(0), toUInt64(0)) as tuple WITH TOTALS FORMAT JSON;
SELECT toInt64(0) as i0, toUInt64(0) as u0, toInt64(9223372036854775807) as ip, toInt64(-9223372036854775808) as in, toUInt64(18446744073709551615) as up, [toInt64(0)] as arr, (toUInt64(0), toUInt64(0)) as tuple WITH TOTALS FORMAT JSONCompact;
SELECT toInt64(0) as i0, toUInt64(0) as u0, toInt64(9223372036854775807) as ip, toInt64(-9223372036854775808) as in, toUInt64(18446744073709551615) as up, [toInt64(0)] as arr, (toUInt64(0), toUInt64(0)) as tuple WITH TOTALS FORMAT JSONEachRow;
SET output_format_json_quote_64bit_integers = 0;
SELECT toInt64(0) as i0, toUInt64(0) as u0, toInt64(9223372036854775807) as ip, toInt64(-9223372036854775808) as in, toUInt64(18446744073709551615) as up, [toInt64(0)] as arr, (toUInt64(0), toUInt64(0)) as tuple WITH TOTALS FORMAT JSON;
SELECT toInt64(0) as i0, toUInt64(0) as u0, toInt64(9223372036854775807) as ip, toInt64(-9223372036854775808) as in, toUInt64(18446744073709551615) as up, [toInt64(0)] as arr, (toUInt64(0), toUInt64(0)) as tuple WITH TOTALS FORMAT JSONCompact;
SELECT toInt64(0) as i0, toUInt64(0) as u0, toInt64(9223372036854775807) as ip, toInt64(-9223372036854775808) as in, toUInt64(18446744073709551615) as up, [toInt64(0)] as arr, (toUInt64(0), toUInt64(0)) as tuple WITH TOTALS FORMAT JSONEachRow;

9
doc/reference_en.html
View File

@ -6616,12 +6616,17 @@ Allows setting a default sampling coefficient for all SELECT queries.
(For tables that don&#39;t support sampling, an exception will be thrown.)
If set to 1, default sampling is not performed.
====input_format_skip_unknown_fields==
==input_format_skip_unknown_fields==
If the parameter is true, INSERT operation will skip columns with unknown names from input.
Otherwise, an exception will be generated.
Otherwise, an exception will be generated, it is default behavior.
The parameter works only for JSONEachRow and TSKV input formats.
==output_format_json_quote_64bit_integers==
If the parameter is true (default value), UInt64 and Int64 numbers are printed as quoted strings in all JSON output formats.
Such behavior is compatible with most JavaScript interpreters that stores all numbers as double-precision floating point numbers.
Otherwise, they are printed as regular numbers.
==Restrictions on query complexity==

6
doc/reference_ru.html
View File

@ -3881,7 +3881,7 @@ Extremes:
}
%%
JSON совместим с JavaScript. Для этого, дополнительно эскейпятся некоторые символы: символ прямого слеша %%/%% экранируется в виде %%\/%%; альтернативные переводы строк %%U+2028%%, %%U+2029%%, на которых ломаются некоторые браузеры, экранируются в виде <span class="inline-example">\u<i>XXXX</i></span>-последовательностей. Эскейпятся ASCII control characters: backspace, form feed, line feed, carriage return, horizontal tab в виде %%\b%%, %%\f%%, %%\n%%, %%\r%%, %%\t%% соответственно, а также остальные байты из диапазона 00-1F с помощью <span class="inline-example">\u<i>XXXX</i></span>-последовательностей. Невалидные UTF-8 последовательности заменяются на replacement character %%<25>%% и, таким образом, выводимый текст будет состоять из валидных UTF-8 последовательностей. Числа типа UInt64 и Int64, для совместимости с JavaScript, выводятся в двойных кавычках.
JSON совместим с JavaScript. Для этого, дополнительно эскейпятся некоторые символы: символ прямого слеша %%/%% экранируется в виде %%\/%%; альтернативные переводы строк %%U+2028%%, %%U+2029%%, на которых ломаются некоторые браузеры, экранируются в виде <span class="inline-example">\u<i>XXXX</i></span>-последовательностей. Эскейпятся ASCII control characters: backspace, form feed, line feed, carriage return, horizontal tab в виде %%\b%%, %%\f%%, %%\n%%, %%\r%%, %%\t%% соответственно, а также остальные байты из диапазона 00-1F с помощью <span class="inline-example">\u<i>XXXX</i></span>-последовательностей. Невалидные UTF-8 последовательности заменяются на replacement character %%<25>%% и, таким образом, выводимый текст будет состоять из валидных UTF-8 последовательностей. Числа типа UInt64 и Int64, для совместимости с JavaScript, по-умолчанию выводятся в двойных кавычках, чтобы они выводились без кавычек можно установить конфигурационный параметр output_format_json_quote_64bit_integers равным 0.
%%rows%% - общее количество выведенных строчек.
%%rows_before_limit_at_least%% - не менее скольких строчек получилось бы, если бы не было LIMIT-а. Выводится только если запрос содержит LIMIT.
@ -6775,6 +6775,10 @@ regions_names_*.txt: TabSeparated (без заголовка), столбцы:
Если значение истино, то при выполнении INSERT из входных данных пропускаются (не рассматриваются) колонки с неизвестными именами, иначе в данной ситуации будет сгенерировано исключение.
Работает для форматов JSONEachRow и TSKV.
==output_format_json_quote_64bit_integers==
Если значение истино, то при использовании JSON* форматов UInt64 и Int64 числа выводятся в кавычках (из соображений совместимости с большинством реализаций JavaScript), иначе - без кавычек.
==Ограничения на сложность запроса==