thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-23 08:02:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

updated FormatSettingsJSON

Browse Source
This commit is contained in:
Nikolai Kochetov 2017-07-05 19:28:57 +03:00 committed by alexey-milovidov
parent 0ead0ee2f1
commit 63119eda7d
10 changed files with 65 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
dbms/src/DataStreams/FormatFactory.cpp
View File

@ -29,7 +29,7 @@
#include <DataStreams/CSVRowOutputStream.h>
#include <DataStreams/MaterializingBlockOutputStream.h>
#include <DataStreams/FormatFactory.h>
#include <DataTypes/FormatSettingsJSON.h>
namespace DB
{
@ -121,6 +121,7 @@ static BlockOutputStreamPtr getOutputImpl(const String & name, WriteBuffer & buf
const Block & sample, const Context & context)
{
const Settings & settings = context.getSettingsRef();
FormatSettingsJSON json_settings(settings.output_format_json_quote_64bit_integers, true);
if (name == "Native")
return std::make_shared<NativeBlockOutputStream>(buf);
@ -163,14 +164,14 @@ static BlockOutputStreamPtr getOutputImpl(const String & name, WriteBuffer & buf
else if (name == "Values")
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<ValuesRowOutputStream>(buf));
else if (name == "JSON")
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONRowOutputStream>(buf, sample,
settings.output_format_write_statistics, settings.output_format_json_quote_64bit_integers));
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONRowOutputStream>(
buf, sample, settings.output_format_write_statistics, json_settings));
else if (name == "JSONCompact")
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONCompactRowOutputStream>(buf, sample,
settings.output_format_write_statistics, settings.output_format_json_quote_64bit_integers));
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONCompactRowOutputStream>(
buf, sample, settings.output_format_write_statistics, json_settings));
else if (name == "JSONEachRow")
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONEachRowRowOutputStream>(buf, sample,
settings.output_format_json_quote_64bit_integers));
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONEachRowRowOutputStream>(
buf, sample, json_settings));
else if (name == "XML")
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<XMLRowOutputStream>(buf, sample,
settings.output_format_write_statistics));

17
dbms/src/DataStreams/JSONCompactRowOutputStream.cpp
View File

@ -6,15 +6,16 @@
namespace DB
{
JSONCompactRowOutputStream::JSONCompactRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_, bool force_quoting_64bit_integers_)
: JSONRowOutputStream(ostr_, sample_, write_statistics_, force_quoting_64bit_integers_)
JSONCompactRowOutputStream::JSONCompactRowOutputStream(WriteBuffer & ostr_, const Block & sample_,
bool write_statistics_, const FormatSettingsJSON & settings_)
: JSONRowOutputStream(ostr_, sample_, write_statistics_, settings_)
{
}
void JSONCompactRowOutputStream::writeField(const IColumn & column, const IDataType & type, size_t row_num)
{
type.serializeTextJSON(column, row_num, *ostr, force_quoting_64bit_integers);
type.serializeTextJSON(column, row_num, *ostr, settings);
++field_number;
}
@ -56,7 +57,7 @@ void JSONCompactRowOutputStream::writeTotals()
writeChar(',', *ostr);
const ColumnWithTypeAndName & column = totals.safeGetByPosition(i);
column.type->serializeTextJSON(*column.column.get(), 0, *ostr, force_quoting_64bit_integers);
column.type->serializeTextJSON(*column.column.get(), 0, *ostr, settings);
}
writeChar(']', *ostr);
@ -64,7 +65,7 @@ void JSONCompactRowOutputStream::writeTotals()
}
static void writeExtremesElement(const char * title, const Block & extremes, size_t row_num, WriteBuffer & ostr, bool force_quoting_64bit_integers)
static void writeExtremesElement(const char * title, const Block & extremes, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON & settings)
{
writeCString("\t\t\"", ostr);
writeCString(title, ostr);
@ -77,7 +78,7 @@ static void writeExtremesElement(const char * title, const Block & extremes, siz
writeChar(',', ostr);
const ColumnWithTypeAndName & column = extremes.safeGetByPosition(i);
column.type->serializeTextJSON(*column.column.get(), row_num, ostr, force_quoting_64bit_integers);
column.type->serializeTextJSON(*column.column.get(), row_num, ostr, settings);
}
writeChar(']', ostr);
@ -92,9 +93,9 @@ void JSONCompactRowOutputStream::writeExtremes()
writeCString("\t\"extremes\":\n", *ostr);
writeCString("\t{\n", *ostr);
writeExtremesElement("min", extremes, 0, *ostr, force_quoting_64bit_integers);
writeExtremesElement("min", extremes, 0, *ostr, settings);
writeCString(",\n", *ostr);
writeExtremesElement("max", extremes, 1, *ostr, force_quoting_64bit_integers);
writeExtremesElement("max", extremes, 1, *ostr, settings);
writeChar('\n', *ostr);
writeCString("\t}", *ostr);

5
dbms/src/DataStreams/JSONCompactRowOutputStream.h
View File

@ -5,16 +5,17 @@
#include <IO/WriteBufferValidUTF8.h>
#include <DataStreams/JSONRowOutputStream.h>
namespace DB
{
class FormatSettingsJSON;
/** The stream for outputting data in the JSONCompact format.
*/
class JSONCompactRowOutputStream : public JSONRowOutputStream
{
public:
JSONCompactRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_, bool force_quoting_64bit_integers_ = true);
JSONCompactRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_, const FormatSettingsJSON & settings);
void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
void writeFieldDelimiter() override;

1
dbms/src/DataStreams/JSONEachRowRowInputStream.h
View File

@ -4,7 +4,6 @@
#include <DataStreams/IRowInputStream.h>
#include <Common/HashTable/HashMap.h>
namespace DB
{

6
dbms/src/DataStreams/JSONEachRowRowOutputStream.cpp
View File

@ -7,8 +7,8 @@ namespace DB
{
JSONEachRowRowOutputStream::JSONEachRowRowOutputStream(WriteBuffer & ostr_, const Block & sample, bool force_quoting_64bit_integers_)
: ostr(ostr_), force_quoting_64bit_integers(force_quoting_64bit_integers_)
JSONEachRowRowOutputStream::JSONEachRowRowOutputStream(WriteBuffer & ostr_, const Block & sample, const FormatSettingsJSON & settings_)
: ostr(ostr_), settings(settings_)
{
size_t columns = sample.columns();
fields.resize(columns);
@ -25,7 +25,7 @@ void JSONEachRowRowOutputStream::writeField(const IColumn & column, const IDataT
{
writeString(fields[field_number], ostr);
writeChar(':', ostr);
type.serializeTextJSON(column, row_num, ostr, force_quoting_64bit_integers);
type.serializeTextJSON(column, row_num, ostr, settings);
++field_number;
}

5
dbms/src/DataStreams/JSONEachRowRowOutputStream.h
View File

@ -3,6 +3,7 @@
#include <Core/Block.h>
#include <IO/WriteBuffer.h>
#include <DataStreams/IRowOutputStream.h>
#include <DataTypes/FormatSettingsJSON.h>
namespace DB
@ -14,7 +15,7 @@ namespace DB
class JSONEachRowRowOutputStream : public IRowOutputStream
{
public:
JSONEachRowRowOutputStream(WriteBuffer & ostr_, const Block & sample, bool force_quoting_64bit_integers_ = true);
JSONEachRowRowOutputStream(WriteBuffer & ostr_, const Block & sample, const FormatSettingsJSON & settings);
void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
void writeFieldDelimiter() override;
@ -31,6 +32,8 @@ private:
size_t field_number = 0;
Names fields;
bool force_quoting_64bit_integers;
FormatSettingsJSON settings;
};
}

17
dbms/src/DataStreams/JSONRowOutputStream.cpp
View File

@ -6,9 +6,8 @@
namespace DB
{
JSONRowOutputStream::JSONRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_, bool force_quoting_64bit_integers_)
: dst_ostr(ostr_), write_statistics(write_statistics_), force_quoting_64bit_integers(force_quoting_64bit_integers_)
JSONRowOutputStream::JSONRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_, const FormatSettingsJSON & settings_)
: dst_ostr(ostr_), write_statistics(write_statistics_), settings(settings_)
{
NamesAndTypesList columns(sample_.getColumnsList());
fields.assign(columns.begin(), columns.end());
@ -72,7 +71,7 @@ void JSONRowOutputStream::writeField(const IColumn & column, const IDataType & t
writeCString("\t\t\t", *ostr);
writeString(fields[field_number].name, *ostr);
writeCString(": ", *ostr);
type.serializeTextJSON(column, row_num, *ostr, force_quoting_64bit_integers);
type.serializeTextJSON(column, row_num, *ostr, settings);
++field_number;
}
@ -152,7 +151,7 @@ void JSONRowOutputStream::writeTotals()
writeCString("\t\t", *ostr);
writeJSONString(column.name, *ostr);
writeCString(": ", *ostr);
column.type->serializeTextJSON(*column.column.get(), 0, *ostr, force_quoting_64bit_integers);
column.type->serializeTextJSON(*column.column.get(), 0, *ostr, settings);
}
writeChar('\n', *ostr);
@ -161,7 +160,7 @@ void JSONRowOutputStream::writeTotals()
}
static void writeExtremesElement(const char * title, const Block & extremes, size_t row_num, WriteBuffer & ostr, bool force_quoting_64bit_integers)
static void writeExtremesElement(const char * title, const Block & extremes, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON & settings)
{
writeCString("\t\t\"", ostr);
writeCString(title, ostr);
@ -179,7 +178,7 @@ static void writeExtremesElement(const char * title, const Block & extremes, siz
writeCString("\t\t\t", ostr);
writeJSONString(column.name, ostr);
writeCString(": ", ostr);
column.type->serializeTextJSON(*column.column.get(), row_num, ostr, force_quoting_64bit_integers);
column.type->serializeTextJSON(*column.column.get(), row_num, ostr, settings);
}
writeChar('\n', ostr);
@ -195,9 +194,9 @@ void JSONRowOutputStream::writeExtremes()
writeCString("\t\"extremes\":\n", *ostr);
writeCString("\t{\n", *ostr);
writeExtremesElement("min", extremes, 0, *ostr, force_quoting_64bit_integers);
writeExtremesElement("min", extremes, 0, *ostr, settings);
writeCString(",\n", *ostr);
writeExtremesElement("max", extremes, 1, *ostr, force_quoting_64bit_integers);
writeExtremesElement("max", extremes, 1, *ostr, settings);
writeChar('\n', *ostr);
writeCString("\t}", *ostr);

6
dbms/src/DataStreams/JSONRowOutputStream.h
View File

@ -5,7 +5,7 @@
#include <IO/WriteBuffer.h>
#include <Common/Stopwatch.h>
#include <DataStreams/IRowOutputStream.h>
#include <DataTypes/FormatSettingsJSON.h>
namespace DB
{
@ -16,7 +16,7 @@ class JSONRowOutputStream : public IRowOutputStream
{
public:
JSONRowOutputStream(WriteBuffer & ostr_, const Block & sample_,
bool write_statistics_, bool force_quoting_64bit_integers_ = true);
bool write_statistics_, const FormatSettingsJSON & settings_);
void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
void writeFieldDelimiter() override;
@ -68,7 +68,7 @@ protected:
Progress progress;
Stopwatch watch;
bool write_statistics;
bool force_quoting_64bit_integers;
FormatSettingsJSON settings;
};
}

2
dbms/src/DataTypes/DataTypeDate.cpp
View File

@ -46,7 +46,7 @@ void DataTypeDate::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) co
static_cast<ColumnUInt16 &>(column).getData().push_back(x); /// It's important to do this at the end - for exception safety.
}
void DataTypeDate::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool, bool) const
void DataTypeDate::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const
{
writeChar('"', ostr);
serializeText(column, row_num, ostr);

31
dbms/src/DataTypes/DataTypeNumberBase.cpp
View File

@ -4,7 +4,7 @@
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <Common/NaNUtils.h>
#include <FormatSettingsJSON.h>
#include <DataTypes/FormatSettingsJSON.h>
namespace DB
@ -65,16 +65,37 @@ void DataTypeNumberBase<T>::deserializeTextQuoted(IColumn & column, ReadBuffer &
template <typename T>
void DataTypeNumberBase<T>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON & settings) const
{
const bool need_quote = std::is_integral<T>::value && (sizeof(T) == 8) && settings.force_quoting_64bit_integers;
writeCString(settings.output_format_json_quote_denormals ? "1" : "0", ostr);
auto x = static_cast<const ColumnVector<T> &>(column).getData()[row_num];
bool is_finite = isFinite(x);
const bool need_quote = (std::is_integral<T>::value && (sizeof(T) == 8) && settings.force_quoting_64bit_integers)
|| (settings.output_format_json_quote_denormals && !is_finite);
if (need_quote)
writeChar('"', ostr);
auto x = static_cast<const ColumnVector<T> &>(column).getData()[row_num];
if (isFinite(x))
if (is_finite)
writeText(x, ostr);
else
else if (!settings.output_format_json_quote_denormals)
writeCString("null", ostr);
else
{
if (std::signbit(x))
{
if (isNaN(x))
writeCString("-nan", ostr);
else
writeCString("-inf", ostr);
}
else
{
if (isNaN(x))
writeCString("nan", ostr);
else
writeCString("inf", ostr);
}
}
if (need_quote)
writeChar('"', ostr);