thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-23 16:12:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

updated FormatSettingsJSON

Browse Source
This commit is contained in:
Nikolai Kochetov 2017-07-05 19:28:57 +03:00 committed by alexey-milovidov
parent 0ead0ee2f1
commit 63119eda7d
10 changed files with 65 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
dbms/src/DataStreams/FormatFactory.cpp
View File

@ -29,7 +29,7 @@
#include <DataStreams/CSVRowOutputStream.h> #include <DataStreams/CSVRowOutputStream.h>
#include <DataStreams/MaterializingBlockOutputStream.h> #include <DataStreams/MaterializingBlockOutputStream.h>
#include <DataStreams/FormatFactory.h> #include <DataStreams/FormatFactory.h>
#include <DataTypes/FormatSettingsJSON.h>
namespace DB namespace DB
{ {
@ -121,6 +121,7 @@ static BlockOutputStreamPtr getOutputImpl(const String & name, WriteBuffer & buf
const Block & sample, const Context & context) const Block & sample, const Context & context)
{ {
const Settings & settings = context.getSettingsRef(); const Settings & settings = context.getSettingsRef();
FormatSettingsJSON json_settings(settings.output_format_json_quote_64bit_integers, true);
if (name == "Native") if (name == "Native")
return std::make_shared<NativeBlockOutputStream>(buf); return std::make_shared<NativeBlockOutputStream>(buf);
@ -163,14 +164,14 @@ static BlockOutputStreamPtr getOutputImpl(const String & name, WriteBuffer & buf
else if (name == "Values") else if (name == "Values")
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<ValuesRowOutputStream>(buf)); return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<ValuesRowOutputStream>(buf));
else if (name == "JSON") else if (name == "JSON")
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONRowOutputStream>(buf, sample, return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONRowOutputStream>(
settings.output_format_write_statistics, settings.output_format_json_quote_64bit_integers)); buf, sample, settings.output_format_write_statistics, json_settings));
else if (name == "JSONCompact") else if (name == "JSONCompact")
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONCompactRowOutputStream>(buf, sample, return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONCompactRowOutputStream>(
settings.output_format_write_statistics, settings.output_format_json_quote_64bit_integers)); buf, sample, settings.output_format_write_statistics, json_settings));
else if (name == "JSONEachRow") else if (name == "JSONEachRow")
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONEachRowRowOutputStream>(buf, sample, return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<JSONEachRowRowOutputStream>(
settings.output_format_json_quote_64bit_integers)); buf, sample, json_settings));
else if (name == "XML") else if (name == "XML")
return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<XMLRowOutputStream>(buf, sample, return std::make_shared<BlockOutputStreamFromRowOutputStream>(std::make_shared<XMLRowOutputStream>(buf, sample,
settings.output_format_write_statistics)); settings.output_format_write_statistics));

17
dbms/src/DataStreams/JSONCompactRowOutputStream.cpp
View File

@ -6,15 +6,16 @@
namespace DB namespace DB
{ {
JSONCompactRowOutputStream::JSONCompactRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_, bool force_quoting_64bit_integers_) JSONCompactRowOutputStream::JSONCompactRowOutputStream(WriteBuffer & ostr_, const Block & sample_,
: JSONRowOutputStream(ostr_, sample_, write_statistics_, force_quoting_64bit_integers_) bool write_statistics_, const FormatSettingsJSON & settings_)
: JSONRowOutputStream(ostr_, sample_, write_statistics_, settings_)
{ {
} }
void JSONCompactRowOutputStream::writeField(const IColumn & column, const IDataType & type, size_t row_num) void JSONCompactRowOutputStream::writeField(const IColumn & column, const IDataType & type, size_t row_num)
{ {
type.serializeTextJSON(column, row_num, *ostr, force_quoting_64bit_integers); type.serializeTextJSON(column, row_num, *ostr, settings);
++field_number; ++field_number;
} }
@ -56,7 +57,7 @@ void JSONCompactRowOutputStream::writeTotals()
writeChar(',', *ostr); writeChar(',', *ostr);
const ColumnWithTypeAndName & column = totals.safeGetByPosition(i); const ColumnWithTypeAndName & column = totals.safeGetByPosition(i);
column.type->serializeTextJSON(*column.column.get(), 0, *ostr, force_quoting_64bit_integers); column.type->serializeTextJSON(*column.column.get(), 0, *ostr, settings);
} }
writeChar(']', *ostr); writeChar(']', *ostr);
@ -64,7 +65,7 @@ void JSONCompactRowOutputStream::writeTotals()
} }
static void writeExtremesElement(const char * title, const Block & extremes, size_t row_num, WriteBuffer & ostr, bool force_quoting_64bit_integers) static void writeExtremesElement(const char * title, const Block & extremes, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON & settings)
{ {
writeCString("\t\t\"", ostr); writeCString("\t\t\"", ostr);
writeCString(title, ostr); writeCString(title, ostr);
@ -77,7 +78,7 @@ static void writeExtremesElement(const char * title, const Block & extremes, siz
writeChar(',', ostr); writeChar(',', ostr);
const ColumnWithTypeAndName & column = extremes.safeGetByPosition(i); const ColumnWithTypeAndName & column = extremes.safeGetByPosition(i);
column.type->serializeTextJSON(*column.column.get(), row_num, ostr, force_quoting_64bit_integers); column.type->serializeTextJSON(*column.column.get(), row_num, ostr, settings);
} }
writeChar(']', ostr); writeChar(']', ostr);
@ -92,9 +93,9 @@ void JSONCompactRowOutputStream::writeExtremes()
writeCString("\t\"extremes\":\n", *ostr); writeCString("\t\"extremes\":\n", *ostr);
writeCString("\t{\n", *ostr); writeCString("\t{\n", *ostr);
writeExtremesElement("min", extremes, 0, *ostr, force_quoting_64bit_integers); writeExtremesElement("min", extremes, 0, *ostr, settings);
writeCString(",\n", *ostr); writeCString(",\n", *ostr);
writeExtremesElement("max", extremes, 1, *ostr, force_quoting_64bit_integers); writeExtremesElement("max", extremes, 1, *ostr, settings);
writeChar('\n', *ostr); writeChar('\n', *ostr);
writeCString("\t}", *ostr); writeCString("\t}", *ostr);

5
dbms/src/DataStreams/JSONCompactRowOutputStream.h
View File

@ -5,16 +5,17 @@
#include <IO/WriteBufferValidUTF8.h> #include <IO/WriteBufferValidUTF8.h>
#include <DataStreams/JSONRowOutputStream.h> #include <DataStreams/JSONRowOutputStream.h>
namespace DB namespace DB
{ {
class FormatSettingsJSON;
/** The stream for outputting data in the JSONCompact format. /** The stream for outputting data in the JSONCompact format.
*/ */
class JSONCompactRowOutputStream : public JSONRowOutputStream class JSONCompactRowOutputStream : public JSONRowOutputStream
{ {
public: public:
JSONCompactRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_, bool force_quoting_64bit_integers_ = true); JSONCompactRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_, const FormatSettingsJSON & settings);
void writeField(const IColumn & column, const IDataType & type, size_t row_num) override; void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
void writeFieldDelimiter() override; void writeFieldDelimiter() override;

1
dbms/src/DataStreams/JSONEachRowRowInputStream.h
View File

@ -4,7 +4,6 @@
#include <DataStreams/IRowInputStream.h> #include <DataStreams/IRowInputStream.h>
#include <Common/HashTable/HashMap.h> #include <Common/HashTable/HashMap.h>
namespace DB namespace DB
{ {

6
dbms/src/DataStreams/JSONEachRowRowOutputStream.cpp
View File

@ -7,8 +7,8 @@ namespace DB
{ {
JSONEachRowRowOutputStream::JSONEachRowRowOutputStream(WriteBuffer & ostr_, const Block & sample, bool force_quoting_64bit_integers_) JSONEachRowRowOutputStream::JSONEachRowRowOutputStream(WriteBuffer & ostr_, const Block & sample, const FormatSettingsJSON & settings_)
: ostr(ostr_), force_quoting_64bit_integers(force_quoting_64bit_integers_) : ostr(ostr_), settings(settings_)
{ {
size_t columns = sample.columns(); size_t columns = sample.columns();
fields.resize(columns); fields.resize(columns);
@ -25,7 +25,7 @@ void JSONEachRowRowOutputStream::writeField(const IColumn & column, const IDataT
{ {
writeString(fields[field_number], ostr); writeString(fields[field_number], ostr);
writeChar(':', ostr); writeChar(':', ostr);
type.serializeTextJSON(column, row_num, ostr, force_quoting_64bit_integers); type.serializeTextJSON(column, row_num, ostr, settings);
++field_number; ++field_number;
} }

5
dbms/src/DataStreams/JSONEachRowRowOutputStream.h
View File

@ -3,6 +3,7 @@
#include <Core/Block.h> #include <Core/Block.h>
#include <IO/WriteBuffer.h> #include <IO/WriteBuffer.h>
#include <DataStreams/IRowOutputStream.h> #include <DataStreams/IRowOutputStream.h>
#include <DataTypes/FormatSettingsJSON.h>
namespace DB namespace DB
@ -14,7 +15,7 @@ namespace DB
class JSONEachRowRowOutputStream : public IRowOutputStream class JSONEachRowRowOutputStream : public IRowOutputStream
{ {
public: public:
JSONEachRowRowOutputStream(WriteBuffer & ostr_, const Block & sample, bool force_quoting_64bit_integers_ = true); JSONEachRowRowOutputStream(WriteBuffer & ostr_, const Block & sample, const FormatSettingsJSON & settings);
void writeField(const IColumn & column, const IDataType & type, size_t row_num) override; void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
void writeFieldDelimiter() override; void writeFieldDelimiter() override;
@ -31,6 +32,8 @@ private:
size_t field_number = 0; size_t field_number = 0;
Names fields; Names fields;
bool force_quoting_64bit_integers; bool force_quoting_64bit_integers;
FormatSettingsJSON settings;
}; };
} }

17
dbms/src/DataStreams/JSONRowOutputStream.cpp
View File

@ -6,9 +6,8 @@
namespace DB namespace DB
{ {
JSONRowOutputStream::JSONRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_, const FormatSettingsJSON & settings_)
JSONRowOutputStream::JSONRowOutputStream(WriteBuffer & ostr_, const Block & sample_, bool write_statistics_, bool force_quoting_64bit_integers_) : dst_ostr(ostr_), write_statistics(write_statistics_), settings(settings_)
: dst_ostr(ostr_), write_statistics(write_statistics_), force_quoting_64bit_integers(force_quoting_64bit_integers_)
{ {
NamesAndTypesList columns(sample_.getColumnsList()); NamesAndTypesList columns(sample_.getColumnsList());
fields.assign(columns.begin(), columns.end()); fields.assign(columns.begin(), columns.end());
@ -72,7 +71,7 @@ void JSONRowOutputStream::writeField(const IColumn & column, const IDataType & t
writeCString("\t\t\t", *ostr); writeCString("\t\t\t", *ostr);
writeString(fields[field_number].name, *ostr); writeString(fields[field_number].name, *ostr);
writeCString(": ", *ostr); writeCString(": ", *ostr);
type.serializeTextJSON(column, row_num, *ostr, force_quoting_64bit_integers); type.serializeTextJSON(column, row_num, *ostr, settings);
++field_number; ++field_number;
} }
@ -152,7 +151,7 @@ void JSONRowOutputStream::writeTotals()
writeCString("\t\t", *ostr); writeCString("\t\t", *ostr);
writeJSONString(column.name, *ostr); writeJSONString(column.name, *ostr);
writeCString(": ", *ostr); writeCString(": ", *ostr);
column.type->serializeTextJSON(*column.column.get(), 0, *ostr, force_quoting_64bit_integers); column.type->serializeTextJSON(*column.column.get(), 0, *ostr, settings);
} }
writeChar('\n', *ostr); writeChar('\n', *ostr);
@ -161,7 +160,7 @@ void JSONRowOutputStream::writeTotals()
} }
static void writeExtremesElement(const char * title, const Block & extremes, size_t row_num, WriteBuffer & ostr, bool force_quoting_64bit_integers) static void writeExtremesElement(const char * title, const Block & extremes, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON & settings)
{ {
writeCString("\t\t\"", ostr); writeCString("\t\t\"", ostr);
writeCString(title, ostr); writeCString(title, ostr);
@ -179,7 +178,7 @@ static void writeExtremesElement(const char * title, const Block & extremes, siz
writeCString("\t\t\t", ostr); writeCString("\t\t\t", ostr);
writeJSONString(column.name, ostr); writeJSONString(column.name, ostr);
writeCString(": ", ostr); writeCString(": ", ostr);
column.type->serializeTextJSON(*column.column.get(), row_num, ostr, force_quoting_64bit_integers); column.type->serializeTextJSON(*column.column.get(), row_num, ostr, settings);
} }
writeChar('\n', ostr); writeChar('\n', ostr);
@ -195,9 +194,9 @@ void JSONRowOutputStream::writeExtremes()
writeCString("\t\"extremes\":\n", *ostr); writeCString("\t\"extremes\":\n", *ostr);
writeCString("\t{\n", *ostr); writeCString("\t{\n", *ostr);
writeExtremesElement("min", extremes, 0, *ostr, force_quoting_64bit_integers); writeExtremesElement("min", extremes, 0, *ostr, settings);
writeCString(",\n", *ostr); writeCString(",\n", *ostr);
writeExtremesElement("max", extremes, 1, *ostr, force_quoting_64bit_integers); writeExtremesElement("max", extremes, 1, *ostr, settings);
writeChar('\n', *ostr); writeChar('\n', *ostr);
writeCString("\t}", *ostr); writeCString("\t}", *ostr);

6
dbms/src/DataStreams/JSONRowOutputStream.h
View File

@ -5,7 +5,7 @@
#include <IO/WriteBuffer.h> #include <IO/WriteBuffer.h>
#include <Common/Stopwatch.h> #include <Common/Stopwatch.h>
#include <DataStreams/IRowOutputStream.h> #include <DataStreams/IRowOutputStream.h>
#include <DataTypes/FormatSettingsJSON.h>
namespace DB namespace DB
{ {
@ -16,7 +16,7 @@ class JSONRowOutputStream : public IRowOutputStream
{ {
public: public:
JSONRowOutputStream(WriteBuffer & ostr_, const Block & sample_, JSONRowOutputStream(WriteBuffer & ostr_, const Block & sample_,
bool write_statistics_, bool force_quoting_64bit_integers_ = true); bool write_statistics_, const FormatSettingsJSON & settings_);
void writeField(const IColumn & column, const IDataType & type, size_t row_num) override; void writeField(const IColumn & column, const IDataType & type, size_t row_num) override;
void writeFieldDelimiter() override; void writeFieldDelimiter() override;
@ -68,7 +68,7 @@ protected:
Progress progress; Progress progress;
Stopwatch watch; Stopwatch watch;
bool write_statistics; bool write_statistics;
bool force_quoting_64bit_integers; FormatSettingsJSON settings;
}; };
} }

2
dbms/src/DataTypes/DataTypeDate.cpp
View File

@ -46,7 +46,7 @@ void DataTypeDate::deserializeTextQuoted(IColumn & column, ReadBuffer & istr) co
static_cast<ColumnUInt16 &>(column).getData().push_back(x); /// It's important to do this at the end - for exception safety. static_cast<ColumnUInt16 &>(column).getData().push_back(x); /// It's important to do this at the end - for exception safety.
} }
void DataTypeDate::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, bool, bool) const void DataTypeDate::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON &) const
{ {
writeChar('"', ostr); writeChar('"', ostr);
serializeText(column, row_num, ostr); serializeText(column, row_num, ostr);

31
dbms/src/DataTypes/DataTypeNumberBase.cpp
View File

@ -4,7 +4,7 @@
#include <IO/ReadHelpers.h> #include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h> #include <IO/WriteHelpers.h>
#include <Common/NaNUtils.h> #include <Common/NaNUtils.h>
#include <FormatSettingsJSON.h> #include <DataTypes/FormatSettingsJSON.h>
namespace DB namespace DB
@ -65,16 +65,37 @@ void DataTypeNumberBase<T>::deserializeTextQuoted(IColumn & column, ReadBuffer &
template <typename T> template <typename T>
void DataTypeNumberBase<T>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON & settings) const void DataTypeNumberBase<T>::serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettingsJSON & settings) const
{ {
const bool need_quote = std::is_integral<T>::value && (sizeof(T) == 8) && settings.force_quoting_64bit_integers; writeCString(settings.output_format_json_quote_denormals ? "1" : "0", ostr);
auto x = static_cast<const ColumnVector<T> &>(column).getData()[row_num];
bool is_finite = isFinite(x);
const bool need_quote = (std::is_integral<T>::value && (sizeof(T) == 8) && settings.force_quoting_64bit_integers)
|| (settings.output_format_json_quote_denormals && !is_finite);
if (need_quote) if (need_quote)
writeChar('"', ostr); writeChar('"', ostr);
auto x = static_cast<const ColumnVector<T> &>(column).getData()[row_num]; if (is_finite)
if (isFinite(x))
writeText(x, ostr); writeText(x, ostr);
else else if (!settings.output_format_json_quote_denormals)
writeCString("null", ostr); writeCString("null", ostr);
else
{
if (std::signbit(x))
{
if (isNaN(x))
writeCString("-nan", ostr);
else
writeCString("-inf", ostr);
}
else
{
if (isNaN(x))
writeCString("nan", ostr);
else
writeCString("inf", ostr);
}
}
if (need_quote) if (need_quote)
writeChar('"', ostr); writeChar('"', ostr);