mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
Merge pull request #14333 from hczhcz/patch-0901
Add JSONStrings formats
This commit is contained in:
commit
f0a65522cc
@ -10,42 +10,51 @@ results of a `SELECT`, and to perform `INSERT`s into a file-backed table.
|
||||
|
||||
The supported formats are:
|
||||
|
||||
| Format | Input | Output |
|
||||
|-----------------------------------------------------------------|-------|--------|
|
||||
| [TabSeparated](#tabseparated) | ✔ | ✔ |
|
||||
| [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ |
|
||||
| [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ |
|
||||
| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ |
|
||||
| [Template](#format-template) | ✔ | ✔ |
|
||||
| [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ |
|
||||
| [CSV](#csv) | ✔ | ✔ |
|
||||
| [CSVWithNames](#csvwithnames) | ✔ | ✔ |
|
||||
| [CustomSeparated](#format-customseparated) | ✔ | ✔ |
|
||||
| [Values](#data-format-values) | ✔ | ✔ |
|
||||
| [Vertical](#vertical) | ✗ | ✔ |
|
||||
| [VerticalRaw](#verticalraw) | ✗ | ✔ |
|
||||
| [JSON](#json) | ✗ | ✔ |
|
||||
| [JSONCompact](#jsoncompact) | ✗ | ✔ |
|
||||
| [JSONEachRow](#jsoneachrow) | ✔ | ✔ |
|
||||
| [TSKV](#tskv) | ✔ | ✔ |
|
||||
| [Pretty](#pretty) | ✗ | ✔ |
|
||||
| [PrettyCompact](#prettycompact) | ✗ | ✔ |
|
||||
| [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ |
|
||||
| [PrettyNoEscapes](#prettynoescapes) | ✗ | ✔ |
|
||||
| [PrettySpace](#prettyspace) | ✗ | ✔ |
|
||||
| [Protobuf](#protobuf) | ✔ | ✔ |
|
||||
| [Avro](#data-format-avro) | ✔ | ✔ |
|
||||
| [AvroConfluent](#data-format-avro-confluent) | ✔ | ✗ |
|
||||
| [Parquet](#data-format-parquet) | ✔ | ✔ |
|
||||
| [Arrow](#data-format-arrow) | ✔ | ✔ |
|
||||
| [ArrowStream](#data-format-arrow-stream) | ✔ | ✔ |
|
||||
| [ORC](#data-format-orc) | ✔ | ✗ |
|
||||
| [RowBinary](#rowbinary) | ✔ | ✔ |
|
||||
| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ |
|
||||
| [Native](#native) | ✔ | ✔ |
|
||||
| [Null](#null) | ✗ | ✔ |
|
||||
| [XML](#xml) | ✗ | ✔ |
|
||||
| [CapnProto](#capnproto) | ✔ | ✗ |
|
||||
| Format | Input | Output |
|
||||
|-----------------------------------------------------------------------------------------|-------|--------|
|
||||
| [TabSeparated](#tabseparated) | ✔ | ✔ |
|
||||
| [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ |
|
||||
| [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ |
|
||||
| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ |
|
||||
| [Template](#format-template) | ✔ | ✔ |
|
||||
| [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ |
|
||||
| [CSV](#csv) | ✔ | ✔ |
|
||||
| [CSVWithNames](#csvwithnames) | ✔ | ✔ |
|
||||
| [CustomSeparated](#format-customseparated) | ✔ | ✔ |
|
||||
| [Values](#data-format-values) | ✔ | ✔ |
|
||||
| [Vertical](#vertical) | ✗ | ✔ |
|
||||
| [VerticalRaw](#verticalraw) | ✗ | ✔ |
|
||||
| [JSON](#json) | ✗ | ✔ |
|
||||
| [JSONString](#jsonstring) | ✗ | ✔ |
|
||||
| [JSONCompact](#jsoncompact) | ✗ | ✔ |
|
||||
| [JSONCompactString](#jsoncompactstring) | ✗ | ✔ |
|
||||
| [JSONEachRow](#jsoneachrow) | ✔ | ✔ |
|
||||
| [JSONEachRowWithProgress](#jsoneachrowwithprogress) | ✗ | ✔ |
|
||||
| [JSONStringEachRow](#jsonstringeachrow) | ✔ | ✔ |
|
||||
| [JSONStringEachRowWithProgress](#jsonstringeachrowwithprogress) | ✗ | ✔ |
|
||||
| [JSONCompactEachRow](#jsoncompacteachrow) | ✔ | ✔ |
|
||||
| [JSONCompactEachRowWithNamesAndTypes](#jsoncompacteachrowwithnamesandtypes) | ✔ | ✔ |
|
||||
| [JSONCompactStringEachRow](#jsoncompactstringeachrow) | ✔ | ✔ |
|
||||
| [JSONCompactStringEachRowWithNamesAndTypes](#jsoncompactstringeachrowwithnamesandtypes) | ✔ | ✔ |
|
||||
| [TSKV](#tskv) | ✔ | ✔ |
|
||||
| [Pretty](#pretty) | ✗ | ✔ |
|
||||
| [PrettyCompact](#prettycompact) | ✗ | ✔ |
|
||||
| [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ |
|
||||
| [PrettyNoEscapes](#prettynoescapes) | ✗ | ✔ |
|
||||
| [PrettySpace](#prettyspace) | ✗ | ✔ |
|
||||
| [Protobuf](#protobuf) | ✔ | ✔ |
|
||||
| [Avro](#data-format-avro) | ✔ | ✔ |
|
||||
| [AvroConfluent](#data-format-avro-confluent) | ✔ | ✗ |
|
||||
| [Parquet](#data-format-parquet) | ✔ | ✔ |
|
||||
| [Arrow](#data-format-arrow) | ✔ | ✔ |
|
||||
| [ArrowStream](#data-format-arrow-stream) | ✔ | ✔ |
|
||||
| [ORC](#data-format-orc) | ✔ | ✗ |
|
||||
| [RowBinary](#rowbinary) | ✔ | ✔ |
|
||||
| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ |
|
||||
| [Native](#native) | ✔ | ✔ |
|
||||
| [Null](#null) | ✗ | ✔ |
|
||||
| [XML](#xml) | ✗ | ✔ |
|
||||
| [CapnProto](#capnproto) | ✔ | ✗ |
|
||||
|
||||
You can control some format processing parameters with the ClickHouse settings. For more information read the [Settings](../operations/settings/settings.md) section.
|
||||
|
||||
@ -392,62 +401,41 @@ SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTA
|
||||
"meta":
|
||||
[
|
||||
{
|
||||
"name": "SearchPhrase",
|
||||
"name": "'hello'",
|
||||
"type": "String"
|
||||
},
|
||||
{
|
||||
"name": "c",
|
||||
"name": "multiply(42, number)",
|
||||
"type": "UInt64"
|
||||
},
|
||||
{
|
||||
"name": "range(5)",
|
||||
"type": "Array(UInt8)"
|
||||
}
|
||||
],
|
||||
|
||||
"data":
|
||||
[
|
||||
{
|
||||
"SearchPhrase": "",
|
||||
"c": "8267016"
|
||||
"'hello'": "hello",
|
||||
"multiply(42, number)": "0",
|
||||
"range(5)": [0,1,2,3,4]
|
||||
},
|
||||
{
|
||||
"SearchPhrase": "bathroom interior design",
|
||||
"c": "2166"
|
||||
"'hello'": "hello",
|
||||
"multiply(42, number)": "42",
|
||||
"range(5)": [0,1,2,3,4]
|
||||
},
|
||||
{
|
||||
"SearchPhrase": "yandex",
|
||||
"c": "1655"
|
||||
},
|
||||
{
|
||||
"SearchPhrase": "spring 2014 fashion",
|
||||
"c": "1549"
|
||||
},
|
||||
{
|
||||
"SearchPhrase": "freeform photos",
|
||||
"c": "1480"
|
||||
"'hello'": "hello",
|
||||
"multiply(42, number)": "84",
|
||||
"range(5)": [0,1,2,3,4]
|
||||
}
|
||||
],
|
||||
|
||||
"totals":
|
||||
{
|
||||
"SearchPhrase": "",
|
||||
"c": "8873898"
|
||||
},
|
||||
"rows": 3,
|
||||
|
||||
"extremes":
|
||||
{
|
||||
"min":
|
||||
{
|
||||
"SearchPhrase": "",
|
||||
"c": "1480"
|
||||
},
|
||||
"max":
|
||||
{
|
||||
"SearchPhrase": "",
|
||||
"c": "8267016"
|
||||
}
|
||||
},
|
||||
|
||||
"rows": 5,
|
||||
|
||||
"rows_before_limit_at_least": 141137
|
||||
"rows_before_limit_at_least": 3
|
||||
}
|
||||
```
|
||||
|
||||
@ -468,63 +456,165 @@ ClickHouse supports [NULL](../sql-reference/syntax.md), which is displayed as `n
|
||||
|
||||
See also the [JSONEachRow](#jsoneachrow) format.
|
||||
|
||||
## JSONString {#jsonstring}
|
||||
|
||||
Differs from JSON only in that data fields are output in strings, not in typed json values.
|
||||
|
||||
Example:
|
||||
|
||||
```json
|
||||
{
|
||||
"meta":
|
||||
[
|
||||
{
|
||||
"name": "'hello'",
|
||||
"type": "String"
|
||||
},
|
||||
{
|
||||
"name": "multiply(42, number)",
|
||||
"type": "UInt64"
|
||||
},
|
||||
{
|
||||
"name": "range(5)",
|
||||
"type": "Array(UInt8)"
|
||||
}
|
||||
],
|
||||
|
||||
"data":
|
||||
[
|
||||
{
|
||||
"'hello'": "hello",
|
||||
"multiply(42, number)": "0",
|
||||
"range(5)": "[0,1,2,3,4]"
|
||||
},
|
||||
{
|
||||
"'hello'": "hello",
|
||||
"multiply(42, number)": "42",
|
||||
"range(5)": "[0,1,2,3,4]"
|
||||
},
|
||||
{
|
||||
"'hello'": "hello",
|
||||
"multiply(42, number)": "84",
|
||||
"range(5)": "[0,1,2,3,4]"
|
||||
}
|
||||
],
|
||||
|
||||
"rows": 3,
|
||||
|
||||
"rows_before_limit_at_least": 3
|
||||
}
|
||||
```
|
||||
|
||||
## JSONCompact {#jsoncompact}
|
||||
## JSONCompactString {#jsoncompactstring}
|
||||
|
||||
Differs from JSON only in that data rows are output in arrays, not in objects.
|
||||
|
||||
Example:
|
||||
|
||||
``` json
|
||||
// JSONCompact
|
||||
{
|
||||
"meta":
|
||||
[
|
||||
{
|
||||
"name": "SearchPhrase",
|
||||
"name": "'hello'",
|
||||
"type": "String"
|
||||
},
|
||||
{
|
||||
"name": "c",
|
||||
"name": "multiply(42, number)",
|
||||
"type": "UInt64"
|
||||
},
|
||||
{
|
||||
"name": "range(5)",
|
||||
"type": "Array(UInt8)"
|
||||
}
|
||||
],
|
||||
|
||||
"data":
|
||||
[
|
||||
["", "8267016"],
|
||||
["bathroom interior design", "2166"],
|
||||
["yandex", "1655"],
|
||||
["fashion trends spring 2014", "1549"],
|
||||
["freeform photo", "1480"]
|
||||
["hello", "0", [0,1,2,3,4]],
|
||||
["hello", "42", [0,1,2,3,4]],
|
||||
["hello", "84", [0,1,2,3,4]]
|
||||
],
|
||||
|
||||
"totals": ["","8873898"],
|
||||
"rows": 3,
|
||||
|
||||
"extremes":
|
||||
{
|
||||
"min": ["","1480"],
|
||||
"max": ["","8267016"]
|
||||
},
|
||||
|
||||
"rows": 5,
|
||||
|
||||
"rows_before_limit_at_least": 141137
|
||||
"rows_before_limit_at_least": 3
|
||||
}
|
||||
```
|
||||
|
||||
This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table).
|
||||
See also the `JSONEachRow` format.
|
||||
```json
|
||||
// JSONCompactString
|
||||
{
|
||||
"meta":
|
||||
[
|
||||
{
|
||||
"name": "'hello'",
|
||||
"type": "String"
|
||||
},
|
||||
{
|
||||
"name": "multiply(42, number)",
|
||||
"type": "UInt64"
|
||||
},
|
||||
{
|
||||
"name": "range(5)",
|
||||
"type": "Array(UInt8)"
|
||||
}
|
||||
],
|
||||
|
||||
## JSONEachRow {#jsoneachrow}
|
||||
"data":
|
||||
[
|
||||
["hello", "0", "[0,1,2,3,4]"],
|
||||
["hello", "42", "[0,1,2,3,4]"],
|
||||
["hello", "84", "[0,1,2,3,4]"]
|
||||
],
|
||||
|
||||
When using this format, ClickHouse outputs rows as separated, newline-delimited JSON objects, but the data as a whole is not valid JSON.
|
||||
"rows": 3,
|
||||
|
||||
``` json
|
||||
{"SearchPhrase":"curtain designs","count()":"1064"}
|
||||
{"SearchPhrase":"baku","count()":"1000"}
|
||||
{"SearchPhrase":"","count()":"8267016"}
|
||||
"rows_before_limit_at_least": 3
|
||||
}
|
||||
```
|
||||
|
||||
When inserting the data, you should provide a separate JSON object for each row.
|
||||
## JSONEachRow {#jsoneachrow}
|
||||
## JSONStringEachRow {#jsonstringeachrow}
|
||||
## JSONCompactEachRow {#jsoncompacteachrow}
|
||||
## JSONCompactStringEachRow {#jsoncompactstringeachrow}
|
||||
|
||||
When using these formats, ClickHouse outputs rows as separated, newline-delimited JSON values, but the data as a whole is not valid JSON.
|
||||
|
||||
``` json
|
||||
{"some_int":42,"some_str":"hello","some_tuple":[1,"a"]} // JSONEachRow
|
||||
[42,"hello",[1,"a"]] // JSONCompactEachRow
|
||||
["42","hello","(2,'a')"] // JSONCompactStringsEachRow
|
||||
```
|
||||
|
||||
When inserting the data, you should provide a separate JSON value for each row.
|
||||
|
||||
## JSONEachRowWithProgress {#jsoneachrowwithprogress}
|
||||
## JSONStringEachRowWithProgress {#jsonstringeachrowwithprogress}
|
||||
|
||||
Differs from JSONEachRow/JSONStringEachRow in that ClickHouse will also yield progress information as JSON objects.
|
||||
|
||||
```json
|
||||
{"row":{"'hello'":"hello","multiply(42, number)":"0","range(5)":[0,1,2,3,4]}}
|
||||
{"row":{"'hello'":"hello","multiply(42, number)":"42","range(5)":[0,1,2,3,4]}}
|
||||
{"row":{"'hello'":"hello","multiply(42, number)":"84","range(5)":[0,1,2,3,4]}}
|
||||
{"progress":{"read_rows":"3","read_bytes":"24","written_rows":"0","written_bytes":"0","total_rows_to_read":"3"}}
|
||||
```
|
||||
|
||||
## JSONCompactEachRowWithNamesAndTypes {#jsoncompacteachrowwithnamesandtypes}
|
||||
## JSONCompactStringEachRowWithNamesAndTypes {#jsoncompactstringeachrowwithnamesandtypes}
|
||||
|
||||
Differs from JSONCompactEachRow/JSONCompactStringEachRow in that the column names and types are written as the first two rows.
|
||||
|
||||
```json
|
||||
["'hello'", "multiply(42, number)", "range(5)"]
|
||||
["String", "UInt64", "Array(UInt8)"]
|
||||
["hello", "0", [0,1,2,3,4]]
|
||||
["hello", "42", [0,1,2,3,4]]
|
||||
["hello", "84", [0,1,2,3,4]]
|
||||
```
|
||||
|
||||
### Inserting Data {#inserting-data}
|
||||
|
||||
|
@ -308,16 +308,30 @@ ReturnType DataTypeNullable::deserializeTextQuoted(IColumn & column, ReadBuffer
|
||||
const DataTypePtr & nested_data_type)
|
||||
{
|
||||
return safeDeserialize<ReturnType>(column, *nested_data_type,
|
||||
[&istr] { return checkStringByFirstCharacterAndAssertTheRestCaseInsensitive("NULL", istr); },
|
||||
[&istr]
|
||||
{
|
||||
return checkStringByFirstCharacterAndAssertTheRestCaseInsensitive("NULL", istr);
|
||||
},
|
||||
[&nested_data_type, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeAsTextQuoted(nested, istr, settings); });
|
||||
}
|
||||
|
||||
|
||||
void DataTypeNullable::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
|
||||
{
|
||||
safeDeserialize(column, *nested_data_type,
|
||||
[&istr] { return checkStringByFirstCharacterAndAssertTheRestCaseInsensitive("NULL", istr); },
|
||||
[this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeAsWholeText(nested, istr, settings); });
|
||||
deserializeWholeText<void>(column, istr, settings, nested_data_type);
|
||||
}
|
||||
|
||||
template <typename ReturnType>
|
||||
ReturnType DataTypeNullable::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings,
|
||||
const DataTypePtr & nested_data_type)
|
||||
{
|
||||
return safeDeserialize<ReturnType>(column, *nested_data_type,
|
||||
[&istr]
|
||||
{
|
||||
return checkStringByFirstCharacterAndAssertTheRestCaseInsensitive("NULL", istr)
|
||||
|| checkStringByFirstCharacterAndAssertTheRest("ᴺᵁᴸᴸ", istr);
|
||||
},
|
||||
[&nested_data_type, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeAsWholeText(nested, istr, settings); });
|
||||
}
|
||||
|
||||
|
||||
@ -544,6 +558,7 @@ DataTypePtr removeNullable(const DataTypePtr & type)
|
||||
}
|
||||
|
||||
|
||||
template bool DataTypeNullable::deserializeWholeText<bool>(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested);
|
||||
template bool DataTypeNullable::deserializeTextEscaped<bool>(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested);
|
||||
template bool DataTypeNullable::deserializeTextQuoted<bool>(IColumn & column, ReadBuffer & istr, const FormatSettings &, const DataTypePtr & nested);
|
||||
template bool DataTypeNullable::deserializeTextCSV<bool>(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested);
|
||||
|
@ -103,6 +103,8 @@ public:
|
||||
/// If ReturnType is bool, check for NULL and deserialize value into non-nullable column (and return true) or insert default value of nested type (and return false)
|
||||
/// If ReturnType is void, deserialize Nullable(T)
|
||||
template <typename ReturnType = bool>
|
||||
static ReturnType deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested);
|
||||
template <typename ReturnType = bool>
|
||||
static ReturnType deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested);
|
||||
template <typename ReturnType = bool>
|
||||
static ReturnType deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &, const DataTypePtr & nested);
|
||||
|
@ -324,13 +324,85 @@ void FormatFactory::registerFileSegmentationEngine(const String & name, FileSegm
|
||||
target = std::move(file_segmentation_engine);
|
||||
}
|
||||
|
||||
/// File Segmentation Engines for parallel reading
|
||||
|
||||
void registerFileSegmentationEngineTabSeparated(FormatFactory & factory);
|
||||
void registerFileSegmentationEngineCSV(FormatFactory & factory);
|
||||
void registerFileSegmentationEngineJSONEachRow(FormatFactory & factory);
|
||||
void registerFileSegmentationEngineRegexp(FormatFactory & factory);
|
||||
void registerFileSegmentationEngineJSONAsString(FormatFactory & factory);
|
||||
|
||||
/// Formats for both input/output.
|
||||
|
||||
void registerInputFormatNative(FormatFactory & factory);
|
||||
void registerOutputFormatNative(FormatFactory & factory);
|
||||
|
||||
void registerInputFormatProcessorNative(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorNative(FormatFactory & factory);
|
||||
void registerInputFormatProcessorRowBinary(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorRowBinary(FormatFactory & factory);
|
||||
void registerInputFormatProcessorTabSeparated(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorTabSeparated(FormatFactory & factory);
|
||||
void registerInputFormatProcessorValues(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorValues(FormatFactory & factory);
|
||||
void registerInputFormatProcessorCSV(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorCSV(FormatFactory & factory);
|
||||
void registerInputFormatProcessorTSKV(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorTSKV(FormatFactory & factory);
|
||||
void registerInputFormatProcessorJSONEachRow(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorJSONEachRow(FormatFactory & factory);
|
||||
void registerInputFormatProcessorJSONCompactEachRow(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorJSONCompactEachRow(FormatFactory & factory);
|
||||
void registerInputFormatProcessorProtobuf(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorProtobuf(FormatFactory & factory);
|
||||
void registerInputFormatProcessorTemplate(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorTemplate(FormatFactory & factory);
|
||||
void registerInputFormatProcessorMsgPack(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorMsgPack(FormatFactory & factory);
|
||||
void registerInputFormatProcessorORC(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorORC(FormatFactory & factory);
|
||||
void registerInputFormatProcessorParquet(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorParquet(FormatFactory & factory);
|
||||
void registerInputFormatProcessorArrow(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorArrow(FormatFactory & factory);
|
||||
void registerInputFormatProcessorAvro(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorAvro(FormatFactory & factory);
|
||||
|
||||
/// Output only (presentational) formats.
|
||||
|
||||
void registerOutputFormatNull(FormatFactory & factory);
|
||||
|
||||
void registerOutputFormatProcessorPretty(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorPrettyCompact(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorPrettySpace(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorVertical(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorJSON(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorJSONCompact(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorJSONEachRowWithProgress(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorXML(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorODBCDriver2(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorNull(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorMySQLWire(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorMarkdown(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorPostgreSQLWire(FormatFactory & factory);
|
||||
|
||||
/// Input only formats.
|
||||
|
||||
void registerInputFormatProcessorRegexp(FormatFactory & factory);
|
||||
void registerInputFormatProcessorJSONAsString(FormatFactory & factory);
|
||||
void registerInputFormatProcessorCapnProto(FormatFactory & factory);
|
||||
|
||||
FormatFactory::FormatFactory()
|
||||
{
|
||||
registerFileSegmentationEngineTabSeparated(*this);
|
||||
registerFileSegmentationEngineCSV(*this);
|
||||
registerFileSegmentationEngineJSONEachRow(*this);
|
||||
registerFileSegmentationEngineRegexp(*this);
|
||||
registerFileSegmentationEngineJSONAsString(*this);
|
||||
|
||||
registerInputFormatNative(*this);
|
||||
registerOutputFormatNative(*this);
|
||||
|
||||
registerOutputFormatProcessorJSONEachRowWithProgress(*this);
|
||||
|
||||
registerInputFormatProcessorNative(*this);
|
||||
registerOutputFormatProcessorNative(*this);
|
||||
registerInputFormatProcessorRowBinary(*this);
|
||||
@ -349,8 +421,11 @@ FormatFactory::FormatFactory()
|
||||
registerOutputFormatProcessorJSONCompactEachRow(*this);
|
||||
registerInputFormatProcessorProtobuf(*this);
|
||||
registerOutputFormatProcessorProtobuf(*this);
|
||||
registerInputFormatProcessorTemplate(*this);
|
||||
registerOutputFormatProcessorTemplate(*this);
|
||||
registerInputFormatProcessorMsgPack(*this);
|
||||
registerOutputFormatProcessorMsgPack(*this);
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
registerInputFormatProcessorCapnProto(*this);
|
||||
registerInputFormatProcessorORC(*this);
|
||||
registerOutputFormatProcessorORC(*this);
|
||||
registerInputFormatProcessorParquet(*this);
|
||||
@ -360,18 +435,6 @@ FormatFactory::FormatFactory()
|
||||
registerInputFormatProcessorAvro(*this);
|
||||
registerOutputFormatProcessorAvro(*this);
|
||||
#endif
|
||||
registerInputFormatProcessorTemplate(*this);
|
||||
registerOutputFormatProcessorTemplate(*this);
|
||||
registerInputFormatProcessorRegexp(*this);
|
||||
registerInputFormatProcessorMsgPack(*this);
|
||||
registerOutputFormatProcessorMsgPack(*this);
|
||||
registerInputFormatProcessorJSONAsString(*this);
|
||||
|
||||
registerFileSegmentationEngineTabSeparated(*this);
|
||||
registerFileSegmentationEngineCSV(*this);
|
||||
registerFileSegmentationEngineJSONEachRow(*this);
|
||||
registerFileSegmentationEngineRegexp(*this);
|
||||
registerFileSegmentationEngineJSONAsString(*this);
|
||||
|
||||
registerOutputFormatNull(*this);
|
||||
|
||||
@ -381,12 +444,19 @@ FormatFactory::FormatFactory()
|
||||
registerOutputFormatProcessorVertical(*this);
|
||||
registerOutputFormatProcessorJSON(*this);
|
||||
registerOutputFormatProcessorJSONCompact(*this);
|
||||
registerOutputFormatProcessorJSONEachRowWithProgress(*this);
|
||||
registerOutputFormatProcessorXML(*this);
|
||||
registerOutputFormatProcessorODBCDriver2(*this);
|
||||
registerOutputFormatProcessorNull(*this);
|
||||
registerOutputFormatProcessorMySQLWire(*this);
|
||||
registerOutputFormatProcessorMarkdown(*this);
|
||||
registerOutputFormatProcessorPostgreSQLWire(*this);
|
||||
|
||||
registerInputFormatProcessorRegexp(*this);
|
||||
registerInputFormatProcessorJSONAsString(*this);
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
registerInputFormatProcessorCapnProto(*this);
|
||||
#endif
|
||||
}
|
||||
|
||||
FormatFactory & FormatFactory::instance()
|
||||
|
@ -141,73 +141,4 @@ private:
|
||||
const Creators & getCreators(const String & name) const;
|
||||
};
|
||||
|
||||
/// Formats for both input/output.
|
||||
|
||||
void registerInputFormatNative(FormatFactory & factory);
|
||||
void registerOutputFormatNative(FormatFactory & factory);
|
||||
|
||||
void registerInputFormatProcessorNative(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorNative(FormatFactory & factory);
|
||||
void registerInputFormatProcessorRowBinary(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorRowBinary(FormatFactory & factory);
|
||||
void registerInputFormatProcessorTabSeparated(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorTabSeparated(FormatFactory & factory);
|
||||
void registerInputFormatProcessorValues(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorValues(FormatFactory & factory);
|
||||
void registerInputFormatProcessorCSV(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorCSV(FormatFactory & factory);
|
||||
void registerInputFormatProcessorTSKV(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorTSKV(FormatFactory & factory);
|
||||
void registerInputFormatProcessorJSONEachRow(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorJSONEachRow(FormatFactory & factory);
|
||||
void registerInputFormatProcessorJSONCompactEachRow(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorJSONCompactEachRow(FormatFactory & factory);
|
||||
void registerInputFormatProcessorParquet(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorParquet(FormatFactory & factory);
|
||||
void registerInputFormatProcessorArrow(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorArrow(FormatFactory & factory);
|
||||
void registerInputFormatProcessorProtobuf(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorProtobuf(FormatFactory & factory);
|
||||
void registerInputFormatProcessorAvro(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorAvro(FormatFactory & factory);
|
||||
void registerInputFormatProcessorTemplate(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorTemplate(FormatFactory & factory);
|
||||
void registerInputFormatProcessorMsgPack(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorMsgPack(FormatFactory & factory);
|
||||
void registerInputFormatProcessorORC(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorORC(FormatFactory & factory);
|
||||
|
||||
|
||||
/// File Segmentation Engines for parallel reading
|
||||
|
||||
void registerFileSegmentationEngineTabSeparated(FormatFactory & factory);
|
||||
void registerFileSegmentationEngineCSV(FormatFactory & factory);
|
||||
void registerFileSegmentationEngineJSONEachRow(FormatFactory & factory);
|
||||
void registerFileSegmentationEngineRegexp(FormatFactory & factory);
|
||||
void registerFileSegmentationEngineJSONAsString(FormatFactory & factory);
|
||||
|
||||
/// Output only (presentational) formats.
|
||||
|
||||
void registerOutputFormatNull(FormatFactory & factory);
|
||||
|
||||
void registerOutputFormatProcessorPretty(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorPrettyCompact(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorPrettySpace(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorPrettyASCII(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorVertical(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorJSON(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorJSONCompact(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorJSONEachRowWithProgress(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorXML(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorODBCDriver2(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorNull(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorMySQLWire(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorMarkdown(FormatFactory & factory);
|
||||
void registerOutputFormatProcessorPostgreSQLWire(FormatFactory & factory);
|
||||
|
||||
/// Input only formats.
|
||||
void registerInputFormatProcessorCapnProto(FormatFactory & factory);
|
||||
void registerInputFormatProcessorRegexp(FormatFactory & factory);
|
||||
void registerInputFormatProcessorJSONAsString(FormatFactory & factory);
|
||||
|
||||
}
|
||||
|
@ -1,4 +1,5 @@
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
|
||||
#include <Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.h>
|
||||
#include <Formats/FormatFactory.h>
|
||||
@ -19,8 +20,9 @@ JSONCompactEachRowRowInputFormat::JSONCompactEachRowRowInputFormat(ReadBuffer &
|
||||
const Block & header_,
|
||||
Params params_,
|
||||
const FormatSettings & format_settings_,
|
||||
bool with_names_)
|
||||
: IRowInputFormat(header_, in_, std::move(params_)), format_settings(format_settings_), with_names(with_names_)
|
||||
bool with_names_,
|
||||
bool yield_strings_)
|
||||
: IRowInputFormat(header_, in_, std::move(params_)), format_settings(format_settings_), with_names(with_names_), yield_strings(yield_strings_)
|
||||
{
|
||||
const auto & sample = getPort().getHeader();
|
||||
size_t num_columns = sample.columns();
|
||||
@ -200,10 +202,26 @@ void JSONCompactEachRowRowInputFormat::readField(size_t index, MutableColumns &
|
||||
{
|
||||
read_columns[index] = true;
|
||||
const auto & type = data_types[index];
|
||||
if (format_settings.null_as_default && !type->isNullable())
|
||||
read_columns[index] = DataTypeNullable::deserializeTextJSON(*columns[index], in, format_settings, type);
|
||||
|
||||
if (yield_strings)
|
||||
{
|
||||
String str;
|
||||
readJSONString(str, in);
|
||||
|
||||
ReadBufferFromString buf(str);
|
||||
|
||||
if (format_settings.null_as_default && !type->isNullable())
|
||||
read_columns[index] = DataTypeNullable::deserializeWholeText(*columns[index], buf, format_settings, type);
|
||||
else
|
||||
type->deserializeAsWholeText(*columns[index], buf, format_settings);
|
||||
}
|
||||
else
|
||||
type->deserializeAsTextJSON(*columns[index], in, format_settings);
|
||||
{
|
||||
if (format_settings.null_as_default && !type->isNullable())
|
||||
read_columns[index] = DataTypeNullable::deserializeTextJSON(*columns[index], in, format_settings, type);
|
||||
else
|
||||
type->deserializeAsTextJSON(*columns[index], in, format_settings);
|
||||
}
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
@ -225,7 +243,7 @@ void registerInputFormatProcessorJSONCompactEachRow(FormatFactory & factory)
|
||||
IRowInputFormat::Params params,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
return std::make_shared<JSONCompactEachRowRowInputFormat>(buf, sample, std::move(params), settings, false);
|
||||
return std::make_shared<JSONCompactEachRowRowInputFormat>(buf, sample, std::move(params), settings, false, false);
|
||||
});
|
||||
|
||||
factory.registerInputFormatProcessor("JSONCompactEachRowWithNamesAndTypes", [](
|
||||
@ -234,7 +252,25 @@ void registerInputFormatProcessorJSONCompactEachRow(FormatFactory & factory)
|
||||
IRowInputFormat::Params params,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
return std::make_shared<JSONCompactEachRowRowInputFormat>(buf, sample, std::move(params), settings, true);
|
||||
return std::make_shared<JSONCompactEachRowRowInputFormat>(buf, sample, std::move(params), settings, true, false);
|
||||
});
|
||||
|
||||
factory.registerInputFormatProcessor("JSONCompactStringsEachRow", [](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
IRowInputFormat::Params params,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
return std::make_shared<JSONCompactEachRowRowInputFormat>(buf, sample, std::move(params), settings, false, true);
|
||||
});
|
||||
|
||||
factory.registerInputFormatProcessor("JSONCompactStringsEachRowWithNamesAndTypes", [](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
IRowInputFormat::Params params,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
return std::make_shared<JSONCompactEachRowRowInputFormat>(buf, sample, std::move(params), settings, true, true);
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -1,7 +1,5 @@
|
||||
#pragma once
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <Core/Block.h>
|
||||
#include <Processors/Formats/IRowInputFormat.h>
|
||||
#include <Formats/FormatSettings.h>
|
||||
@ -12,12 +10,23 @@ namespace DB
|
||||
|
||||
class ReadBuffer;
|
||||
|
||||
/** A stream for reading data in JSONCompactEachRow and JSONCompactEachRowWithNamesAndTypes formats
|
||||
/** A stream for reading data in a bunch of formats:
|
||||
* - JSONCompactEachRow
|
||||
* - JSONCompactEachRowWithNamesAndTypes
|
||||
* - JSONCompactStringsEachRow
|
||||
* - JSONCompactStringsEachRowWithNamesAndTypes
|
||||
*
|
||||
*/
|
||||
class JSONCompactEachRowRowInputFormat : public IRowInputFormat
|
||||
{
|
||||
public:
|
||||
JSONCompactEachRowRowInputFormat(ReadBuffer & in_, const Block & header_, Params params_, const FormatSettings & format_settings_, bool with_names_);
|
||||
JSONCompactEachRowRowInputFormat(
|
||||
ReadBuffer & in_,
|
||||
const Block & header_,
|
||||
Params params_,
|
||||
const FormatSettings & format_settings_,
|
||||
bool with_names_,
|
||||
bool yield_strings_);
|
||||
|
||||
String getName() const override { return "JSONCompactEachRowRowInputFormat"; }
|
||||
|
||||
@ -48,7 +57,10 @@ private:
|
||||
/// This is for the correct exceptions in skipping unknown fields.
|
||||
std::vector<String> names_of_columns;
|
||||
|
||||
/// For *WithNamesAndTypes formats.
|
||||
bool with_names;
|
||||
/// For JSONCompactString* formats.
|
||||
bool yield_strings;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -12,8 +12,9 @@ JSONCompactEachRowRowOutputFormat::JSONCompactEachRowRowOutputFormat(WriteBuffer
|
||||
const Block & header_,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & settings_,
|
||||
bool with_names_)
|
||||
: IRowOutputFormat(header_, out_, callback), settings(settings_), with_names(with_names_)
|
||||
bool with_names_,
|
||||
bool yield_strings_)
|
||||
: IRowOutputFormat(header_, out_, callback), settings(settings_), with_names(with_names_), yield_strings(yield_strings_)
|
||||
{
|
||||
const auto & sample = getPort(PortKind::Main).getHeader();
|
||||
NamesAndTypesList columns(sample.getNamesAndTypesList());
|
||||
@ -23,7 +24,15 @@ JSONCompactEachRowRowOutputFormat::JSONCompactEachRowRowOutputFormat(WriteBuffer
|
||||
|
||||
void JSONCompactEachRowRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num)
|
||||
{
|
||||
type.serializeAsTextJSON(column, row_num, out, settings);
|
||||
if (yield_strings)
|
||||
{
|
||||
WriteBufferFromOwnString buf;
|
||||
|
||||
type.serializeAsText(column, row_num, buf, settings);
|
||||
writeJSONString(buf.str(), out, settings);
|
||||
}
|
||||
else
|
||||
type.serializeAsTextJSON(column, row_num, out, settings);
|
||||
}
|
||||
|
||||
|
||||
@ -97,7 +106,7 @@ void registerOutputFormatProcessorJSONCompactEachRow(FormatFactory & factory)
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
return std::make_shared<JSONCompactEachRowRowOutputFormat>(buf, sample, callback, format_settings, false);
|
||||
return std::make_shared<JSONCompactEachRowRowOutputFormat>(buf, sample, callback, format_settings, false, false);
|
||||
});
|
||||
|
||||
factory.registerOutputFormatProcessor("JSONCompactEachRowWithNamesAndTypes", [](
|
||||
@ -106,7 +115,25 @@ void registerOutputFormatProcessorJSONCompactEachRow(FormatFactory & factory)
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings &format_settings)
|
||||
{
|
||||
return std::make_shared<JSONCompactEachRowRowOutputFormat>(buf, sample, callback, format_settings, true);
|
||||
return std::make_shared<JSONCompactEachRowRowOutputFormat>(buf, sample, callback, format_settings, true, false);
|
||||
});
|
||||
|
||||
factory.registerOutputFormatProcessor("JSONCompactStringsEachRow", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
return std::make_shared<JSONCompactEachRowRowOutputFormat>(buf, sample, callback, format_settings, false, true);
|
||||
});
|
||||
|
||||
factory.registerOutputFormatProcessor("JSONCompactStringsEachRowWithNamesAndTypes", [](
|
||||
WriteBuffer &buf,
|
||||
const Block &sample,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings &format_settings)
|
||||
{
|
||||
return std::make_shared<JSONCompactEachRowRowOutputFormat>(buf, sample, callback, format_settings, true, true);
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -15,7 +15,13 @@ namespace DB
|
||||
class JSONCompactEachRowRowOutputFormat : public IRowOutputFormat
|
||||
{
|
||||
public:
|
||||
JSONCompactEachRowRowOutputFormat(WriteBuffer & out_, const Block & header_, FormatFactory::WriteCallback callback, const FormatSettings & settings_, bool with_names);
|
||||
JSONCompactEachRowRowOutputFormat(
|
||||
WriteBuffer & out_,
|
||||
const Block & header_,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & settings_,
|
||||
bool with_names_,
|
||||
bool yield_strings_);
|
||||
|
||||
String getName() const override { return "JSONCompactEachRowRowOutputFormat"; }
|
||||
|
||||
@ -41,5 +47,6 @@ private:
|
||||
NamesAndTypes fields;
|
||||
|
||||
bool with_names;
|
||||
bool yield_strings;
|
||||
};
|
||||
}
|
||||
|
@ -8,15 +8,28 @@ namespace DB
|
||||
{
|
||||
|
||||
JSONCompactRowOutputFormat::JSONCompactRowOutputFormat(
|
||||
WriteBuffer & out_, const Block & header, FormatFactory::WriteCallback callback, const FormatSettings & settings_)
|
||||
: JSONRowOutputFormat(out_, header, callback, settings_)
|
||||
WriteBuffer & out_,
|
||||
const Block & header,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & settings_,
|
||||
bool yield_strings_)
|
||||
: JSONRowOutputFormat(out_, header, callback, settings_, yield_strings_)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
void JSONCompactRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num)
|
||||
{
|
||||
type.serializeAsTextJSON(column, row_num, *ostr, settings);
|
||||
if (yield_strings)
|
||||
{
|
||||
WriteBufferFromOwnString buf;
|
||||
|
||||
type.serializeAsText(column, row_num, buf, settings);
|
||||
writeJSONString(buf.str(), *ostr, settings);
|
||||
}
|
||||
else
|
||||
type.serializeAsTextJSON(column, row_num, *ostr, settings);
|
||||
|
||||
++field_number;
|
||||
}
|
||||
|
||||
@ -83,7 +96,16 @@ void registerOutputFormatProcessorJSONCompact(FormatFactory & factory)
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
return std::make_shared<JSONCompactRowOutputFormat>(buf, sample, callback, format_settings);
|
||||
return std::make_shared<JSONCompactRowOutputFormat>(buf, sample, callback, format_settings, false);
|
||||
});
|
||||
|
||||
factory.registerOutputFormatProcessor("JSONCompactStrings", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
return std::make_shared<JSONCompactRowOutputFormat>(buf, sample, callback, format_settings, true);
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -11,12 +11,17 @@ namespace DB
|
||||
|
||||
struct FormatSettings;
|
||||
|
||||
/** The stream for outputting data in the JSONCompact format.
|
||||
/** The stream for outputting data in the JSONCompact- formats.
|
||||
*/
|
||||
class JSONCompactRowOutputFormat : public JSONRowOutputFormat
|
||||
{
|
||||
public:
|
||||
JSONCompactRowOutputFormat(WriteBuffer & out_, const Block & header, FormatFactory::WriteCallback callback, const FormatSettings & settings_);
|
||||
JSONCompactRowOutputFormat(
|
||||
WriteBuffer & out_,
|
||||
const Block & header,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & settings_,
|
||||
bool yield_strings_);
|
||||
|
||||
String getName() const override { return "JSONCompactRowOutputFormat"; }
|
||||
|
||||
@ -37,7 +42,6 @@ protected:
|
||||
}
|
||||
|
||||
void writeTotalsFieldDelimiter() override;
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -1,4 +1,5 @@
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
|
||||
#include <Processors/Formats/Impl/JSONEachRowRowInputFormat.h>
|
||||
#include <Formats/JSONEachRowUtils.h>
|
||||
@ -29,8 +30,12 @@ enum
|
||||
|
||||
|
||||
JSONEachRowRowInputFormat::JSONEachRowRowInputFormat(
|
||||
ReadBuffer & in_, const Block & header_, Params params_, const FormatSettings & format_settings_)
|
||||
: IRowInputFormat(header_, in_, std::move(params_)), format_settings(format_settings_), name_map(header_.columns())
|
||||
ReadBuffer & in_,
|
||||
const Block & header_,
|
||||
Params params_,
|
||||
const FormatSettings & format_settings_,
|
||||
bool yield_strings_)
|
||||
: IRowInputFormat(header_, in_, std::move(params_)), format_settings(format_settings_), name_map(header_.columns()), yield_strings(yield_strings_)
|
||||
{
|
||||
size_t num_columns = getPort().getHeader().columns();
|
||||
for (size_t i = 0; i < num_columns; ++i)
|
||||
@ -135,10 +140,26 @@ void JSONEachRowRowInputFormat::readField(size_t index, MutableColumns & columns
|
||||
{
|
||||
seen_columns[index] = read_columns[index] = true;
|
||||
const auto & type = getPort().getHeader().getByPosition(index).type;
|
||||
if (format_settings.null_as_default && !type->isNullable())
|
||||
read_columns[index] = DataTypeNullable::deserializeTextJSON(*columns[index], in, format_settings, type);
|
||||
|
||||
if (yield_strings)
|
||||
{
|
||||
String str;
|
||||
readJSONString(str, in);
|
||||
|
||||
ReadBufferFromString buf(str);
|
||||
|
||||
if (format_settings.null_as_default && !type->isNullable())
|
||||
read_columns[index] = DataTypeNullable::deserializeWholeText(*columns[index], buf, format_settings, type);
|
||||
else
|
||||
type->deserializeAsWholeText(*columns[index], buf, format_settings);
|
||||
}
|
||||
else
|
||||
type->deserializeAsTextJSON(*columns[index], in, format_settings);
|
||||
{
|
||||
if (format_settings.null_as_default && !type->isNullable())
|
||||
read_columns[index] = DataTypeNullable::deserializeTextJSON(*columns[index], in, format_settings, type);
|
||||
else
|
||||
type->deserializeAsTextJSON(*columns[index], in, format_settings);
|
||||
}
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
@ -318,13 +339,23 @@ void registerInputFormatProcessorJSONEachRow(FormatFactory & factory)
|
||||
IRowInputFormat::Params params,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
return std::make_shared<JSONEachRowRowInputFormat>(buf, sample, std::move(params), settings);
|
||||
return std::make_shared<JSONEachRowRowInputFormat>(buf, sample, std::move(params), settings, false);
|
||||
});
|
||||
|
||||
factory.registerInputFormatProcessor("JSONStringsEachRow", [](
|
||||
ReadBuffer & buf,
|
||||
const Block & sample,
|
||||
IRowInputFormat::Params params,
|
||||
const FormatSettings & settings)
|
||||
{
|
||||
return std::make_shared<JSONEachRowRowInputFormat>(buf, sample, std::move(params), settings, true);
|
||||
});
|
||||
}
|
||||
|
||||
void registerFileSegmentationEngineJSONEachRow(FormatFactory & factory)
|
||||
{
|
||||
factory.registerFileSegmentationEngine("JSONEachRow", &fileSegmentationEngineJSONEachRowImpl);
|
||||
factory.registerFileSegmentationEngine("JSONStringsEachRow", &fileSegmentationEngineJSONEachRowImpl);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -20,7 +20,12 @@ class ReadBuffer;
|
||||
class JSONEachRowRowInputFormat : public IRowInputFormat
|
||||
{
|
||||
public:
|
||||
JSONEachRowRowInputFormat(ReadBuffer & in_, const Block & header_, Params params_, const FormatSettings & format_settings_);
|
||||
JSONEachRowRowInputFormat(
|
||||
ReadBuffer & in_,
|
||||
const Block & header_,
|
||||
Params params_,
|
||||
const FormatSettings & format_settings_,
|
||||
bool yield_strings_);
|
||||
|
||||
String getName() const override { return "JSONEachRowRowInputFormat"; }
|
||||
|
||||
@ -75,6 +80,8 @@ private:
|
||||
bool data_in_square_brackets = false;
|
||||
|
||||
bool allow_new_rows = true;
|
||||
|
||||
bool yield_strings;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -8,8 +8,13 @@ namespace DB
|
||||
{
|
||||
|
||||
|
||||
JSONEachRowRowOutputFormat::JSONEachRowRowOutputFormat(WriteBuffer & out_, const Block & header_, FormatFactory::WriteCallback callback, const FormatSettings & settings_)
|
||||
: IRowOutputFormat(header_, out_, callback), settings(settings_)
|
||||
JSONEachRowRowOutputFormat::JSONEachRowRowOutputFormat(
|
||||
WriteBuffer & out_,
|
||||
const Block & header_,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & settings_,
|
||||
bool yield_strings_)
|
||||
: IRowOutputFormat(header_, out_, callback), settings(settings_), yield_strings(yield_strings_)
|
||||
{
|
||||
const auto & sample = getPort(PortKind::Main).getHeader();
|
||||
size_t columns = sample.columns();
|
||||
@ -27,7 +32,17 @@ void JSONEachRowRowOutputFormat::writeField(const IColumn & column, const IDataT
|
||||
{
|
||||
writeString(fields[field_number], out);
|
||||
writeChar(':', out);
|
||||
type.serializeAsTextJSON(column, row_num, out, settings);
|
||||
|
||||
if (yield_strings)
|
||||
{
|
||||
WriteBufferFromOwnString buf;
|
||||
|
||||
type.serializeAsText(column, row_num, buf, settings);
|
||||
writeJSONString(buf.str(), out, settings);
|
||||
}
|
||||
else
|
||||
type.serializeAsTextJSON(column, row_num, out, settings);
|
||||
|
||||
++field_number;
|
||||
}
|
||||
|
||||
@ -59,7 +74,16 @@ void registerOutputFormatProcessorJSONEachRow(FormatFactory & factory)
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
return std::make_shared<JSONEachRowRowOutputFormat>(buf, sample, callback, format_settings);
|
||||
return std::make_shared<JSONEachRowRowOutputFormat>(buf, sample, callback, format_settings, false);
|
||||
});
|
||||
|
||||
factory.registerOutputFormatProcessor("JSONStringsEachRow", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
return std::make_shared<JSONEachRowRowOutputFormat>(buf, sample, callback, format_settings, true);
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -15,7 +15,12 @@ namespace DB
|
||||
class JSONEachRowRowOutputFormat : public IRowOutputFormat
|
||||
{
|
||||
public:
|
||||
JSONEachRowRowOutputFormat(WriteBuffer & out_, const Block & header_, FormatFactory::WriteCallback callback, const FormatSettings & settings_);
|
||||
JSONEachRowRowOutputFormat(
|
||||
WriteBuffer & out_,
|
||||
const Block & header_,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & settings_,
|
||||
bool yield_strings_);
|
||||
|
||||
String getName() const override { return "JSONEachRowRowOutputFormat"; }
|
||||
|
||||
@ -35,6 +40,9 @@ private:
|
||||
Names fields;
|
||||
|
||||
FormatSettings settings;
|
||||
|
||||
protected:
|
||||
bool yield_strings;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -36,7 +36,16 @@ void registerOutputFormatProcessorJSONEachRowWithProgress(FormatFactory & factor
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
return std::make_shared<JSONEachRowWithProgressRowOutputFormat>(buf, sample, callback, format_settings);
|
||||
return std::make_shared<JSONEachRowWithProgressRowOutputFormat>(buf, sample, callback, format_settings, false);
|
||||
});
|
||||
|
||||
factory.registerOutputFormatProcessor("JSONStringsEachRowWithProgress", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
return std::make_shared<JSONEachRowWithProgressRowOutputFormat>(buf, sample, callback, format_settings, true);
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -7,8 +7,13 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
JSONRowOutputFormat::JSONRowOutputFormat(WriteBuffer & out_, const Block & header, FormatFactory::WriteCallback callback, const FormatSettings & settings_)
|
||||
: IRowOutputFormat(header, out_, callback), settings(settings_)
|
||||
JSONRowOutputFormat::JSONRowOutputFormat(
|
||||
WriteBuffer & out_,
|
||||
const Block & header,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & settings_,
|
||||
bool yield_strings_)
|
||||
: IRowOutputFormat(header, out_, callback), settings(settings_), yield_strings(yield_strings_)
|
||||
{
|
||||
const auto & sample = getPort(PortKind::Main).getHeader();
|
||||
NamesAndTypesList columns(sample.getNamesAndTypesList());
|
||||
@ -71,7 +76,17 @@ void JSONRowOutputFormat::writeField(const IColumn & column, const IDataType & t
|
||||
writeCString("\t\t\t", *ostr);
|
||||
writeString(fields[field_number].name, *ostr);
|
||||
writeCString(": ", *ostr);
|
||||
type.serializeAsTextJSON(column, row_num, *ostr, settings);
|
||||
|
||||
if (yield_strings)
|
||||
{
|
||||
WriteBufferFromOwnString buf;
|
||||
|
||||
type.serializeAsText(column, row_num, buf, settings);
|
||||
writeJSONString(buf.str(), *ostr, settings);
|
||||
}
|
||||
else
|
||||
type.serializeAsTextJSON(column, row_num, *ostr, settings);
|
||||
|
||||
++field_number;
|
||||
}
|
||||
|
||||
@ -80,7 +95,17 @@ void JSONRowOutputFormat::writeTotalsField(const IColumn & column, const IDataTy
|
||||
writeCString("\t\t", *ostr);
|
||||
writeString(fields[field_number].name, *ostr);
|
||||
writeCString(": ", *ostr);
|
||||
type.serializeAsTextJSON(column, row_num, *ostr, settings);
|
||||
|
||||
if (yield_strings)
|
||||
{
|
||||
WriteBufferFromOwnString buf;
|
||||
|
||||
type.serializeAsText(column, row_num, buf, settings);
|
||||
writeJSONString(buf.str(), *ostr, settings);
|
||||
}
|
||||
else
|
||||
type.serializeAsTextJSON(column, row_num, *ostr, settings);
|
||||
|
||||
++field_number;
|
||||
}
|
||||
|
||||
@ -249,7 +274,16 @@ void registerOutputFormatProcessorJSON(FormatFactory & factory)
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
return std::make_shared<JSONRowOutputFormat>(buf, sample, callback, format_settings);
|
||||
return std::make_shared<JSONRowOutputFormat>(buf, sample, callback, format_settings, false);
|
||||
});
|
||||
|
||||
factory.registerOutputFormatProcessor("JSONStrings", [](
|
||||
WriteBuffer & buf,
|
||||
const Block & sample,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & format_settings)
|
||||
{
|
||||
return std::make_shared<JSONRowOutputFormat>(buf, sample, callback, format_settings, true);
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -16,7 +16,12 @@ namespace DB
|
||||
class JSONRowOutputFormat : public IRowOutputFormat
|
||||
{
|
||||
public:
|
||||
JSONRowOutputFormat(WriteBuffer & out_, const Block & header, FormatFactory::WriteCallback callback, const FormatSettings & settings_);
|
||||
JSONRowOutputFormat(
|
||||
WriteBuffer & out_,
|
||||
const Block & header,
|
||||
FormatFactory::WriteCallback callback,
|
||||
const FormatSettings & settings_,
|
||||
bool yield_strings_);
|
||||
|
||||
String getName() const override { return "JSONRowOutputFormat"; }
|
||||
|
||||
@ -78,6 +83,8 @@ protected:
|
||||
Progress progress;
|
||||
Stopwatch watch;
|
||||
FormatSettings settings;
|
||||
|
||||
bool yield_strings;
|
||||
};
|
||||
|
||||
}
|
||||
|
@ -18,6 +18,11 @@ JSONEachRow
|
||||
1 world 3 2019-07-23 [1,2,3] ('tuple',3.14)
|
||||
2 Hello 123 2019-06-19 [] ('test',2.71828)
|
||||
3 Hello 42 2019-06-19 [1,2,3] ('default',0.75)
|
||||
JSONStringsEachRow
|
||||
0 1 42 2019-07-22 [10,20,30] ('default',0)
|
||||
1 world 3 2019-07-23 [1,2,3] ('tuple',3.14)
|
||||
2 Hello 123 2019-06-19 [] ('test',2.71828)
|
||||
3 Hello 42 2019-06-19 [1,2,3] ('default',0.75)
|
||||
Template (Quoted)
|
||||
0 1 42 2019-07-22 [10,20,30] ('default',0)
|
||||
1 world 3 2019-07-23 [1,2,3] ('tuple',3.14)
|
||||
|
@ -38,6 +38,14 @@ echo '{"i": null, "s": "1", "n": null, "d": "2019-07-22", "a": [10, 20, 30], "t"
|
||||
$CLICKHOUSE_CLIENT --query="SELECT * FROM null_as_default ORDER BY i";
|
||||
$CLICKHOUSE_CLIENT --query="TRUNCATE TABLE null_as_default";
|
||||
|
||||
echo 'JSONStringsEachRow'
|
||||
echo '{"i": "null", "s": "1", "n": "ᴺᵁᴸᴸ", "d": "2019-07-22", "a": "[10, 20, 30]", "t": "NULL"}
|
||||
{"i": "1", "s": "world", "n": "3", "d": "2019-07-23", "a": "null", "t": "('\''tuple'\'', 3.14)"}
|
||||
{"i": "2", "s": "null", "n": "123", "d": "null", "a": "[]", "t": "('\''test'\'', 2.71828)"}
|
||||
{"i": "3", "s": "null", "n": "null", "d": "null", "a": "null", "t": "null"}' | $CLICKHOUSE_CLIENT --input_format_null_as_default=1 --query="INSERT INTO null_as_default FORMAT JSONStringsEachRow";
|
||||
$CLICKHOUSE_CLIENT --query="SELECT * FROM null_as_default ORDER BY i";
|
||||
$CLICKHOUSE_CLIENT --query="TRUNCATE TABLE null_as_default";
|
||||
|
||||
echo 'Template (Quoted)'
|
||||
echo 'NULL, '\''1'\'', null, '\''2019-07-22'\'', [10, 20, 30], NuLl
|
||||
1, '\''world'\'', 3, '\''2019-07-23'\'', NULL, ('\''tuple'\'', 3.14)
|
||||
|
@ -0,0 +1,22 @@
|
||||
1
|
||||
{"value":"1","name":"a"}
|
||||
{"value":"2","name":"b"}
|
||||
{"value":"3","name":"c"}
|
||||
2
|
||||
{"name":"a","c":"1"}
|
||||
{"name":"b","c":"1"}
|
||||
{"name":"c","c":"1"}
|
||||
3
|
||||
{"row":{"a":"1"}}
|
||||
{"progress":{"read_rows":"1","read_bytes":"1","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}
|
||||
4
|
||||
{"row":{"a":"1"}}
|
||||
{"progress":{"read_rows":"1","read_bytes":"1","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}
|
||||
5
|
||||
{"v1":"first","v2":"1","v3":"2","v4":"0"}
|
||||
{"v1":"second","v2":"2","v3":"0","v4":"6"}
|
||||
6
|
||||
{"v1":"first","v2":"1","v3":"2","v4":"8"}
|
||||
{"v1":"second","v2":"2","v3":"32","v4":"6"}
|
||||
7
|
||||
{"v1":"16","n.id":"[15,16,17]","n.name":"['first','second','third']"}
|
38
tests/queries/0_stateless/01446_json_strings_each_row.sql
Normal file
38
tests/queries/0_stateless/01446_json_strings_each_row.sql
Normal file
@ -0,0 +1,38 @@
|
||||
DROP TABLE IF EXISTS test_table;
|
||||
DROP TABLE IF EXISTS test_table_2;
|
||||
SELECT 1;
|
||||
/* Check JSONStringsEachRow Output */
|
||||
CREATE TABLE test_table (value UInt8, name String) ENGINE = MergeTree() ORDER BY value;
|
||||
INSERT INTO test_table VALUES (1, 'a'), (2, 'b'), (3, 'c');
|
||||
SELECT * FROM test_table FORMAT JSONStringsEachRow;
|
||||
SELECT 2;
|
||||
/* Check Totals */
|
||||
SELECT name, count() AS c FROM test_table GROUP BY name WITH TOTALS ORDER BY name FORMAT JSONStringsEachRow;
|
||||
SELECT 3;
|
||||
/* Check JSONStringsEachRowWithProgress Output */
|
||||
SELECT 1 as a FROM system.one FORMAT JSONStringsEachRowWithProgress;
|
||||
SELECT 4;
|
||||
/* Check Totals */
|
||||
SELECT 1 as a FROM system.one GROUP BY a WITH TOTALS ORDER BY a FORMAT JSONStringsEachRowWithProgress;
|
||||
DROP TABLE IF EXISTS test_table;
|
||||
SELECT 5;
|
||||
/* Check JSONStringsEachRow Input */
|
||||
CREATE TABLE test_table (v1 String, v2 UInt8, v3 DEFAULT v2 * 16, v4 UInt8 DEFAULT 8) ENGINE = MergeTree() ORDER BY v2;
|
||||
INSERT INTO test_table FORMAT JSONStringsEachRow {"v1": "first", "v2": "1", "v3": "2", "v4": "NULL"} {"v1": "second", "v2": "2", "v3": "null", "v4": "6"};
|
||||
SELECT * FROM test_table FORMAT JSONStringsEachRow;
|
||||
TRUNCATE TABLE test_table;
|
||||
SELECT 6;
|
||||
/* Check input_format_null_as_default = 1 */
|
||||
SET input_format_null_as_default = 1;
|
||||
INSERT INTO test_table FORMAT JSONStringsEachRow {"v1": "first", "v2": "1", "v3": "2", "v4": "ᴺᵁᴸᴸ"} {"v1": "second", "v2": "2", "v3": "null", "v4": "6"};
|
||||
SELECT * FROM test_table FORMAT JSONStringsEachRow;
|
||||
TRUNCATE TABLE test_table;
|
||||
SELECT 7;
|
||||
/* Check Nested */
|
||||
CREATE TABLE test_table_2 (v1 UInt8, n Nested(id UInt8, name String)) ENGINE = MergeTree() ORDER BY v1;
|
||||
INSERT INTO test_table_2 FORMAT JSONStringsEachRow {"v1": "16", "n.id": "[15, 16, 17]", "n.name": "['first', 'second', 'third']"};
|
||||
SELECT * FROM test_table_2 FORMAT JSONStringsEachRow;
|
||||
TRUNCATE TABLE test_table_2;
|
||||
|
||||
DROP TABLE IF EXISTS test_table;
|
||||
DROP TABLE IF EXISTS test_table_2;
|
43
tests/queries/0_stateless/01447_json_strings.reference
Normal file
43
tests/queries/0_stateless/01447_json_strings.reference
Normal file
@ -0,0 +1,43 @@
|
||||
{
|
||||
"meta":
|
||||
[
|
||||
{
|
||||
"name": "1",
|
||||
"type": "UInt8"
|
||||
},
|
||||
{
|
||||
"name": "'a'",
|
||||
"type": "String"
|
||||
},
|
||||
{
|
||||
"name": "[1, 2, 3]",
|
||||
"type": "Array(UInt8)"
|
||||
},
|
||||
{
|
||||
"name": "tuple(1, 'a')",
|
||||
"type": "Tuple(UInt8, String)"
|
||||
},
|
||||
{
|
||||
"name": "NULL",
|
||||
"type": "Nullable(Nothing)"
|
||||
},
|
||||
{
|
||||
"name": "nan",
|
||||
"type": "Float64"
|
||||
}
|
||||
],
|
||||
|
||||
"data":
|
||||
[
|
||||
{
|
||||
"1": "1",
|
||||
"'a'": "a",
|
||||
"[1, 2, 3]": "[1,2,3]",
|
||||
"tuple(1, 'a')": "(1,'a')",
|
||||
"NULL": "ᴺᵁᴸᴸ",
|
||||
"nan": "nan"
|
||||
}
|
||||
],
|
||||
|
||||
"rows": 1
|
||||
}
|
10
tests/queries/0_stateless/01447_json_strings.sql
Normal file
10
tests/queries/0_stateless/01447_json_strings.sql
Normal file
@ -0,0 +1,10 @@
|
||||
SET output_format_write_statistics = 0;
|
||||
|
||||
SELECT
|
||||
1,
|
||||
'a',
|
||||
[1, 2, 3],
|
||||
(1, 'a'),
|
||||
null,
|
||||
nan
|
||||
FORMAT JSONStrings;
|
@ -0,0 +1,47 @@
|
||||
1
|
||||
["1", "a"]
|
||||
["2", "b"]
|
||||
["3", "c"]
|
||||
2
|
||||
["a", "1"]
|
||||
["b", "1"]
|
||||
["c", "1"]
|
||||
3
|
||||
["value", "name"]
|
||||
["UInt8", "String"]
|
||||
["1", "a"]
|
||||
["2", "b"]
|
||||
["3", "c"]
|
||||
4
|
||||
["name", "c"]
|
||||
["String", "UInt64"]
|
||||
["a", "1"]
|
||||
["b", "1"]
|
||||
["c", "1"]
|
||||
|
||||
["", "3"]
|
||||
5
|
||||
["first", "1", "2", "0"]
|
||||
["second", "2", "0", "6"]
|
||||
6
|
||||
["first", "1", "2", "8"]
|
||||
["second", "2", "32", "6"]
|
||||
7
|
||||
["16", "[15,16,17]", "['first','second','third']"]
|
||||
8
|
||||
["first", "1", "2", "0"]
|
||||
["second", "2", "0", "6"]
|
||||
9
|
||||
["first", "1", "2", "8"]
|
||||
["second", "2", "32", "6"]
|
||||
10
|
||||
["first", "1", "16", "8"]
|
||||
["second", "2", "32", "8"]
|
||||
11
|
||||
["v1", "v2", "v3", "v4"]
|
||||
["String", "UInt8", "UInt16", "UInt8"]
|
||||
["", "2", "3", "1"]
|
||||
12
|
||||
["v1", "n.id", "n.name"]
|
||||
["UInt8", "Array(UInt8)", "Array(String)"]
|
||||
["16", "[15,16,17]", "['first','second','third']"]
|
@ -0,0 +1,63 @@
|
||||
DROP TABLE IF EXISTS test_table;
|
||||
DROP TABLE IF EXISTS test_table_2;
|
||||
SELECT 1;
|
||||
/* Check JSONCompactStringsEachRow Output */
|
||||
CREATE TABLE test_table (value UInt8, name String) ENGINE = MergeTree() ORDER BY value;
|
||||
INSERT INTO test_table VALUES (1, 'a'), (2, 'b'), (3, 'c');
|
||||
SELECT * FROM test_table FORMAT JSONCompactStringsEachRow;
|
||||
SELECT 2;
|
||||
/* Check Totals */
|
||||
SELECT name, count() AS c FROM test_table GROUP BY name WITH TOTALS ORDER BY name FORMAT JSONCompactStringsEachRow;
|
||||
SELECT 3;
|
||||
/* Check JSONCompactStringsEachRowWithNamesAndTypes Output */
|
||||
SELECT * FROM test_table FORMAT JSONCompactStringsEachRowWithNamesAndTypes;
|
||||
SELECT 4;
|
||||
/* Check Totals */
|
||||
SELECT name, count() AS c FROM test_table GROUP BY name WITH TOTALS ORDER BY name FORMAT JSONCompactStringsEachRowWithNamesAndTypes;
|
||||
DROP TABLE IF EXISTS test_table;
|
||||
SELECT 5;
|
||||
/* Check JSONCompactStringsEachRow Input */
|
||||
CREATE TABLE test_table (v1 String, v2 UInt8, v3 DEFAULT v2 * 16, v4 UInt8 DEFAULT 8) ENGINE = MergeTree() ORDER BY v2;
|
||||
INSERT INTO test_table FORMAT JSONCompactStringsEachRow ["first", "1", "2", "NULL"] ["second", "2", "null", "6"];
|
||||
SELECT * FROM test_table FORMAT JSONCompactStringsEachRow;
|
||||
TRUNCATE TABLE test_table;
|
||||
SELECT 6;
|
||||
/* Check input_format_null_as_default = 1 */
|
||||
SET input_format_null_as_default = 1;
|
||||
INSERT INTO test_table FORMAT JSONCompactStringsEachRow ["first", "1", "2", "ᴺᵁᴸᴸ"] ["second", "2", "null", "6"];
|
||||
SELECT * FROM test_table FORMAT JSONCompactStringsEachRow;
|
||||
TRUNCATE TABLE test_table;
|
||||
SELECT 7;
|
||||
/* Check Nested */
|
||||
CREATE TABLE test_table_2 (v1 UInt8, n Nested(id UInt8, name String)) ENGINE = MergeTree() ORDER BY v1;
|
||||
INSERT INTO test_table_2 FORMAT JSONCompactStringsEachRow ["16", "[15, 16, 17]", "['first', 'second', 'third']"];
|
||||
SELECT * FROM test_table_2 FORMAT JSONCompactStringsEachRow;
|
||||
TRUNCATE TABLE test_table_2;
|
||||
SELECT 8;
|
||||
/* Check JSONCompactStringsEachRowWithNamesAndTypes Output */
|
||||
SET input_format_null_as_default = 0;
|
||||
INSERT INTO test_table FORMAT JSONCompactStringsEachRowWithNamesAndTypes ["v1", "v2", "v3", "v4"]["String","UInt8","UInt16","UInt8"]["first", "1", "2", "null"]["second", "2", "null", "6"];
|
||||
SELECT * FROM test_table FORMAT JSONCompactStringsEachRow;
|
||||
TRUNCATE TABLE test_table;
|
||||
SELECT 9;
|
||||
/* Check input_format_null_as_default = 1 */
|
||||
SET input_format_null_as_default = 1;
|
||||
INSERT INTO test_table FORMAT JSONCompactStringsEachRowWithNamesAndTypes ["v1", "v2", "v3", "v4"]["String","UInt8","UInt16","UInt8"]["first", "1", "2", "null"] ["second", "2", "null", "6"];
|
||||
SELECT * FROM test_table FORMAT JSONCompactStringsEachRow;
|
||||
SELECT 10;
|
||||
/* Check Header */
|
||||
TRUNCATE TABLE test_table;
|
||||
SET input_format_skip_unknown_fields = 1;
|
||||
INSERT INTO test_table FORMAT JSONCompactStringsEachRowWithNamesAndTypes ["v1", "v2", "invalid_column"]["String", "UInt8", "UInt8"]["first", "1", "32"]["second", "2", "64"];
|
||||
SELECT * FROM test_table FORMAT JSONCompactStringsEachRow;
|
||||
SELECT 11;
|
||||
TRUNCATE TABLE test_table;
|
||||
INSERT INTO test_table FORMAT JSONCompactStringsEachRowWithNamesAndTypes ["v4", "v2", "v3"]["UInt8", "UInt8", "UInt16"]["1", "2", "3"]
|
||||
SELECT * FROM test_table FORMAT JSONCompactStringsEachRowWithNamesAndTypes;
|
||||
SELECT 12;
|
||||
/* Check Nested */
|
||||
INSERT INTO test_table_2 FORMAT JSONCompactStringsEachRowWithNamesAndTypes ["v1", "n.id", "n.name"]["UInt8", "Array(UInt8)", "Array(String)"]["16", "[15, 16, 17]", "['first', 'second', 'third']"];
|
||||
SELECT * FROM test_table_2 FORMAT JSONCompactStringsEachRowWithNamesAndTypes;
|
||||
|
||||
DROP TABLE IF EXISTS test_table;
|
||||
DROP TABLE IF EXISTS test_table_2;
|
@ -0,0 +1,36 @@
|
||||
{
|
||||
"meta":
|
||||
[
|
||||
{
|
||||
"name": "1",
|
||||
"type": "UInt8"
|
||||
},
|
||||
{
|
||||
"name": "'a'",
|
||||
"type": "String"
|
||||
},
|
||||
{
|
||||
"name": "[1, 2, 3]",
|
||||
"type": "Array(UInt8)"
|
||||
},
|
||||
{
|
||||
"name": "tuple(1, 'a')",
|
||||
"type": "Tuple(UInt8, String)"
|
||||
},
|
||||
{
|
||||
"name": "NULL",
|
||||
"type": "Nullable(Nothing)"
|
||||
},
|
||||
{
|
||||
"name": "nan",
|
||||
"type": "Float64"
|
||||
}
|
||||
],
|
||||
|
||||
"data":
|
||||
[
|
||||
["1", "a", "[1,2,3]", "(1,'a')", "ᴺᵁᴸᴸ", "nan"]
|
||||
],
|
||||
|
||||
"rows": 1
|
||||
}
|
10
tests/queries/0_stateless/01449_json_compact_strings.sql
Normal file
10
tests/queries/0_stateless/01449_json_compact_strings.sql
Normal file
@ -0,0 +1,10 @@
|
||||
SET output_format_write_statistics = 0;
|
||||
|
||||
SELECT
|
||||
1,
|
||||
'a',
|
||||
[1, 2, 3],
|
||||
(1, 'a'),
|
||||
null,
|
||||
nan
|
||||
FORMAT JSONCompactStrings;
|
Loading…
Reference in New Issue
Block a user