thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-29 11:02:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into mysql_replication_tablemapevent

Browse Source
This commit is contained in:
BohuTANG 2020-09-11 08:49:17 +08:00
commit faa394b27c
42 changed files with 1004 additions and 260 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -17,5 +17,4 @@ ClickHouse is an open-source column-oriented database management system that all
## Upcoming Events ## Upcoming Events
* [ClickHouse Data Integration Virtual Meetup](https://www.eventbrite.com/e/clickhouse-september-virtual-meetup-data-integration-tickets-117421895049) on September 10, 2020.
* [ClickHouse talk at Ya.Subbotnik (in Russian)](https://ya.cc/t/cIBI-3yECj5JF) on September 12, 2020. * [ClickHouse talk at Ya.Subbotnik (in Russian)](https://ya.cc/t/cIBI-3yECj5JF) on September 12, 2020.

2
base/common/types.h
View File

@ -1,8 +1,6 @@
#pragma once #pragma once
#include <algorithm>
#include <cstdint> #include <cstdint>
#include <cstdlib>
#include <string> #include <string>
#include <type_traits> #include <type_traits>

290
docs/en/interfaces/formats.md
View File

@ -10,42 +10,51 @@ results of a `SELECT`, and to perform `INSERT`s into a file-backed table.
The supported formats are: The supported formats are:
| Format | Input | Output | | Format | Input | Output |
|-----------------------------------------------------------------|-------|--------| |-----------------------------------------------------------------------------------------|-------|--------|
| [TabSeparated](#tabseparated) | ✔ | ✔ | | [TabSeparated](#tabseparated) | ✔ | ✔ |
| [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ | | [TabSeparatedRaw](#tabseparatedraw) | ✔ | ✔ |
| [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ | | [TabSeparatedWithNames](#tabseparatedwithnames) | ✔ | ✔ |
| [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ | | [TabSeparatedWithNamesAndTypes](#tabseparatedwithnamesandtypes) | ✔ | ✔ |
| [Template](#format-template) | ✔ | ✔ | | [Template](#format-template) | ✔ | ✔ |
| [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ | | [TemplateIgnoreSpaces](#templateignorespaces) | ✔ | ✗ |
| [CSV](#csv) | ✔ | ✔ | | [CSV](#csv) | ✔ | ✔ |
| [CSVWithNames](#csvwithnames) | ✔ | ✔ | | [CSVWithNames](#csvwithnames) | ✔ | ✔ |
| [CustomSeparated](#format-customseparated) | ✔ | ✔ | | [CustomSeparated](#format-customseparated) | ✔ | ✔ |
| [Values](#data-format-values) | ✔ | ✔ | | [Values](#data-format-values) | ✔ | ✔ |
| [Vertical](#vertical) | ✗ | ✔ | | [Vertical](#vertical) | ✗ | ✔ |
| [VerticalRaw](#verticalraw) | ✗ | ✔ | | [VerticalRaw](#verticalraw) | ✗ | ✔ |
| [JSON](#json) | ✗ | ✔ | | [JSON](#json) | ✗ | ✔ |
| [JSONCompact](#jsoncompact) | ✗ | ✔ | | [JSONString](#jsonstring) | ✗ | ✔ |
| [JSONEachRow](#jsoneachrow) | ✔ | ✔ | | [JSONCompact](#jsoncompact) | ✗ | ✔ |
| [TSKV](#tskv) | ✔ | ✔ | | [JSONCompactString](#jsoncompactstring) | ✗ | ✔ |
| [Pretty](#pretty) | ✗ | ✔ | | [JSONEachRow](#jsoneachrow) | ✔ | ✔ |
| [PrettyCompact](#prettycompact) | ✗ | ✔ | | [JSONEachRowWithProgress](#jsoneachrowwithprogress) | ✗ | ✔ |
| [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ | | [JSONStringEachRow](#jsonstringeachrow) | ✔ | ✔ |
| [PrettyNoEscapes](#prettynoescapes) | ✗ | ✔ | | [JSONStringEachRowWithProgress](#jsonstringeachrowwithprogress) | ✗ | ✔ |
| [PrettySpace](#prettyspace) | ✗ | ✔ | | [JSONCompactEachRow](#jsoncompacteachrow) | ✔ | ✔ |
| [Protobuf](#protobuf) | ✔ | ✔ | | [JSONCompactEachRowWithNamesAndTypes](#jsoncompacteachrowwithnamesandtypes) | ✔ | ✔ |
| [Avro](#data-format-avro) | ✔ | ✔ | | [JSONCompactStringEachRow](#jsoncompactstringeachrow) | ✔ | ✔ |
| [AvroConfluent](#data-format-avro-confluent) | ✔ | ✗ | | [JSONCompactStringEachRowWithNamesAndTypes](#jsoncompactstringeachrowwithnamesandtypes) | ✔ | ✔ |
| [Parquet](#data-format-parquet) | ✔ | ✔ | | [TSKV](#tskv) | ✔ | ✔ |
| [Arrow](#data-format-arrow) | ✔ | ✔ | | [Pretty](#pretty) | ✗ | ✔ |
| [ArrowStream](#data-format-arrow-stream) | ✔ | ✔ | | [PrettyCompact](#prettycompact) | ✗ | ✔ |
| [ORC](#data-format-orc) | ✔ | ✗ | | [PrettyCompactMonoBlock](#prettycompactmonoblock) | ✗ | ✔ |
| [RowBinary](#rowbinary) | ✔ | ✔ | | [PrettyNoEscapes](#prettynoescapes) | ✗ | ✔ |
| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ | | [PrettySpace](#prettyspace) | ✗ | ✔ |
| [Native](#native) | ✔ | ✔ | | [Protobuf](#protobuf) | ✔ | ✔ |
| [Null](#null) | ✗ | ✔ | | [Avro](#data-format-avro) | ✔ | ✔ |
| [XML](#xml) | ✗ | ✔ | | [AvroConfluent](#data-format-avro-confluent) | ✔ | ✗ |
| [CapnProto](#capnproto) | ✔ | ✗ | | [Parquet](#data-format-parquet) | ✔ | ✔ |
| [Arrow](#data-format-arrow) | ✔ | ✔ |
| [ArrowStream](#data-format-arrow-stream) | ✔ | ✔ |
| [ORC](#data-format-orc) | ✔ | ✗ |
| [RowBinary](#rowbinary) | ✔ | ✔ |
| [RowBinaryWithNamesAndTypes](#rowbinarywithnamesandtypes) | ✔ | ✔ |
| [Native](#native) | ✔ | ✔ |
| [Null](#null) | ✗ | ✔ |
| [XML](#xml) | ✗ | ✔ |
| [CapnProto](#capnproto) | ✔ | ✗ |
You can control some format processing parameters with the ClickHouse settings. For more information read the [Settings](../operations/settings/settings.md) section. You can control some format processing parameters with the ClickHouse settings. For more information read the [Settings](../operations/settings/settings.md) section.
@ -392,62 +401,41 @@ SELECT SearchPhrase, count() AS c FROM test.hits GROUP BY SearchPhrase WITH TOTA
"meta": "meta":
[ [
{ {
"name": "SearchPhrase", "name": "'hello'",
"type": "String" "type": "String"
}, },
{ {
"name": "c", "name": "multiply(42, number)",
"type": "UInt64" "type": "UInt64"
},
{
"name": "range(5)",
"type": "Array(UInt8)"
} }
], ],
"data": "data":
[ [
{ {
"SearchPhrase": "", "'hello'": "hello",
"c": "8267016" "multiply(42, number)": "0",
"range(5)": [0,1,2,3,4]
}, },
{ {
"SearchPhrase": "bathroom interior design", "'hello'": "hello",
"c": "2166" "multiply(42, number)": "42",
"range(5)": [0,1,2,3,4]
}, },
{ {
"SearchPhrase": "yandex", "'hello'": "hello",
"c": "1655" "multiply(42, number)": "84",
}, "range(5)": [0,1,2,3,4]
{
"SearchPhrase": "spring 2014 fashion",
"c": "1549"
},
{
"SearchPhrase": "freeform photos",
"c": "1480"
} }
], ],
"totals": "rows": 3,
{
"SearchPhrase": "",
"c": "8873898"
},
"extremes": "rows_before_limit_at_least": 3
{
"min":
{
"SearchPhrase": "",
"c": "1480"
},
"max":
{
"SearchPhrase": "",
"c": "8267016"
}
},
"rows": 5,
"rows_before_limit_at_least": 141137
} }
``` ```
@ -468,63 +456,165 @@ ClickHouse supports [NULL](../sql-reference/syntax.md), which is displayed as `n
See also the [JSONEachRow](#jsoneachrow) format. See also the [JSONEachRow](#jsoneachrow) format.
## JSONString {#jsonstring}
Differs from JSON only in that data fields are output in strings, not in typed json values.
Example:
```json
{
"meta":
[
{
"name": "'hello'",
"type": "String"
},
{
"name": "multiply(42, number)",
"type": "UInt64"
},
{
"name": "range(5)",
"type": "Array(UInt8)"
}
],
"data":
[
{
"'hello'": "hello",
"multiply(42, number)": "0",
"range(5)": "[0,1,2,3,4]"
},
{
"'hello'": "hello",
"multiply(42, number)": "42",
"range(5)": "[0,1,2,3,4]"
},
{
"'hello'": "hello",
"multiply(42, number)": "84",
"range(5)": "[0,1,2,3,4]"
}
],
"rows": 3,
"rows_before_limit_at_least": 3
}
```
## JSONCompact {#jsoncompact} ## JSONCompact {#jsoncompact}
## JSONCompactString {#jsoncompactstring}
Differs from JSON only in that data rows are output in arrays, not in objects. Differs from JSON only in that data rows are output in arrays, not in objects.
Example: Example:
``` json ``` json
// JSONCompact
{ {
"meta": "meta":
[ [
{ {
"name": "SearchPhrase", "name": "'hello'",
"type": "String" "type": "String"
}, },
{ {
"name": "c", "name": "multiply(42, number)",
"type": "UInt64" "type": "UInt64"
},
{
"name": "range(5)",
"type": "Array(UInt8)"
} }
], ],
"data": "data":
[ [
["", "8267016"], ["hello", "0", [0,1,2,3,4]],
["bathroom interior design", "2166"], ["hello", "42", [0,1,2,3,4]],
["yandex", "1655"], ["hello", "84", [0,1,2,3,4]]
["fashion trends spring 2014", "1549"],
["freeform photo", "1480"]
], ],
"totals": ["","8873898"], "rows": 3,
"extremes": "rows_before_limit_at_least": 3
{
"min": ["","1480"],
"max": ["","8267016"]
},
"rows": 5,
"rows_before_limit_at_least": 141137
} }
``` ```
This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table). ```json
See also the `JSONEachRow` format. // JSONCompactString
{
"meta":
[
{
"name": "'hello'",
"type": "String"
},
{
"name": "multiply(42, number)",
"type": "UInt64"
},
{
"name": "range(5)",
"type": "Array(UInt8)"
}
],
## JSONEachRow {#jsoneachrow} "data":
[
["hello", "0", "[0,1,2,3,4]"],
["hello", "42", "[0,1,2,3,4]"],
["hello", "84", "[0,1,2,3,4]"]
],
When using this format, ClickHouse outputs rows as separated, newline-delimited JSON objects, but the data as a whole is not valid JSON. "rows": 3,
``` json "rows_before_limit_at_least": 3
{"SearchPhrase":"curtain designs","count()":"1064"} }
{"SearchPhrase":"baku","count()":"1000"}
{"SearchPhrase":"","count()":"8267016"}
``` ```
When inserting the data, you should provide a separate JSON object for each row. ## JSONEachRow {#jsoneachrow}
## JSONStringEachRow {#jsonstringeachrow}
## JSONCompactEachRow {#jsoncompacteachrow}
## JSONCompactStringEachRow {#jsoncompactstringeachrow}
When using these formats, ClickHouse outputs rows as separated, newline-delimited JSON values, but the data as a whole is not valid JSON.
``` json
{"some_int":42,"some_str":"hello","some_tuple":[1,"a"]} // JSONEachRow
[42,"hello",[1,"a"]] // JSONCompactEachRow
["42","hello","(2,'a')"] // JSONCompactStringsEachRow
```
When inserting the data, you should provide a separate JSON value for each row.
## JSONEachRowWithProgress {#jsoneachrowwithprogress}
## JSONStringEachRowWithProgress {#jsonstringeachrowwithprogress}
Differs from JSONEachRow/JSONStringEachRow in that ClickHouse will also yield progress information as JSON objects.
```json
{"row":{"'hello'":"hello","multiply(42, number)":"0","range(5)":[0,1,2,3,4]}}
{"row":{"'hello'":"hello","multiply(42, number)":"42","range(5)":[0,1,2,3,4]}}
{"row":{"'hello'":"hello","multiply(42, number)":"84","range(5)":[0,1,2,3,4]}}
{"progress":{"read_rows":"3","read_bytes":"24","written_rows":"0","written_bytes":"0","total_rows_to_read":"3"}}
```
## JSONCompactEachRowWithNamesAndTypes {#jsoncompacteachrowwithnamesandtypes}
## JSONCompactStringEachRowWithNamesAndTypes {#jsoncompactstringeachrowwithnamesandtypes}
Differs from JSONCompactEachRow/JSONCompactStringEachRow in that the column names and types are written as the first two rows.
```json
["'hello'", "multiply(42, number)", "range(5)"]
["String", "UInt64", "Array(UInt8)"]
["hello", "0", [0,1,2,3,4]]
["hello", "42", [0,1,2,3,4]]
["hello", "84", [0,1,2,3,4]]
```
### Inserting Data {#inserting-data} ### Inserting Data {#inserting-data}

6
programs/client/Client.cpp
View File

@ -866,6 +866,8 @@ private:
// will exit. The ping() would be the best match here, but it's // will exit. The ping() would be the best match here, but it's
// private, probably for a good reason that the protocol doesn't allow // private, probably for a good reason that the protocol doesn't allow
// pings at any possible moment. // pings at any possible moment.
// Don't forget to reset the default database which might have changed.
connection->setDefaultDatabase("");
connection->forceConnected(connection_parameters.timeouts); connection->forceConnected(connection_parameters.timeouts);
if (text.size() > 4 * 1024) if (text.size() > 4 * 1024)
@ -1103,7 +1105,9 @@ private:
{ {
last_exception_received_from_server = std::make_unique<Exception>(getCurrentExceptionMessage(true), getCurrentExceptionCode()); last_exception_received_from_server = std::make_unique<Exception>(getCurrentExceptionMessage(true), getCurrentExceptionCode());
received_exception_from_server = true; received_exception_from_server = true;
std::cerr << "Error on processing query: " << ast_to_process->formatForErrorMessage() << std::endl << last_exception_received_from_server->message(); fmt::print(stderr, "Error on processing query '{}': {}\n",
ast_to_process->formatForErrorMessage(),
last_exception_received_from_server->message());
} }
if (!connection->isConnected()) if (!connection->isConnected())

23
src/DataTypes/DataTypeNullable.cpp
View File

@ -308,16 +308,30 @@ ReturnType DataTypeNullable::deserializeTextQuoted(IColumn & column, ReadBuffer
const DataTypePtr & nested_data_type) const DataTypePtr & nested_data_type)
{ {
return safeDeserialize<ReturnType>(column, *nested_data_type, return safeDeserialize<ReturnType>(column, *nested_data_type,
[&istr] { return checkStringByFirstCharacterAndAssertTheRestCaseInsensitive("NULL", istr); }, [&istr]
{
return checkStringByFirstCharacterAndAssertTheRestCaseInsensitive("NULL", istr);
},
[&nested_data_type, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeAsTextQuoted(nested, istr, settings); }); [&nested_data_type, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeAsTextQuoted(nested, istr, settings); });
} }
void DataTypeNullable::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const void DataTypeNullable::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const
{ {
safeDeserialize(column, *nested_data_type, deserializeWholeText<void>(column, istr, settings, nested_data_type);
[&istr] { return checkStringByFirstCharacterAndAssertTheRestCaseInsensitive("NULL", istr); }, }
[this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeAsWholeText(nested, istr, settings); });
template <typename ReturnType>
ReturnType DataTypeNullable::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings,
const DataTypePtr & nested_data_type)
{
return safeDeserialize<ReturnType>(column, *nested_data_type,
[&istr]
{
return checkStringByFirstCharacterAndAssertTheRestCaseInsensitive("NULL", istr)
|| checkStringByFirstCharacterAndAssertTheRest("ᴺᵁᴸᴸ", istr);
},
[&nested_data_type, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeAsWholeText(nested, istr, settings); });
} }
@ -544,6 +558,7 @@ DataTypePtr removeNullable(const DataTypePtr & type)
} }
template bool DataTypeNullable::deserializeWholeText<bool>(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested);
template bool DataTypeNullable::deserializeTextEscaped<bool>(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested); template bool DataTypeNullable::deserializeTextEscaped<bool>(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested);
template bool DataTypeNullable::deserializeTextQuoted<bool>(IColumn & column, ReadBuffer & istr, const FormatSettings &, const DataTypePtr & nested); template bool DataTypeNullable::deserializeTextQuoted<bool>(IColumn & column, ReadBuffer & istr, const FormatSettings &, const DataTypePtr & nested);
template bool DataTypeNullable::deserializeTextCSV<bool>(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested); template bool DataTypeNullable::deserializeTextCSV<bool>(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested);

2
src/DataTypes/DataTypeNullable.h
View File

@ -103,6 +103,8 @@ public:
/// If ReturnType is bool, check for NULL and deserialize value into non-nullable column (and return true) or insert default value of nested type (and return false) /// If ReturnType is bool, check for NULL and deserialize value into non-nullable column (and return true) or insert default value of nested type (and return false)
/// If ReturnType is void, deserialize Nullable(T) /// If ReturnType is void, deserialize Nullable(T)
template <typename ReturnType = bool> template <typename ReturnType = bool>
static ReturnType deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested);
template <typename ReturnType = bool>
static ReturnType deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested); static ReturnType deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested);
template <typename ReturnType = bool> template <typename ReturnType = bool>
static ReturnType deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &, const DataTypePtr & nested); static ReturnType deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &, const DataTypePtr & nested);

100
src/Formats/FormatFactory.cpp
View File

@ -324,13 +324,85 @@ void FormatFactory::registerFileSegmentationEngine(const String & name, FileSegm
target = std::move(file_segmentation_engine); target = std::move(file_segmentation_engine);
} }
/// File Segmentation Engines for parallel reading
void registerFileSegmentationEngineTabSeparated(FormatFactory & factory);
void registerFileSegmentationEngineCSV(FormatFactory & factory);
void registerFileSegmentationEngineJSONEachRow(FormatFactory & factory);
void registerFileSegmentationEngineRegexp(FormatFactory & factory);
void registerFileSegmentationEngineJSONAsString(FormatFactory & factory);
/// Formats for both input/output.
void registerInputFormatNative(FormatFactory & factory);
void registerOutputFormatNative(FormatFactory & factory);
void registerInputFormatProcessorNative(FormatFactory & factory);
void registerOutputFormatProcessorNative(FormatFactory & factory);
void registerInputFormatProcessorRowBinary(FormatFactory & factory);
void registerOutputFormatProcessorRowBinary(FormatFactory & factory);
void registerInputFormatProcessorTabSeparated(FormatFactory & factory);
void registerOutputFormatProcessorTabSeparated(FormatFactory & factory);
void registerInputFormatProcessorValues(FormatFactory & factory);
void registerOutputFormatProcessorValues(FormatFactory & factory);
void registerInputFormatProcessorCSV(FormatFactory & factory);
void registerOutputFormatProcessorCSV(FormatFactory & factory);
void registerInputFormatProcessorTSKV(FormatFactory & factory);
void registerOutputFormatProcessorTSKV(FormatFactory & factory);
void registerInputFormatProcessorJSONEachRow(FormatFactory & factory);
void registerOutputFormatProcessorJSONEachRow(FormatFactory & factory);
void registerInputFormatProcessorJSONCompactEachRow(FormatFactory & factory);
void registerOutputFormatProcessorJSONCompactEachRow(FormatFactory & factory);
void registerInputFormatProcessorProtobuf(FormatFactory & factory);
void registerOutputFormatProcessorProtobuf(FormatFactory & factory);
void registerInputFormatProcessorTemplate(FormatFactory & factory);
void registerOutputFormatProcessorTemplate(FormatFactory & factory);
void registerInputFormatProcessorMsgPack(FormatFactory & factory);
void registerOutputFormatProcessorMsgPack(FormatFactory & factory);
void registerInputFormatProcessorORC(FormatFactory & factory);
void registerOutputFormatProcessorORC(FormatFactory & factory);
void registerInputFormatProcessorParquet(FormatFactory & factory);
void registerOutputFormatProcessorParquet(FormatFactory & factory);
void registerInputFormatProcessorArrow(FormatFactory & factory);
void registerOutputFormatProcessorArrow(FormatFactory & factory);
void registerInputFormatProcessorAvro(FormatFactory & factory);
void registerOutputFormatProcessorAvro(FormatFactory & factory);
/// Output only (presentational) formats.
void registerOutputFormatNull(FormatFactory & factory);
void registerOutputFormatProcessorPretty(FormatFactory & factory);
void registerOutputFormatProcessorPrettyCompact(FormatFactory & factory);
void registerOutputFormatProcessorPrettySpace(FormatFactory & factory);
void registerOutputFormatProcessorVertical(FormatFactory & factory);
void registerOutputFormatProcessorJSON(FormatFactory & factory);
void registerOutputFormatProcessorJSONCompact(FormatFactory & factory);
void registerOutputFormatProcessorJSONEachRowWithProgress(FormatFactory & factory);
void registerOutputFormatProcessorXML(FormatFactory & factory);
void registerOutputFormatProcessorODBCDriver2(FormatFactory & factory);
void registerOutputFormatProcessorNull(FormatFactory & factory);
void registerOutputFormatProcessorMySQLWire(FormatFactory & factory);
void registerOutputFormatProcessorMarkdown(FormatFactory & factory);
void registerOutputFormatProcessorPostgreSQLWire(FormatFactory & factory);
/// Input only formats.
void registerInputFormatProcessorRegexp(FormatFactory & factory);
void registerInputFormatProcessorJSONAsString(FormatFactory & factory);
void registerInputFormatProcessorCapnProto(FormatFactory & factory);
FormatFactory::FormatFactory() FormatFactory::FormatFactory()
{ {
registerFileSegmentationEngineTabSeparated(*this);
registerFileSegmentationEngineCSV(*this);
registerFileSegmentationEngineJSONEachRow(*this);
registerFileSegmentationEngineRegexp(*this);
registerFileSegmentationEngineJSONAsString(*this);
registerInputFormatNative(*this); registerInputFormatNative(*this);
registerOutputFormatNative(*this); registerOutputFormatNative(*this);
registerOutputFormatProcessorJSONEachRowWithProgress(*this);
registerInputFormatProcessorNative(*this); registerInputFormatProcessorNative(*this);
registerOutputFormatProcessorNative(*this); registerOutputFormatProcessorNative(*this);
registerInputFormatProcessorRowBinary(*this); registerInputFormatProcessorRowBinary(*this);
@ -349,8 +421,11 @@ FormatFactory::FormatFactory()
registerOutputFormatProcessorJSONCompactEachRow(*this); registerOutputFormatProcessorJSONCompactEachRow(*this);
registerInputFormatProcessorProtobuf(*this); registerInputFormatProcessorProtobuf(*this);
registerOutputFormatProcessorProtobuf(*this); registerOutputFormatProcessorProtobuf(*this);
registerInputFormatProcessorTemplate(*this);
registerOutputFormatProcessorTemplate(*this);
registerInputFormatProcessorMsgPack(*this);
registerOutputFormatProcessorMsgPack(*this);
#if !defined(ARCADIA_BUILD) #if !defined(ARCADIA_BUILD)
registerInputFormatProcessorCapnProto(*this);
registerInputFormatProcessorORC(*this); registerInputFormatProcessorORC(*this);
registerOutputFormatProcessorORC(*this); registerOutputFormatProcessorORC(*this);
registerInputFormatProcessorParquet(*this); registerInputFormatProcessorParquet(*this);
@ -360,18 +435,6 @@ FormatFactory::FormatFactory()
registerInputFormatProcessorAvro(*this); registerInputFormatProcessorAvro(*this);
registerOutputFormatProcessorAvro(*this); registerOutputFormatProcessorAvro(*this);
#endif #endif
registerInputFormatProcessorTemplate(*this);
registerOutputFormatProcessorTemplate(*this);
registerInputFormatProcessorRegexp(*this);
registerInputFormatProcessorMsgPack(*this);
registerOutputFormatProcessorMsgPack(*this);
registerInputFormatProcessorJSONAsString(*this);
registerFileSegmentationEngineTabSeparated(*this);
registerFileSegmentationEngineCSV(*this);
registerFileSegmentationEngineJSONEachRow(*this);
registerFileSegmentationEngineRegexp(*this);
registerFileSegmentationEngineJSONAsString(*this);
registerOutputFormatNull(*this); registerOutputFormatNull(*this);
@ -381,12 +444,19 @@ FormatFactory::FormatFactory()
registerOutputFormatProcessorVertical(*this); registerOutputFormatProcessorVertical(*this);
registerOutputFormatProcessorJSON(*this); registerOutputFormatProcessorJSON(*this);
registerOutputFormatProcessorJSONCompact(*this); registerOutputFormatProcessorJSONCompact(*this);
registerOutputFormatProcessorJSONEachRowWithProgress(*this);
registerOutputFormatProcessorXML(*this); registerOutputFormatProcessorXML(*this);
registerOutputFormatProcessorODBCDriver2(*this); registerOutputFormatProcessorODBCDriver2(*this);
registerOutputFormatProcessorNull(*this); registerOutputFormatProcessorNull(*this);
registerOutputFormatProcessorMySQLWire(*this); registerOutputFormatProcessorMySQLWire(*this);
registerOutputFormatProcessorMarkdown(*this); registerOutputFormatProcessorMarkdown(*this);
registerOutputFormatProcessorPostgreSQLWire(*this); registerOutputFormatProcessorPostgreSQLWire(*this);
registerInputFormatProcessorRegexp(*this);
registerInputFormatProcessorJSONAsString(*this);
#if !defined(ARCADIA_BUILD)
registerInputFormatProcessorCapnProto(*this);
#endif
} }
FormatFactory & FormatFactory::instance() FormatFactory & FormatFactory::instance()

69
src/Formats/FormatFactory.h
View File

@ -141,73 +141,4 @@ private:
const Creators & getCreators(const String & name) const; const Creators & getCreators(const String & name) const;
}; };
/// Formats for both input/output.
void registerInputFormatNative(FormatFactory & factory);
void registerOutputFormatNative(FormatFactory & factory);
void registerInputFormatProcessorNative(FormatFactory & factory);
void registerOutputFormatProcessorNative(FormatFactory & factory);
void registerInputFormatProcessorRowBinary(FormatFactory & factory);
void registerOutputFormatProcessorRowBinary(FormatFactory & factory);
void registerInputFormatProcessorTabSeparated(FormatFactory & factory);
void registerOutputFormatProcessorTabSeparated(FormatFactory & factory);
void registerInputFormatProcessorValues(FormatFactory & factory);
void registerOutputFormatProcessorValues(FormatFactory & factory);
void registerInputFormatProcessorCSV(FormatFactory & factory);
void registerOutputFormatProcessorCSV(FormatFactory & factory);
void registerInputFormatProcessorTSKV(FormatFactory & factory);
void registerOutputFormatProcessorTSKV(FormatFactory & factory);
void registerInputFormatProcessorJSONEachRow(FormatFactory & factory);
void registerOutputFormatProcessorJSONEachRow(FormatFactory & factory);
void registerInputFormatProcessorJSONCompactEachRow(FormatFactory & factory);
void registerOutputFormatProcessorJSONCompactEachRow(FormatFactory & factory);
void registerInputFormatProcessorParquet(FormatFactory & factory);
void registerOutputFormatProcessorParquet(FormatFactory & factory);
void registerInputFormatProcessorArrow(FormatFactory & factory);
void registerOutputFormatProcessorArrow(FormatFactory & factory);
void registerInputFormatProcessorProtobuf(FormatFactory & factory);
void registerOutputFormatProcessorProtobuf(FormatFactory & factory);
void registerInputFormatProcessorAvro(FormatFactory & factory);
void registerOutputFormatProcessorAvro(FormatFactory & factory);
void registerInputFormatProcessorTemplate(FormatFactory & factory);
void registerOutputFormatProcessorTemplate(FormatFactory & factory);
void registerInputFormatProcessorMsgPack(FormatFactory & factory);
void registerOutputFormatProcessorMsgPack(FormatFactory & factory);
void registerInputFormatProcessorORC(FormatFactory & factory);
void registerOutputFormatProcessorORC(FormatFactory & factory);
/// File Segmentation Engines for parallel reading
void registerFileSegmentationEngineTabSeparated(FormatFactory & factory);
void registerFileSegmentationEngineCSV(FormatFactory & factory);
void registerFileSegmentationEngineJSONEachRow(FormatFactory & factory);
void registerFileSegmentationEngineRegexp(FormatFactory & factory);
void registerFileSegmentationEngineJSONAsString(FormatFactory & factory);
/// Output only (presentational) formats.
void registerOutputFormatNull(FormatFactory & factory);
void registerOutputFormatProcessorPretty(FormatFactory & factory);
void registerOutputFormatProcessorPrettyCompact(FormatFactory & factory);
void registerOutputFormatProcessorPrettySpace(FormatFactory & factory);
void registerOutputFormatProcessorPrettyASCII(FormatFactory & factory);
void registerOutputFormatProcessorVertical(FormatFactory & factory);
void registerOutputFormatProcessorJSON(FormatFactory & factory);
void registerOutputFormatProcessorJSONCompact(FormatFactory & factory);
void registerOutputFormatProcessorJSONEachRowWithProgress(FormatFactory & factory);
void registerOutputFormatProcessorXML(FormatFactory & factory);
void registerOutputFormatProcessorODBCDriver2(FormatFactory & factory);
void registerOutputFormatProcessorNull(FormatFactory & factory);
void registerOutputFormatProcessorMySQLWire(FormatFactory & factory);
void registerOutputFormatProcessorMarkdown(FormatFactory & factory);
void registerOutputFormatProcessorPostgreSQLWire(FormatFactory & factory);
/// Input only formats.
void registerInputFormatProcessorCapnProto(FormatFactory & factory);
void registerInputFormatProcessorRegexp(FormatFactory & factory);
void registerInputFormatProcessorJSONAsString(FormatFactory & factory);
} }

24
src/Functions/CMakeLists.txt
View File

@ -53,8 +53,28 @@ endif()
target_include_directories(clickhouse_functions SYSTEM PRIVATE ${SPARSEHASH_INCLUDE_DIR}) target_include_directories(clickhouse_functions SYSTEM PRIVATE ${SPARSEHASH_INCLUDE_DIR})
# Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size. if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE"
target_compile_options(clickhouse_functions PRIVATE "-g0") OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO"
OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL")
set (STRIP_DSF_DEFAULT ON)
else()
set (STRIP_DSF_DEFAULT OFF)
endif()
option(STRIP_DEBUG_SYMBOLS_FUNCTIONS
"Do not generate debugger info for ClickHouse functions.
Provides faster linking and lower binary size.
Tradeoff is the inability to debug some source files with e.g. gdb
(empty stack frames and no local variables)."
${STRIP_DSF_DEFAULT})
if (STRIP_DEBUG_SYMBOLS_FUNCTIONS)
message(WARNING "Not generating debugger info for ClickHouse functions")
target_compile_options(clickhouse_functions PRIVATE "-g0")
else()
message(STATUS "Generating debugger info for ClickHouse functions")
endif()
if (USE_ICU) if (USE_ICU)
target_link_libraries (clickhouse_functions PRIVATE ${ICU_LIBRARIES}) target_link_libraries (clickhouse_functions PRIVATE ${ICU_LIBRARIES})

5
src/Functions/GatherUtils/CMakeLists.txt
View File

@ -3,5 +3,6 @@ add_headers_and_sources(clickhouse_functions_gatherutils .)
add_library(clickhouse_functions_gatherutils ${clickhouse_functions_gatherutils_sources} ${clickhouse_functions_gatherutils_headers}) add_library(clickhouse_functions_gatherutils ${clickhouse_functions_gatherutils_sources} ${clickhouse_functions_gatherutils_headers})
target_link_libraries(clickhouse_functions_gatherutils PRIVATE dbms) target_link_libraries(clickhouse_functions_gatherutils PRIVATE dbms)
# Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size. if (STRIP_DEBUG_SYMBOLS_FUNCTIONS)
target_compile_options(clickhouse_functions_gatherutils PRIVATE "-g0") target_compile_options(clickhouse_functions_gatherutils PRIVATE "-g0")
endif()

5
src/Functions/URL/CMakeLists.txt
View File

@ -3,8 +3,9 @@ add_headers_and_sources(clickhouse_functions_url .)
add_library(clickhouse_functions_url ${clickhouse_functions_url_sources} ${clickhouse_functions_url_headers}) add_library(clickhouse_functions_url ${clickhouse_functions_url_sources} ${clickhouse_functions_url_headers})
target_link_libraries(clickhouse_functions_url PRIVATE dbms) target_link_libraries(clickhouse_functions_url PRIVATE dbms)
# Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size. if (STRIP_DEBUG_SYMBOLS_FUNCTIONS)
target_compile_options(clickhouse_functions_url PRIVATE "-g0") target_compile_options(clickhouse_functions_url PRIVATE "-g0")
endif()
# TODO: move Functions/Regexps.h to some lib and use here # TODO: move Functions/Regexps.h to some lib and use here
target_link_libraries(clickhouse_functions_url PRIVATE hyperscan) target_link_libraries(clickhouse_functions_url PRIVATE hyperscan)

5
src/Functions/array/CMakeLists.txt
View File

@ -3,5 +3,6 @@ add_headers_and_sources(clickhouse_functions_array .)
add_library(clickhouse_functions_array ${clickhouse_functions_array_sources} ${clickhouse_functions_array_headers}) add_library(clickhouse_functions_array ${clickhouse_functions_array_sources} ${clickhouse_functions_array_headers})
target_link_libraries(clickhouse_functions_array PRIVATE dbms clickhouse_functions_gatherutils) target_link_libraries(clickhouse_functions_array PRIVATE dbms clickhouse_functions_gatherutils)
# Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size. if (STRIP_DEBUG_SYMBOLS_FUNCTIONS)
target_compile_options(clickhouse_functions_array PRIVATE "-g0") target_compile_options(clickhouse_functions_array PRIVATE "-g0")
endif()

50
src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp
View File

@ -1,4 +1,5 @@
#include <IO/ReadHelpers.h> #include <IO/ReadHelpers.h>
#include <IO/ReadBufferFromString.h>
#include <Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.h> #include <Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.h>
#include <Formats/FormatFactory.h> #include <Formats/FormatFactory.h>
@ -19,8 +20,9 @@ JSONCompactEachRowRowInputFormat::JSONCompactEachRowRowInputFormat(ReadBuffer &
const Block & header_, const Block & header_,
Params params_, Params params_,
const FormatSettings & format_settings_, const FormatSettings & format_settings_,
bool with_names_) bool with_names_,
: IRowInputFormat(header_, in_, std::move(params_)), format_settings(format_settings_), with_names(with_names_) bool yield_strings_)
: IRowInputFormat(header_, in_, std::move(params_)), format_settings(format_settings_), with_names(with_names_), yield_strings(yield_strings_)
{ {
const auto & sample = getPort().getHeader(); const auto & sample = getPort().getHeader();
size_t num_columns = sample.columns(); size_t num_columns = sample.columns();
@ -200,10 +202,26 @@ void JSONCompactEachRowRowInputFormat::readField(size_t index, MutableColumns &
{ {
read_columns[index] = true; read_columns[index] = true;
const auto & type = data_types[index]; const auto & type = data_types[index];
if (format_settings.null_as_default && !type->isNullable())
read_columns[index] = DataTypeNullable::deserializeTextJSON(*columns[index], in, format_settings, type); if (yield_strings)
{
String str;
readJSONString(str, in);
ReadBufferFromString buf(str);
if (format_settings.null_as_default && !type->isNullable())
read_columns[index] = DataTypeNullable::deserializeWholeText(*columns[index], buf, format_settings, type);
else
type->deserializeAsWholeText(*columns[index], buf, format_settings);
}
else else
type->deserializeAsTextJSON(*columns[index], in, format_settings); {
if (format_settings.null_as_default && !type->isNullable())
read_columns[index] = DataTypeNullable::deserializeTextJSON(*columns[index], in, format_settings, type);
else
type->deserializeAsTextJSON(*columns[index], in, format_settings);
}
} }
catch (Exception & e) catch (Exception & e)
{ {
@ -225,7 +243,7 @@ void registerInputFormatProcessorJSONCompactEachRow(FormatFactory & factory)
IRowInputFormat::Params params, IRowInputFormat::Params params,
const FormatSettings & settings) const FormatSettings & settings)
{ {
return std::make_shared<JSONCompactEachRowRowInputFormat>(buf, sample, std::move(params), settings, false); return std::make_shared<JSONCompactEachRowRowInputFormat>(buf, sample, std::move(params), settings, false, false);
}); });
factory.registerInputFormatProcessor("JSONCompactEachRowWithNamesAndTypes", []( factory.registerInputFormatProcessor("JSONCompactEachRowWithNamesAndTypes", [](
@ -234,7 +252,25 @@ void registerInputFormatProcessorJSONCompactEachRow(FormatFactory & factory)
IRowInputFormat::Params params, IRowInputFormat::Params params,
const FormatSettings & settings) const FormatSettings & settings)
{ {
return std::make_shared<JSONCompactEachRowRowInputFormat>(buf, sample, std::move(params), settings, true); return std::make_shared<JSONCompactEachRowRowInputFormat>(buf, sample, std::move(params), settings, true, false);
});
factory.registerInputFormatProcessor("JSONCompactStringsEachRow", [](
ReadBuffer & buf,
const Block & sample,
IRowInputFormat::Params params,
const FormatSettings & settings)
{
return std::make_shared<JSONCompactEachRowRowInputFormat>(buf, sample, std::move(params), settings, false, true);
});
factory.registerInputFormatProcessor("JSONCompactStringsEachRowWithNamesAndTypes", [](
ReadBuffer & buf,
const Block & sample,
IRowInputFormat::Params params,
const FormatSettings & settings)
{
return std::make_shared<JSONCompactEachRowRowInputFormat>(buf, sample, std::move(params), settings, true, true);
}); });
} }

20
src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.h
View File

@ -1,7 +1,5 @@
#pragma once #pragma once
#pragma once
#include <Core/Block.h> #include <Core/Block.h>
#include <Processors/Formats/IRowInputFormat.h> #include <Processors/Formats/IRowInputFormat.h>
#include <Formats/FormatSettings.h> #include <Formats/FormatSettings.h>
@ -12,12 +10,23 @@ namespace DB
class ReadBuffer; class ReadBuffer;
/** A stream for reading data in JSONCompactEachRow and JSONCompactEachRowWithNamesAndTypes formats /** A stream for reading data in a bunch of formats:
* - JSONCompactEachRow
* - JSONCompactEachRowWithNamesAndTypes
* - JSONCompactStringsEachRow
* - JSONCompactStringsEachRowWithNamesAndTypes
*
*/ */
class JSONCompactEachRowRowInputFormat : public IRowInputFormat class JSONCompactEachRowRowInputFormat : public IRowInputFormat
{ {
public: public:
JSONCompactEachRowRowInputFormat(ReadBuffer & in_, const Block & header_, Params params_, const FormatSettings & format_settings_, bool with_names_); JSONCompactEachRowRowInputFormat(
ReadBuffer & in_,
const Block & header_,
Params params_,
const FormatSettings & format_settings_,
bool with_names_,
bool yield_strings_);
String getName() const override { return "JSONCompactEachRowRowInputFormat"; } String getName() const override { return "JSONCompactEachRowRowInputFormat"; }
@ -48,7 +57,10 @@ private:
/// This is for the correct exceptions in skipping unknown fields. /// This is for the correct exceptions in skipping unknown fields.
std::vector<String> names_of_columns; std::vector<String> names_of_columns;
/// For *WithNamesAndTypes formats.
bool with_names; bool with_names;
/// For JSONCompactString* formats.
bool yield_strings;
}; };
} }

37
src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.cpp
View File

@ -12,8 +12,9 @@ JSONCompactEachRowRowOutputFormat::JSONCompactEachRowRowOutputFormat(WriteBuffer
const Block & header_, const Block & header_,
FormatFactory::WriteCallback callback, FormatFactory::WriteCallback callback,
const FormatSettings & settings_, const FormatSettings & settings_,
bool with_names_) bool with_names_,
: IRowOutputFormat(header_, out_, callback), settings(settings_), with_names(with_names_) bool yield_strings_)
: IRowOutputFormat(header_, out_, callback), settings(settings_), with_names(with_names_), yield_strings(yield_strings_)
{ {
const auto & sample = getPort(PortKind::Main).getHeader(); const auto & sample = getPort(PortKind::Main).getHeader();
NamesAndTypesList columns(sample.getNamesAndTypesList()); NamesAndTypesList columns(sample.getNamesAndTypesList());
@ -23,7 +24,15 @@ JSONCompactEachRowRowOutputFormat::JSONCompactEachRowRowOutputFormat(WriteBuffer
void JSONCompactEachRowRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num) void JSONCompactEachRowRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num)
{ {
type.serializeAsTextJSON(column, row_num, out, settings); if (yield_strings)
{
WriteBufferFromOwnString buf;
type.serializeAsText(column, row_num, buf, settings);
writeJSONString(buf.str(), out, settings);
}
else
type.serializeAsTextJSON(column, row_num, out, settings);
} }
@ -97,7 +106,7 @@ void registerOutputFormatProcessorJSONCompactEachRow(FormatFactory & factory)
FormatFactory::WriteCallback callback, FormatFactory::WriteCallback callback,
const FormatSettings & format_settings) const FormatSettings & format_settings)
{ {
return std::make_shared<JSONCompactEachRowRowOutputFormat>(buf, sample, callback, format_settings, false); return std::make_shared<JSONCompactEachRowRowOutputFormat>(buf, sample, callback, format_settings, false, false);
}); });
factory.registerOutputFormatProcessor("JSONCompactEachRowWithNamesAndTypes", []( factory.registerOutputFormatProcessor("JSONCompactEachRowWithNamesAndTypes", [](
@ -106,7 +115,25 @@ void registerOutputFormatProcessorJSONCompactEachRow(FormatFactory & factory)
FormatFactory::WriteCallback callback, FormatFactory::WriteCallback callback,
const FormatSettings &format_settings) const FormatSettings &format_settings)
{ {
return std::make_shared<JSONCompactEachRowRowOutputFormat>(buf, sample, callback, format_settings, true); return std::make_shared<JSONCompactEachRowRowOutputFormat>(buf, sample, callback, format_settings, true, false);
});
factory.registerOutputFormatProcessor("JSONCompactStringsEachRow", [](
WriteBuffer & buf,
const Block & sample,
FormatFactory::WriteCallback callback,
const FormatSettings & format_settings)
{
return std::make_shared<JSONCompactEachRowRowOutputFormat>(buf, sample, callback, format_settings, false, true);
});
factory.registerOutputFormatProcessor("JSONCompactStringsEachRowWithNamesAndTypes", [](
WriteBuffer &buf,
const Block &sample,
FormatFactory::WriteCallback callback,
const FormatSettings &format_settings)
{
return std::make_shared<JSONCompactEachRowRowOutputFormat>(buf, sample, callback, format_settings, true, true);
}); });
} }

9
src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h
View File

@ -15,7 +15,13 @@ namespace DB
class JSONCompactEachRowRowOutputFormat : public IRowOutputFormat class JSONCompactEachRowRowOutputFormat : public IRowOutputFormat
{ {
public: public:
JSONCompactEachRowRowOutputFormat(WriteBuffer & out_, const Block & header_, FormatFactory::WriteCallback callback, const FormatSettings & settings_, bool with_names); JSONCompactEachRowRowOutputFormat(
WriteBuffer & out_,
const Block & header_,
FormatFactory::WriteCallback callback,
const FormatSettings & settings_,
bool with_names_,
bool yield_strings_);
String getName() const override { return "JSONCompactEachRowRowOutputFormat"; } String getName() const override { return "JSONCompactEachRowRowOutputFormat"; }
@ -41,5 +47,6 @@ private:
NamesAndTypes fields; NamesAndTypes fields;
bool with_names; bool with_names;
bool yield_strings;
}; };
} }

30
src/Processors/Formats/Impl/JSONCompactRowOutputFormat.cpp
View File

@ -8,15 +8,28 @@ namespace DB
{ {
JSONCompactRowOutputFormat::JSONCompactRowOutputFormat( JSONCompactRowOutputFormat::JSONCompactRowOutputFormat(
WriteBuffer & out_, const Block & header, FormatFactory::WriteCallback callback, const FormatSettings & settings_) WriteBuffer & out_,
: JSONRowOutputFormat(out_, header, callback, settings_) const Block & header,
FormatFactory::WriteCallback callback,
const FormatSettings & settings_,
bool yield_strings_)
: JSONRowOutputFormat(out_, header, callback, settings_, yield_strings_)
{ {
} }
void JSONCompactRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num) void JSONCompactRowOutputFormat::writeField(const IColumn & column, const IDataType & type, size_t row_num)
{ {
type.serializeAsTextJSON(column, row_num, *ostr, settings); if (yield_strings)
{
WriteBufferFromOwnString buf;
type.serializeAsText(column, row_num, buf, settings);
writeJSONString(buf.str(), *ostr, settings);
}
else
type.serializeAsTextJSON(column, row_num, *ostr, settings);
++field_number; ++field_number;
} }
@ -83,7 +96,16 @@ void registerOutputFormatProcessorJSONCompact(FormatFactory & factory)
FormatFactory::WriteCallback callback, FormatFactory::WriteCallback callback,
const FormatSettings & format_settings) const FormatSettings & format_settings)
{ {
return std::make_shared<JSONCompactRowOutputFormat>(buf, sample, callback, format_settings); return std::make_shared<JSONCompactRowOutputFormat>(buf, sample, callback, format_settings, false);
});
factory.registerOutputFormatProcessor("JSONCompactStrings", [](
WriteBuffer & buf,
const Block & sample,
FormatFactory::WriteCallback callback,
const FormatSettings & format_settings)
{
return std::make_shared<JSONCompactRowOutputFormat>(buf, sample, callback, format_settings, true);
}); });
} }

10
src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h
View File

@ -11,12 +11,17 @@ namespace DB
struct FormatSettings; struct FormatSettings;
/** The stream for outputting data in the JSONCompact format. /** The stream for outputting data in the JSONCompact- formats.
*/ */
class JSONCompactRowOutputFormat : public JSONRowOutputFormat class JSONCompactRowOutputFormat : public JSONRowOutputFormat
{ {
public: public:
JSONCompactRowOutputFormat(WriteBuffer & out_, const Block & header, FormatFactory::WriteCallback callback, const FormatSettings & settings_); JSONCompactRowOutputFormat(
WriteBuffer & out_,
const Block & header,
FormatFactory::WriteCallback callback,
const FormatSettings & settings_,
bool yield_strings_);
String getName() const override { return "JSONCompactRowOutputFormat"; } String getName() const override { return "JSONCompactRowOutputFormat"; }
@ -37,7 +42,6 @@ protected:
} }
void writeTotalsFieldDelimiter() override; void writeTotalsFieldDelimiter() override;
}; };
} }

43
src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
View File

@ -1,4 +1,5 @@
#include <IO/ReadHelpers.h> #include <IO/ReadHelpers.h>
#include <IO/ReadBufferFromString.h>
#include <Processors/Formats/Impl/JSONEachRowRowInputFormat.h> #include <Processors/Formats/Impl/JSONEachRowRowInputFormat.h>
#include <Formats/JSONEachRowUtils.h> #include <Formats/JSONEachRowUtils.h>
@ -29,8 +30,12 @@ enum
JSONEachRowRowInputFormat::JSONEachRowRowInputFormat( JSONEachRowRowInputFormat::JSONEachRowRowInputFormat(
ReadBuffer & in_, const Block & header_, Params params_, const FormatSettings & format_settings_) ReadBuffer & in_,
: IRowInputFormat(header_, in_, std::move(params_)), format_settings(format_settings_), name_map(header_.columns()) const Block & header_,
Params params_,
const FormatSettings & format_settings_,
bool yield_strings_)
: IRowInputFormat(header_, in_, std::move(params_)), format_settings(format_settings_), name_map(header_.columns()), yield_strings(yield_strings_)
{ {
size_t num_columns = getPort().getHeader().columns(); size_t num_columns = getPort().getHeader().columns();
for (size_t i = 0; i < num_columns; ++i) for (size_t i = 0; i < num_columns; ++i)
@ -135,10 +140,26 @@ void JSONEachRowRowInputFormat::readField(size_t index, MutableColumns & columns
{ {
seen_columns[index] = read_columns[index] = true; seen_columns[index] = read_columns[index] = true;
const auto & type = getPort().getHeader().getByPosition(index).type; const auto & type = getPort().getHeader().getByPosition(index).type;
if (format_settings.null_as_default && !type->isNullable())
read_columns[index] = DataTypeNullable::deserializeTextJSON(*columns[index], in, format_settings, type); if (yield_strings)
{
String str;
readJSONString(str, in);
ReadBufferFromString buf(str);
if (format_settings.null_as_default && !type->isNullable())
read_columns[index] = DataTypeNullable::deserializeWholeText(*columns[index], buf, format_settings, type);
else
type->deserializeAsWholeText(*columns[index], buf, format_settings);
}
else else
type->deserializeAsTextJSON(*columns[index], in, format_settings); {
if (format_settings.null_as_default && !type->isNullable())
read_columns[index] = DataTypeNullable::deserializeTextJSON(*columns[index], in, format_settings, type);
else
type->deserializeAsTextJSON(*columns[index], in, format_settings);
}
} }
catch (Exception & e) catch (Exception & e)
{ {
@ -318,13 +339,23 @@ void registerInputFormatProcessorJSONEachRow(FormatFactory & factory)
IRowInputFormat::Params params, IRowInputFormat::Params params,
const FormatSettings & settings) const FormatSettings & settings)
{ {
return std::make_shared<JSONEachRowRowInputFormat>(buf, sample, std::move(params), settings); return std::make_shared<JSONEachRowRowInputFormat>(buf, sample, std::move(params), settings, false);
});
factory.registerInputFormatProcessor("JSONStringsEachRow", [](
ReadBuffer & buf,
const Block & sample,
IRowInputFormat::Params params,
const FormatSettings & settings)
{
return std::make_shared<JSONEachRowRowInputFormat>(buf, sample, std::move(params), settings, true);
}); });
} }
void registerFileSegmentationEngineJSONEachRow(FormatFactory & factory) void registerFileSegmentationEngineJSONEachRow(FormatFactory & factory)
{ {
factory.registerFileSegmentationEngine("JSONEachRow", &fileSegmentationEngineJSONEachRowImpl); factory.registerFileSegmentationEngine("JSONEachRow", &fileSegmentationEngineJSONEachRowImpl);
factory.registerFileSegmentationEngine("JSONStringsEachRow", &fileSegmentationEngineJSONEachRowImpl);
} }
} }

9
src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h
View File

@ -20,7 +20,12 @@ class ReadBuffer;
class JSONEachRowRowInputFormat : public IRowInputFormat class JSONEachRowRowInputFormat : public IRowInputFormat
{ {
public: public:
JSONEachRowRowInputFormat(ReadBuffer & in_, const Block & header_, Params params_, const FormatSettings & format_settings_); JSONEachRowRowInputFormat(
ReadBuffer & in_,
const Block & header_,
Params params_,
const FormatSettings & format_settings_,
bool yield_strings_);
String getName() const override { return "JSONEachRowRowInputFormat"; } String getName() const override { return "JSONEachRowRowInputFormat"; }
@ -75,6 +80,8 @@ private:
bool data_in_square_brackets = false; bool data_in_square_brackets = false;
bool allow_new_rows = true; bool allow_new_rows = true;
bool yield_strings;
}; };
} }

32
src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.cpp
View File

@ -8,8 +8,13 @@ namespace DB
{ {
JSONEachRowRowOutputFormat::JSONEachRowRowOutputFormat(WriteBuffer & out_, const Block & header_, FormatFactory::WriteCallback callback, const FormatSettings & settings_) JSONEachRowRowOutputFormat::JSONEachRowRowOutputFormat(
: IRowOutputFormat(header_, out_, callback), settings(settings_) WriteBuffer & out_,
const Block & header_,
FormatFactory::WriteCallback callback,
const FormatSettings & settings_,
bool yield_strings_)
: IRowOutputFormat(header_, out_, callback), settings(settings_), yield_strings(yield_strings_)
{ {
const auto & sample = getPort(PortKind::Main).getHeader(); const auto & sample = getPort(PortKind::Main).getHeader();
size_t columns = sample.columns(); size_t columns = sample.columns();
@ -27,7 +32,17 @@ void JSONEachRowRowOutputFormat::writeField(const IColumn & column, const IDataT
{ {
writeString(fields[field_number], out); writeString(fields[field_number], out);
writeChar(':', out); writeChar(':', out);
type.serializeAsTextJSON(column, row_num, out, settings);
if (yield_strings)
{
WriteBufferFromOwnString buf;
type.serializeAsText(column, row_num, buf, settings);
writeJSONString(buf.str(), out, settings);
}
else
type.serializeAsTextJSON(column, row_num, out, settings);
++field_number; ++field_number;
} }
@ -59,7 +74,16 @@ void registerOutputFormatProcessorJSONEachRow(FormatFactory & factory)
FormatFactory::WriteCallback callback, FormatFactory::WriteCallback callback,
const FormatSettings & format_settings) const FormatSettings & format_settings)
{ {
return std::make_shared<JSONEachRowRowOutputFormat>(buf, sample, callback, format_settings); return std::make_shared<JSONEachRowRowOutputFormat>(buf, sample, callback, format_settings, false);
});
factory.registerOutputFormatProcessor("JSONStringsEachRow", [](
WriteBuffer & buf,
const Block & sample,
FormatFactory::WriteCallback callback,
const FormatSettings & format_settings)
{
return std::make_shared<JSONEachRowRowOutputFormat>(buf, sample, callback, format_settings, true);
}); });
} }

10
src/Processors/Formats/Impl/JSONEachRowRowOutputFormat.h
View File

@ -15,7 +15,12 @@ namespace DB
class JSONEachRowRowOutputFormat : public IRowOutputFormat class JSONEachRowRowOutputFormat : public IRowOutputFormat
{ {
public: public:
JSONEachRowRowOutputFormat(WriteBuffer & out_, const Block & header_, FormatFactory::WriteCallback callback, const FormatSettings & settings_); JSONEachRowRowOutputFormat(
WriteBuffer & out_,
const Block & header_,
FormatFactory::WriteCallback callback,
const FormatSettings & settings_,
bool yield_strings_);
String getName() const override { return "JSONEachRowRowOutputFormat"; } String getName() const override { return "JSONEachRowRowOutputFormat"; }
@ -35,6 +40,9 @@ private:
Names fields; Names fields;
FormatSettings settings; FormatSettings settings;
protected:
bool yield_strings;
}; };
} }

11
src/Processors/Formats/Impl/JSONEachRowWithProgressRowOutputFormat.cpp
View File

@ -36,7 +36,16 @@ void registerOutputFormatProcessorJSONEachRowWithProgress(FormatFactory & factor
FormatFactory::WriteCallback callback, FormatFactory::WriteCallback callback,
const FormatSettings & format_settings) const FormatSettings & format_settings)
{ {
return std::make_shared<JSONEachRowWithProgressRowOutputFormat>(buf, sample, callback, format_settings); return std::make_shared<JSONEachRowWithProgressRowOutputFormat>(buf, sample, callback, format_settings, false);
});
factory.registerOutputFormatProcessor("JSONStringsEachRowWithProgress", [](
WriteBuffer & buf,
const Block & sample,
FormatFactory::WriteCallback callback,
const FormatSettings & format_settings)
{
return std::make_shared<JSONEachRowWithProgressRowOutputFormat>(buf, sample, callback, format_settings, true);
}); });
} }

44
src/Processors/Formats/Impl/JSONRowOutputFormat.cpp
View File

@ -7,8 +7,13 @@
namespace DB namespace DB
{ {
JSONRowOutputFormat::JSONRowOutputFormat(WriteBuffer & out_, const Block & header, FormatFactory::WriteCallback callback, const FormatSettings & settings_) JSONRowOutputFormat::JSONRowOutputFormat(
: IRowOutputFormat(header, out_, callback), settings(settings_) WriteBuffer & out_,
const Block & header,
FormatFactory::WriteCallback callback,
const FormatSettings & settings_,
bool yield_strings_)
: IRowOutputFormat(header, out_, callback), settings(settings_), yield_strings(yield_strings_)
{ {
const auto & sample = getPort(PortKind::Main).getHeader(); const auto & sample = getPort(PortKind::Main).getHeader();
NamesAndTypesList columns(sample.getNamesAndTypesList()); NamesAndTypesList columns(sample.getNamesAndTypesList());
@ -71,7 +76,17 @@ void JSONRowOutputFormat::writeField(const IColumn & column, const IDataType & t
writeCString("\t\t\t", *ostr); writeCString("\t\t\t", *ostr);
writeString(fields[field_number].name, *ostr); writeString(fields[field_number].name, *ostr);
writeCString(": ", *ostr); writeCString(": ", *ostr);
type.serializeAsTextJSON(column, row_num, *ostr, settings);
if (yield_strings)
{
WriteBufferFromOwnString buf;
type.serializeAsText(column, row_num, buf, settings);
writeJSONString(buf.str(), *ostr, settings);
}
else
type.serializeAsTextJSON(column, row_num, *ostr, settings);
++field_number; ++field_number;
} }
@ -80,7 +95,17 @@ void JSONRowOutputFormat::writeTotalsField(const IColumn & column, const IDataTy
writeCString("\t\t", *ostr); writeCString("\t\t", *ostr);
writeString(fields[field_number].name, *ostr); writeString(fields[field_number].name, *ostr);
writeCString(": ", *ostr); writeCString(": ", *ostr);
type.serializeAsTextJSON(column, row_num, *ostr, settings);
if (yield_strings)
{
WriteBufferFromOwnString buf;
type.serializeAsText(column, row_num, buf, settings);
writeJSONString(buf.str(), *ostr, settings);
}
else
type.serializeAsTextJSON(column, row_num, *ostr, settings);
++field_number; ++field_number;
} }
@ -249,7 +274,16 @@ void registerOutputFormatProcessorJSON(FormatFactory & factory)
FormatFactory::WriteCallback callback, FormatFactory::WriteCallback callback,
const FormatSettings & format_settings) const FormatSettings & format_settings)
{ {
return std::make_shared<JSONRowOutputFormat>(buf, sample, callback, format_settings); return std::make_shared<JSONRowOutputFormat>(buf, sample, callback, format_settings, false);
});
factory.registerOutputFormatProcessor("JSONStrings", [](
WriteBuffer & buf,
const Block & sample,
FormatFactory::WriteCallback callback,
const FormatSettings & format_settings)
{
return std::make_shared<JSONRowOutputFormat>(buf, sample, callback, format_settings, true);
}); });
} }

9
src/Processors/Formats/Impl/JSONRowOutputFormat.h
View File

@ -16,7 +16,12 @@ namespace DB
class JSONRowOutputFormat : public IRowOutputFormat class JSONRowOutputFormat : public IRowOutputFormat
{ {
public: public:
JSONRowOutputFormat(WriteBuffer & out_, const Block & header, FormatFactory::WriteCallback callback, const FormatSettings & settings_); JSONRowOutputFormat(
WriteBuffer & out_,
const Block & header,
FormatFactory::WriteCallback callback,
const FormatSettings & settings_,
bool yield_strings_);
String getName() const override { return "JSONRowOutputFormat"; } String getName() const override { return "JSONRowOutputFormat"; }
@ -78,6 +83,8 @@ protected:
Progress progress; Progress progress;
Stopwatch watch; Stopwatch watch;
FormatSettings settings; FormatSettings settings;
bool yield_strings;
}; };
} }

16
src/Processors/IAccumulatingTransform.cpp
View File

@ -49,13 +49,15 @@ IAccumulatingTransform::Status IAccumulatingTransform::prepare()
return Status::Finished; return Status::Finished;
} }
/// Close input if flag was set manually. if (input.isFinished())
finished_input = true;
if (finished_input) if (finished_input)
{
/// Close input if flag was set manually.
input.close(); input.close();
/// Read from totals port if has it. /// Read from totals port if has it.
if (input.isFinished())
{
if (inputs.size() > 1) if (inputs.size() > 1)
{ {
auto & totals_input = inputs.back(); auto & totals_input = inputs.back();
@ -69,12 +71,8 @@ IAccumulatingTransform::Status IAccumulatingTransform::prepare()
totals_input.close(); totals_input.close();
} }
} }
}
/// Generate output block. /// Generate output block.
if (input.isFinished())
{
finished_input = true;
return Status::Ready; return Status::Ready;
} }

12
src/Storages/SelectQueryDescription.cpp
View File

@ -105,14 +105,14 @@ SelectQueryDescription SelectQueryDescription::getSelectQueryFromASTForMatView(c
if (new_select.list_of_selects->children.size() != 1) if (new_select.list_of_selects->children.size() != 1)
throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW); throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW);
SelectQueryDescription result; auto & new_inner_query = new_select.list_of_selects->children.at(0);
auto & select_query = new_inner_query->as<ASTSelectQuery &>();
result.inner_query = new_select.list_of_selects->children.at(0)->clone();
auto & select_query = result.inner_query->as<ASTSelectQuery &>();
checkAllowedQueries(select_query); checkAllowedQueries(select_query);
SelectQueryDescription result;
result.select_table_id = extractDependentTableFromSelectQuery(select_query, context); result.select_table_id = extractDependentTableFromSelectQuery(select_query, context);
result.select_query = select->clone(); result.select_query = new_select.clone();
result.inner_query = new_inner_query->clone();
return result; return result;
} }

34
src/Storages/StorageReplicatedMergeTree.cpp
View File

@ -216,7 +216,31 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree(
getStorageID().getFullTableName() + " (StorageReplicatedMergeTree::mutationsFinalizingTask)", [this] { mutationsFinalizingTask(); }); getStorageID().getFullTableName() + " (StorageReplicatedMergeTree::mutationsFinalizingTask)", [this] { mutationsFinalizingTask(); });
if (global_context.hasZooKeeper()) if (global_context.hasZooKeeper())
current_zookeeper = global_context.getZooKeeper(); {
/// It's possible for getZooKeeper() to timeout if zookeeper host(s) can't
/// be reached. In such cases Poco::Exception is thrown after a connection
/// timeout - refer to src/Common/ZooKeeper/ZooKeeperImpl.cpp:866 for more info.
///
/// Side effect of this is that the CreateQuery gets interrupted and it exits.
/// But the data Directories for the tables being created aren't cleaned up.
/// This unclean state will hinder table creation on any retries and will
/// complain that the Directory for table already exists.
///
/// To achieve a clean state on failed table creations, catch this error and
/// call dropIfEmpty() method only if the operation isn't ATTACH then proceed
/// throwing the exception. Without this, the Directory for the tables need
/// to be manually deleted before retrying the CreateQuery.
try
{
current_zookeeper = global_context.getZooKeeper();
}
catch (...)
{
if (!attach)
dropIfEmpty();
throw;
}
}
bool skip_sanity_checks = false; bool skip_sanity_checks = false;
@ -239,7 +263,10 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree(
if (!current_zookeeper) if (!current_zookeeper)
{ {
if (!attach) if (!attach)
{
dropIfEmpty();
throw Exception("Can't create replicated table without ZooKeeper", ErrorCodes::NO_ZOOKEEPER); throw Exception("Can't create replicated table without ZooKeeper", ErrorCodes::NO_ZOOKEEPER);
}
/// Do not activate the replica. It will be readonly. /// Do not activate the replica. It will be readonly.
LOG_ERROR(log, "No ZooKeeper: table will be in readonly mode."); LOG_ERROR(log, "No ZooKeeper: table will be in readonly mode.");
@ -589,7 +616,10 @@ bool StorageReplicatedMergeTree::createTableIfNotExists(const StorageMetadataPtr
return true; return true;
} }
throw Exception("Cannot create table, because it is created concurrently every time or because of logical error", ErrorCodes::LOGICAL_ERROR); /// Do not use LOGICAL_ERROR code, because it may happen if user has specified wrong zookeeper_path
throw Exception("Cannot create table, because it is created concurrently every time "
"or because of wrong zookeeper_path "
"or because of logical error", ErrorCodes::REPLICA_IS_ALREADY_EXIST);
} }
void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metadata_snapshot) void StorageReplicatedMergeTree::createReplica(const StorageMetadataPtr & metadata_snapshot)

0
tests/integration/test_cleanup_dir_after_bad_zk_conn/__init__.py Normal file
View File

73
tests/integration/test_cleanup_dir_after_bad_zk_conn/test.py Normal file
View File

@ -0,0 +1,73 @@
import time
import pytest
from helpers.cluster import ClickHouseCluster
from helpers.network import PartitionManager
cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', with_zookeeper=True)
@pytest.fixture(scope="module")
def start_cluster():
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
# This tests if the data directory for a table is cleaned up if there is a Zookeeper
# connection exception during a CreateQuery operation involving ReplicatedMergeTree tables.
# Test flow is as follows:
# 1. Configure cluster with ZooKeeper and create a database.
# 2. Drop all connections to ZooKeeper.
# 3. Try creating the table and there will be a Poco:Exception.
# 4. Try creating the table again and there should not be any error
# that indicates that the directory for table already exists.
# 5. Final step is to restore ZooKeeper connection and verify that
# the table creation works.
def test_cleanup_dir_after_bad_zk_conn(start_cluster):
node1.query("CREATE DATABASE replica;")
query_create = '''CREATE TABLE replica.test
(
id Int64,
event_time DateTime
)
Engine=ReplicatedMergeTree('/clickhouse/tables/replica/test', 'node1')
PARTITION BY toYYYYMMDD(event_time)
ORDER BY id;'''
with PartitionManager() as pm:
pm.drop_instance_zk_connections(node1)
time.sleep(3)
error = node1.query_and_get_error(query_create)
assert "Poco::Exception. Code: 1000" and \
"All connection tries failed while connecting to ZooKeeper" in error
error = node1.query_and_get_error(query_create)
assert "Directory for table data data/replica/test/ already exists" not in error
node1.query(query_create)
node1.query('''INSERT INTO replica.test VALUES (1, now())''')
assert "1\n" in node1.query('''SELECT count() from replica.test FORMAT TSV''')
def test_cleanup_dir_after_wrong_replica_name(start_cluster):
node1.query("CREATE TABLE test2_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r1') ORDER BY n")
error = node1.query_and_get_error("CREATE TABLE test2_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r1') ORDER BY n")
assert "already exists" in error
node1.query("CREATE TABLE test_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test2/', 'r2') ORDER BY n")
def test_cleanup_dir_after_wrong_zk_path(start_cluster):
node1.query("CREATE TABLE test3_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test3/', 'r1') ORDER BY n")
error = node1.query_and_get_error("CREATE TABLE test3_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/', 'r2') ORDER BY n")
assert "Cannot create" in error
node1.query("CREATE TABLE test3_r2 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test3/', 'r2') ORDER BY n")
def test_attach_without_zk(start_cluster):
node1.query("CREATE TABLE test4_r1 (n UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test4/', 'r1') ORDER BY n")
node1.query("DETACH TABLE test4_r1")
with PartitionManager() as pm:
pm._add_rule({'probability': 0.5, 'source': node1.ip_address, 'destination_port': 2181, 'action': 'DROP'})
try:
node1.query("ATTACH TABLE test4_r1")
except:
pass
node1.query("ATTACH TABLE IF NOT EXISTS test4_r1")
node1.query("SELECT * FROM test4_r1")

5
tests/queries/0_stateless/01016_input_null_as_default.reference
View File

@ -18,6 +18,11 @@ JSONEachRow
1 world 3 2019-07-23 [1,2,3] ('tuple',3.14) 1 world 3 2019-07-23 [1,2,3] ('tuple',3.14)
2 Hello 123 2019-06-19 [] ('test',2.71828) 2 Hello 123 2019-06-19 [] ('test',2.71828)
3 Hello 42 2019-06-19 [1,2,3] ('default',0.75) 3 Hello 42 2019-06-19 [1,2,3] ('default',0.75)
JSONStringsEachRow
0 1 42 2019-07-22 [10,20,30] ('default',0)
1 world 3 2019-07-23 [1,2,3] ('tuple',3.14)
2 Hello 123 2019-06-19 [] ('test',2.71828)
3 Hello 42 2019-06-19 [1,2,3] ('default',0.75)
Template (Quoted) Template (Quoted)
0 1 42 2019-07-22 [10,20,30] ('default',0) 0 1 42 2019-07-22 [10,20,30] ('default',0)
1 world 3 2019-07-23 [1,2,3] ('tuple',3.14) 1 world 3 2019-07-23 [1,2,3] ('tuple',3.14)

8
tests/queries/0_stateless/01016_input_null_as_default.sh
View File

@ -38,6 +38,14 @@ echo '{"i": null, "s": "1", "n": null, "d": "2019-07-22", "a": [10, 20, 30], "t"
$CLICKHOUSE_CLIENT --query="SELECT * FROM null_as_default ORDER BY i"; $CLICKHOUSE_CLIENT --query="SELECT * FROM null_as_default ORDER BY i";
$CLICKHOUSE_CLIENT --query="TRUNCATE TABLE null_as_default"; $CLICKHOUSE_CLIENT --query="TRUNCATE TABLE null_as_default";
echo 'JSONStringsEachRow'
echo '{"i": "null", "s": "1", "n": "ᴺᵁᴸᴸ", "d": "2019-07-22", "a": "[10, 20, 30]", "t": "NULL"}
{"i": "1", "s": "world", "n": "3", "d": "2019-07-23", "a": "null", "t": "('\''tuple'\'', 3.14)"}
{"i": "2", "s": "null", "n": "123", "d": "null", "a": "[]", "t": "('\''test'\'', 2.71828)"}
{"i": "3", "s": "null", "n": "null", "d": "null", "a": "null", "t": "null"}' | $CLICKHOUSE_CLIENT --input_format_null_as_default=1 --query="INSERT INTO null_as_default FORMAT JSONStringsEachRow";
$CLICKHOUSE_CLIENT --query="SELECT * FROM null_as_default ORDER BY i";
$CLICKHOUSE_CLIENT --query="TRUNCATE TABLE null_as_default";
echo 'Template (Quoted)' echo 'Template (Quoted)'
echo 'NULL, '\''1'\'', null, '\''2019-07-22'\'', [10, 20, 30], NuLl echo 'NULL, '\''1'\'', null, '\''2019-07-22'\'', [10, 20, 30], NuLl
1, '\''world'\'', 3, '\''2019-07-23'\'', NULL, ('\''tuple'\'', 3.14) 1, '\''world'\'', 3, '\''2019-07-23'\'', NULL, ('\''tuple'\'', 3.14)

22
tests/queries/0_stateless/01446_json_strings_each_row.reference Normal file
View File

@ -0,0 +1,22 @@
1
{"value":"1","name":"a"}
{"value":"2","name":"b"}
{"value":"3","name":"c"}
2
{"name":"a","c":"1"}
{"name":"b","c":"1"}
{"name":"c","c":"1"}
3
{"row":{"a":"1"}}
{"progress":{"read_rows":"1","read_bytes":"1","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}
4
{"row":{"a":"1"}}
{"progress":{"read_rows":"1","read_bytes":"1","written_rows":"0","written_bytes":"0","total_rows_to_read":"0"}}
5
{"v1":"first","v2":"1","v3":"2","v4":"0"}
{"v1":"second","v2":"2","v3":"0","v4":"6"}
6
{"v1":"first","v2":"1","v3":"2","v4":"8"}
{"v1":"second","v2":"2","v3":"32","v4":"6"}
7
{"v1":"16","n.id":"[15,16,17]","n.name":"['first','second','third']"}

38
tests/queries/0_stateless/01446_json_strings_each_row.sql Normal file
View File

@ -0,0 +1,38 @@
DROP TABLE IF EXISTS test_table;
DROP TABLE IF EXISTS test_table_2;
SELECT 1;
/* Check JSONStringsEachRow Output */
CREATE TABLE test_table (value UInt8, name String) ENGINE = MergeTree() ORDER BY value;
INSERT INTO test_table VALUES (1, 'a'), (2, 'b'), (3, 'c');
SELECT * FROM test_table FORMAT JSONStringsEachRow;
SELECT 2;
/* Check Totals */
SELECT name, count() AS c FROM test_table GROUP BY name WITH TOTALS ORDER BY name FORMAT JSONStringsEachRow;
SELECT 3;
/* Check JSONStringsEachRowWithProgress Output */
SELECT 1 as a FROM system.one FORMAT JSONStringsEachRowWithProgress;
SELECT 4;
/* Check Totals */
SELECT 1 as a FROM system.one GROUP BY a WITH TOTALS ORDER BY a FORMAT JSONStringsEachRowWithProgress;
DROP TABLE IF EXISTS test_table;
SELECT 5;
/* Check JSONStringsEachRow Input */
CREATE TABLE test_table (v1 String, v2 UInt8, v3 DEFAULT v2 * 16, v4 UInt8 DEFAULT 8) ENGINE = MergeTree() ORDER BY v2;
INSERT INTO test_table FORMAT JSONStringsEachRow {"v1": "first", "v2": "1", "v3": "2", "v4": "NULL"} {"v1": "second", "v2": "2", "v3": "null", "v4": "6"};
SELECT * FROM test_table FORMAT JSONStringsEachRow;
TRUNCATE TABLE test_table;
SELECT 6;
/* Check input_format_null_as_default = 1 */
SET input_format_null_as_default = 1;
INSERT INTO test_table FORMAT JSONStringsEachRow {"v1": "first", "v2": "1", "v3": "2", "v4": "ᴺᵁᴸᴸ"} {"v1": "second", "v2": "2", "v3": "null", "v4": "6"};
SELECT * FROM test_table FORMAT JSONStringsEachRow;
TRUNCATE TABLE test_table;
SELECT 7;
/* Check Nested */
CREATE TABLE test_table_2 (v1 UInt8, n Nested(id UInt8, name String)) ENGINE = MergeTree() ORDER BY v1;
INSERT INTO test_table_2 FORMAT JSONStringsEachRow {"v1": "16", "n.id": "[15, 16, 17]", "n.name": "['first', 'second', 'third']"};
SELECT * FROM test_table_2 FORMAT JSONStringsEachRow;
TRUNCATE TABLE test_table_2;
DROP TABLE IF EXISTS test_table;
DROP TABLE IF EXISTS test_table_2;

43
tests/queries/0_stateless/01447_json_strings.reference Normal file
View File

@ -0,0 +1,43 @@
{
"meta":
[
{
"name": "1",
"type": "UInt8"
},
{
"name": "'a'",
"type": "String"
},
{
"name": "[1, 2, 3]",
"type": "Array(UInt8)"
},
{
"name": "tuple(1, 'a')",
"type": "Tuple(UInt8, String)"
},
{
"name": "NULL",
"type": "Nullable(Nothing)"
},
{
"name": "nan",
"type": "Float64"
}
],
"data":
[
{
"1": "1",
"'a'": "a",
"[1, 2, 3]": "[1,2,3]",
"tuple(1, 'a')": "(1,'a')",
"NULL": "ᴺᵁᴸᴸ",
"nan": "nan"
}
],
"rows": 1
}

10
tests/queries/0_stateless/01447_json_strings.sql Normal file
View File

@ -0,0 +1,10 @@
SET output_format_write_statistics = 0;
SELECT
1,
'a',
[1, 2, 3],
(1, 'a'),
null,
nan
FORMAT JSONStrings;

47
tests/queries/0_stateless/01448_json_compact_strings_each_row.reference Normal file
View File

@ -0,0 +1,47 @@
1
["1", "a"]
["2", "b"]
["3", "c"]
2
["a", "1"]
["b", "1"]
["c", "1"]
3
["value", "name"]
["UInt8", "String"]
["1", "a"]
["2", "b"]
["3", "c"]
4
["name", "c"]
["String", "UInt64"]
["a", "1"]
["b", "1"]
["c", "1"]
["", "3"]
5
["first", "1", "2", "0"]
["second", "2", "0", "6"]
6
["first", "1", "2", "8"]
["second", "2", "32", "6"]
7
["16", "[15,16,17]", "['first','second','third']"]
8
["first", "1", "2", "0"]
["second", "2", "0", "6"]
9
["first", "1", "2", "8"]
["second", "2", "32", "6"]
10
["first", "1", "16", "8"]
["second", "2", "32", "8"]
11
["v1", "v2", "v3", "v4"]
["String", "UInt8", "UInt16", "UInt8"]
["", "2", "3", "1"]
12
["v1", "n.id", "n.name"]
["UInt8", "Array(UInt8)", "Array(String)"]
["16", "[15,16,17]", "['first','second','third']"]

63
tests/queries/0_stateless/01448_json_compact_strings_each_row.sql Normal file
View File

@ -0,0 +1,63 @@
DROP TABLE IF EXISTS test_table;
DROP TABLE IF EXISTS test_table_2;
SELECT 1;
/* Check JSONCompactStringsEachRow Output */
CREATE TABLE test_table (value UInt8, name String) ENGINE = MergeTree() ORDER BY value;
INSERT INTO test_table VALUES (1, 'a'), (2, 'b'), (3, 'c');
SELECT * FROM test_table FORMAT JSONCompactStringsEachRow;
SELECT 2;
/* Check Totals */
SELECT name, count() AS c FROM test_table GROUP BY name WITH TOTALS ORDER BY name FORMAT JSONCompactStringsEachRow;
SELECT 3;
/* Check JSONCompactStringsEachRowWithNamesAndTypes Output */
SELECT * FROM test_table FORMAT JSONCompactStringsEachRowWithNamesAndTypes;
SELECT 4;
/* Check Totals */
SELECT name, count() AS c FROM test_table GROUP BY name WITH TOTALS ORDER BY name FORMAT JSONCompactStringsEachRowWithNamesAndTypes;
DROP TABLE IF EXISTS test_table;
SELECT 5;
/* Check JSONCompactStringsEachRow Input */
CREATE TABLE test_table (v1 String, v2 UInt8, v3 DEFAULT v2 * 16, v4 UInt8 DEFAULT 8) ENGINE = MergeTree() ORDER BY v2;
INSERT INTO test_table FORMAT JSONCompactStringsEachRow ["first", "1", "2", "NULL"] ["second", "2", "null", "6"];
SELECT * FROM test_table FORMAT JSONCompactStringsEachRow;
TRUNCATE TABLE test_table;
SELECT 6;
/* Check input_format_null_as_default = 1 */
SET input_format_null_as_default = 1;
INSERT INTO test_table FORMAT JSONCompactStringsEachRow ["first", "1", "2", "ᴺᵁᴸᴸ"] ["second", "2", "null", "6"];
SELECT * FROM test_table FORMAT JSONCompactStringsEachRow;
TRUNCATE TABLE test_table;
SELECT 7;
/* Check Nested */
CREATE TABLE test_table_2 (v1 UInt8, n Nested(id UInt8, name String)) ENGINE = MergeTree() ORDER BY v1;
INSERT INTO test_table_2 FORMAT JSONCompactStringsEachRow ["16", "[15, 16, 17]", "['first', 'second', 'third']"];
SELECT * FROM test_table_2 FORMAT JSONCompactStringsEachRow;
TRUNCATE TABLE test_table_2;
SELECT 8;
/* Check JSONCompactStringsEachRowWithNamesAndTypes Output */
SET input_format_null_as_default = 0;
INSERT INTO test_table FORMAT JSONCompactStringsEachRowWithNamesAndTypes ["v1", "v2", "v3", "v4"]["String","UInt8","UInt16","UInt8"]["first", "1", "2", "null"]["second", "2", "null", "6"];
SELECT * FROM test_table FORMAT JSONCompactStringsEachRow;
TRUNCATE TABLE test_table;
SELECT 9;
/* Check input_format_null_as_default = 1 */
SET input_format_null_as_default = 1;
INSERT INTO test_table FORMAT JSONCompactStringsEachRowWithNamesAndTypes ["v1", "v2", "v3", "v4"]["String","UInt8","UInt16","UInt8"]["first", "1", "2", "null"] ["second", "2", "null", "6"];
SELECT * FROM test_table FORMAT JSONCompactStringsEachRow;
SELECT 10;
/* Check Header */
TRUNCATE TABLE test_table;
SET input_format_skip_unknown_fields = 1;
INSERT INTO test_table FORMAT JSONCompactStringsEachRowWithNamesAndTypes ["v1", "v2", "invalid_column"]["String", "UInt8", "UInt8"]["first", "1", "32"]["second", "2", "64"];
SELECT * FROM test_table FORMAT JSONCompactStringsEachRow;
SELECT 11;
TRUNCATE TABLE test_table;
INSERT INTO test_table FORMAT JSONCompactStringsEachRowWithNamesAndTypes ["v4", "v2", "v3"]["UInt8", "UInt8", "UInt16"]["1", "2", "3"]
SELECT * FROM test_table FORMAT JSONCompactStringsEachRowWithNamesAndTypes;
SELECT 12;
/* Check Nested */
INSERT INTO test_table_2 FORMAT JSONCompactStringsEachRowWithNamesAndTypes ["v1", "n.id", "n.name"]["UInt8", "Array(UInt8)", "Array(String)"]["16", "[15, 16, 17]", "['first', 'second', 'third']"];
SELECT * FROM test_table_2 FORMAT JSONCompactStringsEachRowWithNamesAndTypes;
DROP TABLE IF EXISTS test_table;
DROP TABLE IF EXISTS test_table_2;

36
tests/queries/0_stateless/01449_json_compact_strings.reference Normal file
View File

@ -0,0 +1,36 @@
{
"meta":
[
{
"name": "1",
"type": "UInt8"
},
{
"name": "'a'",
"type": "String"
},
{
"name": "[1, 2, 3]",
"type": "Array(UInt8)"
},
{
"name": "tuple(1, 'a')",
"type": "Tuple(UInt8, String)"
},
{
"name": "NULL",
"type": "Nullable(Nothing)"
},
{
"name": "nan",
"type": "Float64"
}
],
"data":
[
["1", "a", "[1,2,3]", "(1,'a')", "ᴺᵁᴸᴸ", "nan"]
],
"rows": 1
}

10
tests/queries/0_stateless/01449_json_compact_strings.sql Normal file
View File

@ -0,0 +1,10 @@
SET output_format_write_statistics = 0;
SELECT
1,
'a',
[1, 2, 3],
(1, 'a'),
null,
nan
FORMAT JSONCompactStrings;

0
tests/queries/0_stateless/01492_array_join_crash_13829.reference Normal file
View File

1
tests/queries/0_stateless/01492_array_join_crash_13829.sql Normal file
View File

@ -0,0 +1 @@
SELECT NULL = countEqual(materialize([arrayJoin([NULL, NULL, NULL]), NULL AS x, arrayJoin([255, 1025, NULL, NULL]), arrayJoin([2, 1048576, NULL, NULL])]), materialize(x)) format Null;