From d8fce448a29eecff6e1dc77299f63c3e75f0fbbc Mon Sep 17 00:00:00 2001 From: hcz Date: Wed, 9 Sep 2020 14:20:14 +0800 Subject: [PATCH] Implement null_as_default for JSONStrings formats --- src/DataTypes/DataTypeNullable.cpp | 12 ++++++++++-- src/DataTypes/DataTypeNullable.h | 2 ++ .../Impl/JSONCompactEachRowRowInputFormat.cpp | 7 ++++--- .../Formats/Impl/JSONEachRowRowInputFormat.cpp | 7 ++++--- .../01016_input_null_as_default.reference | 5 +++++ .../0_stateless/01016_input_null_as_default.sh | 8 ++++++++ 6 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/DataTypes/DataTypeNullable.cpp b/src/DataTypes/DataTypeNullable.cpp index 3318196b951..6e452b2759b 100644 --- a/src/DataTypes/DataTypeNullable.cpp +++ b/src/DataTypes/DataTypeNullable.cpp @@ -318,13 +318,20 @@ ReturnType DataTypeNullable::deserializeTextQuoted(IColumn & column, ReadBuffer void DataTypeNullable::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - safeDeserialize(column, *nested_data_type, + deserializeWholeText(column, istr, settings, nested_data_type); +} + +template +ReturnType DataTypeNullable::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, + const DataTypePtr & nested_data_type) +{ + return safeDeserialize(column, *nested_data_type, [&istr] { return checkStringByFirstCharacterAndAssertTheRestCaseInsensitive("NULL", istr) || checkStringByFirstCharacterAndAssertTheRest("ᴺᵁᴸᴸ", istr); }, - [this, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeAsWholeText(nested, istr, settings); }); + [&nested_data_type, &istr, &settings] (IColumn & nested) { nested_data_type->deserializeAsWholeText(nested, istr, settings); }); } @@ -551,6 +558,7 @@ DataTypePtr removeNullable(const DataTypePtr & type) } +template bool DataTypeNullable::deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested); template bool DataTypeNullable::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested); template bool DataTypeNullable::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &, const DataTypePtr & nested); template bool DataTypeNullable::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested); diff --git a/src/DataTypes/DataTypeNullable.h b/src/DataTypes/DataTypeNullable.h index 22d403da6c4..587eecdf32e 100644 --- a/src/DataTypes/DataTypeNullable.h +++ b/src/DataTypes/DataTypeNullable.h @@ -103,6 +103,8 @@ public: /// If ReturnType is bool, check for NULL and deserialize value into non-nullable column (and return true) or insert default value of nested type (and return false) /// If ReturnType is void, deserialize Nullable(T) template + static ReturnType deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested); + template static ReturnType deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const DataTypePtr & nested); template static ReturnType deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &, const DataTypePtr & nested); diff --git a/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp index eb697ce5318..49c8d29ca2f 100644 --- a/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp @@ -205,14 +205,15 @@ void JSONCompactEachRowRowInputFormat::readField(size_t index, MutableColumns & if (yield_strings) { - // notice: null_as_default on "null" strings is not supported - String str; readJSONString(str, in); ReadBufferFromString buf(str); - type->deserializeAsWholeText(*columns[index], buf, format_settings); + if (format_settings.null_as_default && !type->isNullable()) + read_columns[index] = DataTypeNullable::deserializeWholeText(*columns[index], buf, format_settings, type); + else + type->deserializeAsWholeText(*columns[index], buf, format_settings); } else { diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp index 9ba82fbb009..ab775a3e7aa 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp @@ -146,14 +146,15 @@ void JSONEachRowRowInputFormat::readField(size_t index, MutableColumns & columns if (yield_strings) { - // notice: null_as_default on "null" strings is not supported - String str; readJSONString(str, in); ReadBufferFromString buf(str); - type->deserializeAsWholeText(*columns[index], buf, format_settings); + if (format_settings.null_as_default && !type->isNullable()) + read_columns[index] = DataTypeNullable::deserializeWholeText(*columns[index], buf, format_settings, type); + else + type->deserializeAsWholeText(*columns[index], buf, format_settings); } else { diff --git a/tests/queries/0_stateless/01016_input_null_as_default.reference b/tests/queries/0_stateless/01016_input_null_as_default.reference index ba9657bf16e..d7010f42d4e 100644 --- a/tests/queries/0_stateless/01016_input_null_as_default.reference +++ b/tests/queries/0_stateless/01016_input_null_as_default.reference @@ -18,6 +18,11 @@ JSONEachRow 1 world 3 2019-07-23 [1,2,3] ('tuple',3.14) 2 Hello 123 2019-06-19 [] ('test',2.71828) 3 Hello 42 2019-06-19 [1,2,3] ('default',0.75) +JSONStringsEachRow +0 1 42 2019-07-22 [10,20,30] ('default',0) +1 world 3 2019-07-23 [1,2,3] ('tuple',3.14) +2 Hello 123 2019-06-19 [] ('test',2.71828) +3 Hello 42 2019-06-19 [1,2,3] ('default',0.75) Template (Quoted) 0 1 42 2019-07-22 [10,20,30] ('default',0) 1 world 3 2019-07-23 [1,2,3] ('tuple',3.14) diff --git a/tests/queries/0_stateless/01016_input_null_as_default.sh b/tests/queries/0_stateless/01016_input_null_as_default.sh index a40287eaba8..f31e6591e97 100755 --- a/tests/queries/0_stateless/01016_input_null_as_default.sh +++ b/tests/queries/0_stateless/01016_input_null_as_default.sh @@ -38,6 +38,14 @@ echo '{"i": null, "s": "1", "n": null, "d": "2019-07-22", "a": [10, 20, 30], "t" $CLICKHOUSE_CLIENT --query="SELECT * FROM null_as_default ORDER BY i"; $CLICKHOUSE_CLIENT --query="TRUNCATE TABLE null_as_default"; +echo 'JSONStringsEachRow' +echo '{"i": "null", "s": "1", "n": "ᴺᵁᴸᴸ", "d": "2019-07-22", "a": "[10, 20, 30]", "t": "NULL"} +{"i": "1", "s": "world", "n": "3", "d": "2019-07-23", "a": "null", "t": "('\''tuple'\'', 3.14)"} +{"i": "2", "s": "null", "n": "123", "d": "null", "a": "[]", "t": "('\''test'\'', 2.71828)"} +{"i": "3", "s": "null", "n": "null", "d": "null", "a": "null", "t": "null"}' | $CLICKHOUSE_CLIENT --input_format_null_as_default=1 --query="INSERT INTO null_as_default FORMAT JSONStringsEachRow"; +$CLICKHOUSE_CLIENT --query="SELECT * FROM null_as_default ORDER BY i"; +$CLICKHOUSE_CLIENT --query="TRUNCATE TABLE null_as_default"; + echo 'Template (Quoted)' echo 'NULL, '\''1'\'', null, '\''2019-07-22'\'', [10, 20, 30], NuLl 1, '\''world'\'', 3, '\''2019-07-23'\'', NULL, ('\''tuple'\'', 3.14)