diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 8892c6d8d3f..8fc66bef5b1 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1342,6 +1342,7 @@ SELECT * FROM json_each_row_nested - [input_format_json_ignore_unknown_keys_in_named_tuple](/docs/en/operations/settings/settings-formats.md/#input_format_json_ignore_unknown_keys_in_named_tuple) - ignore unknown keys in json object for named tuples. Default value - `false`. - [input_format_json_compact_allow_variable_number_of_columns](/docs/en/operations/settings/settings-formats.md/#input_format_json_compact_allow_variable_number_of_columns) - allow variable number of columns in JSONCompact/JSONCompactEachRow format, ignore extra columns and use default values on missing columns. Default value - `false`. - [input_format_json_throw_on_bad_escape_sequence](/docs/en/operations/settings/settings-formats.md/#input_format_json_throw_on_bad_escape_sequence) - throw an exception if JSON string contains bad escape sequence. If disabled, bad escape sequences will remain as is in the data. Default value - `true`. +- [input_format_json_empty_as_default](/docs/en/operations/settings/settings-formats.md/#input_format_json_empty_as_default) - treat empty fields in JSON input as default values. Default value - `false`. For complex default expressions [input_format_defaults_for_omitted_fields](/docs/en/operations/settings/settings-formats.md/#input_format_defaults_for_omitted_fields) must be enabled too. - [output_format_json_quote_64bit_integers](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_64bit_integers) - controls quoting of 64-bit integers in JSON output format. Default value - `true`. - [output_format_json_quote_64bit_floats](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_64bit_floats) - controls quoting of 64-bit floats in JSON output format. Default value - `false`. - [output_format_json_quote_denormals](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_denormals) - enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format. Default value - `false`. diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md index f8b40cd81ac..758d5c1ab49 100644 --- a/docs/en/operations/settings/settings-formats.md +++ b/docs/en/operations/settings/settings-formats.md @@ -741,6 +741,17 @@ Possible values: Default value: 0. +### input_format_json_empty_as_default {#input_format_json_empty_as_default} + +When enabled, replace empty input fields in JSON with default values. For complex default expressions `input_format_defaults_for_omitted_fields` must be enabled too. + +Possible values: + ++ 0 — Disable. ++ 1 — Enable. + +Default value: 0. + ## TSV format settings {#tsv-format-settings} ### input_format_tsv_empty_as_default {#input_format_tsv_empty_as_default} diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 0808e8eb49f..90045b9a8ff 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -1133,6 +1133,7 @@ class IColumn; M(Bool, input_format_json_throw_on_bad_escape_sequence, true, "Throw an exception if JSON string contains bad escape sequence in JSON input formats. If disabled, bad escape sequences will remain as is in the data", 0) \ M(Bool, input_format_json_ignore_unnecessary_fields, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields", 0) \ M(UInt64, input_format_json_max_depth, 1000, "Maximum depth of a field in JSON. This is not a strict limit, it does not have to be applied precisely.", 0) \ + M(Bool, input_format_json_empty_as_default, false, "Treat empty fields in JSON input as default values.", 0) \ M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \ M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \ M(Bool, input_format_try_infer_datetimes, true, "Try to infer datetimes from string fields while schema inference in text formats", 0) \ diff --git a/src/DataTypes/DataTypeAggregateFunction.h b/src/DataTypes/DataTypeAggregateFunction.h index e3a4f9726d9..2ab2e53100b 100644 --- a/src/DataTypes/DataTypeAggregateFunction.h +++ b/src/DataTypes/DataTypeAggregateFunction.h @@ -64,6 +64,8 @@ public: SerializationPtr doGetDefaultSerialization() const override; bool supportsSparseSerialization() const override { return false; } + bool isNonTriviallySerializedAsStringJSON() const override { return true; } + bool isVersioned() const; /// Version is not empty only if it was parsed from AST or implicitly cast to 0 or version according diff --git a/src/DataTypes/DataTypeDate.h b/src/DataTypes/DataTypeDate.h index 0e08b9ba2ca..cb7a603705d 100644 --- a/src/DataTypes/DataTypeDate.h +++ b/src/DataTypes/DataTypeDate.h @@ -17,6 +17,7 @@ public: bool canBeUsedAsVersion() const override { return true; } bool canBeInsideNullable() const override { return true; } + bool isNonTriviallySerializedAsStringJSON() const override { return true; } bool equals(const IDataType & rhs) const override; diff --git a/src/DataTypes/DataTypeDate32.h b/src/DataTypes/DataTypeDate32.h index 65633e7a228..53bd010b7cf 100644 --- a/src/DataTypes/DataTypeDate32.h +++ b/src/DataTypes/DataTypeDate32.h @@ -18,6 +18,7 @@ public: bool canBeUsedAsVersion() const override { return true; } bool canBeInsideNullable() const override { return true; } + bool isNonTriviallySerializedAsStringJSON() const override { return true; } bool equals(const IDataType & rhs) const override; diff --git a/src/DataTypes/DataTypeDateTime.h b/src/DataTypes/DataTypeDateTime.h index 5519240dee1..11b579920ba 100644 --- a/src/DataTypes/DataTypeDateTime.h +++ b/src/DataTypes/DataTypeDateTime.h @@ -44,6 +44,7 @@ public: bool canBeUsedAsVersion() const override { return true; } bool canBeInsideNullable() const override { return true; } + bool isNonTriviallySerializedAsStringJSON() const override { return true; } bool equals(const IDataType & rhs) const override; diff --git a/src/DataTypes/DataTypeDateTime64.h b/src/DataTypes/DataTypeDateTime64.h index 64cedd798d1..dd5ff7e6550 100644 --- a/src/DataTypes/DataTypeDateTime64.h +++ b/src/DataTypes/DataTypeDateTime64.h @@ -39,6 +39,8 @@ public: bool isSummable() const override { return false; } + bool isNonTriviallySerializedAsStringJSON() const override { return true; } + protected: SerializationPtr doGetDefaultSerialization() const override; }; diff --git a/src/DataTypes/DataTypeIPv4andIPv6.h b/src/DataTypes/DataTypeIPv4andIPv6.h index 5aea55751a7..520af4f21e0 100644 --- a/src/DataTypes/DataTypeIPv4andIPv6.h +++ b/src/DataTypes/DataTypeIPv4andIPv6.h @@ -46,6 +46,7 @@ public: size_t getSizeOfValueInMemory() const override { return sizeof(IPv4); } bool isCategorial() const override { return true; } bool canBeInsideLowCardinality() const override { return true; } + bool isNonTriviallySerializedAsStringJSON() const override { return true; } SerializationPtr doGetDefaultSerialization() const override { return std::make_shared>(); } }; @@ -84,6 +85,7 @@ public: size_t getSizeOfValueInMemory() const override { return sizeof(IPv6); } bool isCategorial() const override { return true; } bool canBeInsideLowCardinality() const override { return true; } + bool isNonTriviallySerializedAsStringJSON() const override { return true; } SerializationPtr doGetDefaultSerialization() const override { return std::make_shared>(); } }; diff --git a/src/DataTypes/DataTypeUUID.h b/src/DataTypes/DataTypeUUID.h index 90cdd90d68d..1aeab1b78ba 100644 --- a/src/DataTypes/DataTypeUUID.h +++ b/src/DataTypes/DataTypeUUID.h @@ -42,6 +42,7 @@ public: size_t getSizeOfValueInMemory() const override { return sizeof(UUID); } bool isCategorial() const override { return true; } bool canBeInsideLowCardinality() const override { return true; } + bool isNonTriviallySerializedAsStringJSON() const override { return true; } SerializationPtr doGetDefaultSerialization() const override; }; diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h index 397ae3d8be9..8033e82d1bc 100644 --- a/src/DataTypes/IDataType.h +++ b/src/DataTypes/IDataType.h @@ -328,6 +328,9 @@ public: /// Updates avg_value_size_hint for newly read column. Uses to optimize deserialization. Zero expected for first column. static void updateAvgValueSizeHint(const IColumn & column, double & avg_value_size_hint); + /// non-numeric non-string data type serialized as JSON string + virtual bool isNonTriviallySerializedAsStringJSON() const { return false; } + protected: friend class DataTypeFactory; friend class AggregateFunctionSimpleState; diff --git a/src/DataTypes/Serializations/ISerialization.h b/src/DataTypes/Serializations/ISerialization.h index 5d0bf60c59f..76c88fe7522 100644 --- a/src/DataTypes/Serializations/ISerialization.h +++ b/src/DataTypes/Serializations/ISerialization.h @@ -398,12 +398,20 @@ public: virtual void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const = 0; virtual void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const = 0; virtual bool tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const; + /// The following two methods are implemented only for non-numeric non-string simple data types. + virtual void deserializeTextNoEmptyCheckJSON(IColumn & /*column*/, ReadBuffer & /*istr*/, const FormatSettings &) const + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method deserializeTextNoEmptyCheckJSON is not supported"); + } + virtual bool tryDeserializeTextNoEmptyCheckJSON(IColumn & /*column*/, ReadBuffer & /*istr*/, const FormatSettings &) const + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method tryDeserializeTextNoEmptyCheckJSON is not supported"); + } virtual void serializeTextJSONPretty(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings, size_t /*indent*/) const { serializeTextJSON(column, row_num, ostr, settings); } - /** Text serialization for putting into the XML format. */ virtual void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const diff --git a/src/DataTypes/Serializations/SerializationAggregateFunction.cpp b/src/DataTypes/Serializations/SerializationAggregateFunction.cpp index 41b198890e4..39c3f389619 100644 --- a/src/DataTypes/Serializations/SerializationAggregateFunction.cpp +++ b/src/DataTypes/Serializations/SerializationAggregateFunction.cpp @@ -182,7 +182,7 @@ void SerializationAggregateFunction::serializeTextJSON(const IColumn & column, s } -void SerializationAggregateFunction::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const +void SerializationAggregateFunction::deserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { String s; readJSONString(s, istr, settings.json); diff --git a/src/DataTypes/Serializations/SerializationAggregateFunction.h b/src/DataTypes/Serializations/SerializationAggregateFunction.h index c45fc79f714..6afa1cd4e97 100644 --- a/src/DataTypes/Serializations/SerializationAggregateFunction.h +++ b/src/DataTypes/Serializations/SerializationAggregateFunction.h @@ -3,12 +3,13 @@ #include #include +#include namespace DB { -class SerializationAggregateFunction final : public ISerialization +class SerializationAggregateFunction final : public SerializationAsStringNonTrivialJSON { private: AggregateFunctionPtr function; @@ -37,7 +38,7 @@ public: void deserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; - void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; + void deserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; void serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; diff --git a/src/DataTypes/Serializations/SerializationAsStringNonTrivialJSON.h b/src/DataTypes/Serializations/SerializationAsStringNonTrivialJSON.h new file mode 100644 index 00000000000..7d8375368a7 --- /dev/null +++ b/src/DataTypes/Serializations/SerializationAsStringNonTrivialJSON.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + +namespace DB +{ + +/** Serialization for non-numeric non-string data types serialized as JSON strings + * For these data types, we support an option, input_format_json_empty_as_default, which, when set to 1, + * allows for JSON deserialization to treat an encountered empty string as a default value for the specified type. + * Derived classes must implement the following methods: + * deserializeTextNoEmptyCheckJSON() and tryDeserializeTextNoEmptyCheckJSON() + * instead of deserializeTextJSON() and tryDeserializeTextJSON() respectively. + */ +template +requires std::derived_from +class SerializationAsStringNonTrivialJSON : public T +{ +public: + using T::T; + + void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & format_settings) const override + { + if (format_settings.json.empty_as_default && tryMatchEmptyString(istr)) + column.insertDefault(); + else + deserializeTextNoEmptyCheckJSON(column, istr, format_settings); + } + + bool tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & format_settings) const override + { + if (format_settings.json.empty_as_default && tryMatchEmptyString(istr)) + { + column.insertDefault(); + return true; + } + else + return tryDeserializeTextNoEmptyCheckJSON(column, istr, format_settings); + } + + virtual void deserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override = 0; + + virtual bool tryDeserializeTextNoEmptyCheckJSON(IColumn & /*column*/, ReadBuffer & /*istr*/, const FormatSettings &) const override + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method tryDeserializeTextNoEmptyCheckJSON is not supported"); + } +}; + +} diff --git a/src/DataTypes/Serializations/SerializationDate.cpp b/src/DataTypes/Serializations/SerializationDate.cpp index 38e1bb87b6d..3f122189c22 100644 --- a/src/DataTypes/Serializations/SerializationDate.cpp +++ b/src/DataTypes/Serializations/SerializationDate.cpp @@ -85,7 +85,7 @@ void SerializationDate::serializeTextJSON(const IColumn & column, size_t row_num writeChar('"', ostr); } -void SerializationDate::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void SerializationDate::deserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const { DayNum x; assertChar('"', istr); @@ -94,7 +94,7 @@ void SerializationDate::deserializeTextJSON(IColumn & column, ReadBuffer & istr, assert_cast(column).getData().push_back(x); } -bool SerializationDate::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationDate::tryDeserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const { DayNum x; if (!checkChar('"', istr) || !tryReadDateText(x, istr, time_zone) || !checkChar('"', istr)) diff --git a/src/DataTypes/Serializations/SerializationDate.h b/src/DataTypes/Serializations/SerializationDate.h index dcf79eb49da..10c83171527 100644 --- a/src/DataTypes/Serializations/SerializationDate.h +++ b/src/DataTypes/Serializations/SerializationDate.h @@ -1,12 +1,13 @@ #pragma once #include +#include #include namespace DB { -class SerializationDate final : public SerializationNumber +class SerializationDate final : public SerializationAsStringNonTrivialJSON> { public: explicit SerializationDate(const DateLUTImpl & time_zone_ = DateLUT::instance()); @@ -21,8 +22,8 @@ public: void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; bool tryDeserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; - void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; - bool tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; + void deserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; + bool tryDeserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; bool tryDeserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; diff --git a/src/DataTypes/Serializations/SerializationDate32.cpp b/src/DataTypes/Serializations/SerializationDate32.cpp index 70a22d59e42..8ad07b534ce 100644 --- a/src/DataTypes/Serializations/SerializationDate32.cpp +++ b/src/DataTypes/Serializations/SerializationDate32.cpp @@ -83,7 +83,7 @@ void SerializationDate32::serializeTextJSON(const IColumn & column, size_t row_n writeChar('"', ostr); } -void SerializationDate32::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void SerializationDate32::deserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const { ExtendedDayNum x; assertChar('"', istr); @@ -92,7 +92,7 @@ void SerializationDate32::deserializeTextJSON(IColumn & column, ReadBuffer & ist assert_cast(column).getData().push_back(x); } -bool SerializationDate32::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationDate32::tryDeserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const { ExtendedDayNum x; if (!checkChar('"', istr) || !tryReadDateText(x, istr, time_zone) || !checkChar('"', istr)) diff --git a/src/DataTypes/Serializations/SerializationDate32.h b/src/DataTypes/Serializations/SerializationDate32.h index be2e2b76c1d..ac6239fbc2b 100644 --- a/src/DataTypes/Serializations/SerializationDate32.h +++ b/src/DataTypes/Serializations/SerializationDate32.h @@ -1,11 +1,12 @@ #pragma once #include +#include #include namespace DB { -class SerializationDate32 final : public SerializationNumber +class SerializationDate32 final : public SerializationAsStringNonTrivialJSON> { public: explicit SerializationDate32(const DateLUTImpl & time_zone_ = DateLUT::instance()); @@ -20,8 +21,8 @@ public: void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; bool tryDeserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; - void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; - bool tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; + void deserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; + bool tryDeserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; bool tryDeserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; diff --git a/src/DataTypes/Serializations/SerializationDateTime.cpp b/src/DataTypes/Serializations/SerializationDateTime.cpp index c5c819ce7fa..80b2d51140d 100644 --- a/src/DataTypes/Serializations/SerializationDateTime.cpp +++ b/src/DataTypes/Serializations/SerializationDateTime.cpp @@ -180,7 +180,7 @@ void SerializationDateTime::serializeTextJSON(const IColumn & column, size_t row writeChar('"', ostr); } -void SerializationDateTime::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const +void SerializationDateTime::deserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { time_t x = 0; if (checkChar('"', istr)) @@ -196,7 +196,7 @@ void SerializationDateTime::deserializeTextJSON(IColumn & column, ReadBuffer & i assert_cast(column).getData().push_back(static_cast(x)); } -bool SerializationDateTime::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const +bool SerializationDateTime::tryDeserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { time_t x = 0; if (checkChar('"', istr)) diff --git a/src/DataTypes/Serializations/SerializationDateTime.h b/src/DataTypes/Serializations/SerializationDateTime.h index 584b0c4116b..0041f221ccf 100644 --- a/src/DataTypes/Serializations/SerializationDateTime.h +++ b/src/DataTypes/Serializations/SerializationDateTime.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include class DateLUTImpl; @@ -8,7 +9,7 @@ class DateLUTImpl; namespace DB { -class SerializationDateTime final : public SerializationNumber, public TimezoneMixin +class SerializationDateTime final : public SerializationAsStringNonTrivialJSON>, public TimezoneMixin { public: explicit SerializationDateTime(const TimezoneMixin & time_zone_); @@ -23,8 +24,8 @@ public: void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; bool tryDeserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; - void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; - bool tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; + void deserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; + bool tryDeserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; bool tryDeserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; diff --git a/src/DataTypes/Serializations/SerializationDateTime64.cpp b/src/DataTypes/Serializations/SerializationDateTime64.cpp index 442e29edd52..cdac7f785d1 100644 --- a/src/DataTypes/Serializations/SerializationDateTime64.cpp +++ b/src/DataTypes/Serializations/SerializationDateTime64.cpp @@ -15,7 +15,7 @@ namespace DB SerializationDateTime64::SerializationDateTime64( UInt32 scale_, const TimezoneMixin & time_zone_) - : SerializationDecimalBase(DecimalUtils::max_precision, scale_) + : SerializationAsStringNonTrivialJSON>(DecimalUtils::max_precision, scale_) , TimezoneMixin(time_zone_) { } @@ -170,7 +170,7 @@ void SerializationDateTime64::serializeTextJSON(const IColumn & column, size_t r writeChar('"', ostr); } -void SerializationDateTime64::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const +void SerializationDateTime64::deserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { DateTime64 x = 0; if (checkChar('"', istr)) @@ -185,7 +185,7 @@ void SerializationDateTime64::deserializeTextJSON(IColumn & column, ReadBuffer & assert_cast(column).getData().push_back(x); } -bool SerializationDateTime64::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const +bool SerializationDateTime64::tryDeserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { DateTime64 x = 0; if (checkChar('"', istr)) diff --git a/src/DataTypes/Serializations/SerializationDateTime64.h b/src/DataTypes/Serializations/SerializationDateTime64.h index b49bd1e9098..a3bc4d1ad4e 100644 --- a/src/DataTypes/Serializations/SerializationDateTime64.h +++ b/src/DataTypes/Serializations/SerializationDateTime64.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include class DateLUTImpl; @@ -8,7 +9,7 @@ class DateLUTImpl; namespace DB { -class SerializationDateTime64 final : public SerializationDecimalBase, public TimezoneMixin +class SerializationDateTime64 final : public SerializationAsStringNonTrivialJSON>, public TimezoneMixin { public: SerializationDateTime64(UInt32 scale_, const TimezoneMixin & time_zone_); @@ -25,8 +26,8 @@ public: void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; bool tryDeserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; - void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; - bool tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; + void deserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; + bool tryDeserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; bool tryDeserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; diff --git a/src/DataTypes/Serializations/SerializationIPv4andIPv6.cpp b/src/DataTypes/Serializations/SerializationIPv4andIPv6.cpp index c1beceb4533..ecd50a0b9b8 100644 --- a/src/DataTypes/Serializations/SerializationIPv4andIPv6.cpp +++ b/src/DataTypes/Serializations/SerializationIPv4andIPv6.cpp @@ -69,7 +69,7 @@ void SerializationIP::serializeTextJSON(const DB::IColumn & column, size_t } template -void SerializationIP::deserializeTextJSON(DB::IColumn & column, DB::ReadBuffer & istr, const DB::FormatSettings & settings) const +void SerializationIP::deserializeTextNoEmptyCheckJSON(DB::IColumn & column, DB::ReadBuffer & istr, const DB::FormatSettings & settings) const { IPv x; assertChar('"', istr); @@ -84,7 +84,7 @@ void SerializationIP::deserializeTextJSON(DB::IColumn & column, DB::ReadBuf } template -bool SerializationIP::tryDeserializeTextJSON(DB::IColumn & column, DB::ReadBuffer & istr, const DB::FormatSettings &) const +bool SerializationIP::tryDeserializeTextNoEmptyCheckJSON(DB::IColumn & column, DB::ReadBuffer & istr, const DB::FormatSettings &) const { IPv x; if (!checkChar('"', istr) || !tryReadText(x, istr) || !checkChar('"', istr)) diff --git a/src/DataTypes/Serializations/SerializationIPv4andIPv6.h b/src/DataTypes/Serializations/SerializationIPv4andIPv6.h index a53f257646b..44f36252741 100644 --- a/src/DataTypes/Serializations/SerializationIPv4andIPv6.h +++ b/src/DataTypes/Serializations/SerializationIPv4andIPv6.h @@ -4,13 +4,14 @@ #include #include #include +#include #include namespace DB { template -class SerializationIP : public SimpleTextSerialization +class SerializationIP : public SerializationAsStringNonTrivialJSON { public: void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; @@ -22,8 +23,8 @@ public: bool tryDeserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override; - void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; - bool tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; + void deserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; + bool tryDeserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const override; void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &/* settings*/) const override; diff --git a/src/DataTypes/Serializations/SerializationNullable.cpp b/src/DataTypes/Serializations/SerializationNullable.cpp index e72dd3a42f5..72c6a661dde 100644 --- a/src/DataTypes/Serializations/SerializationNullable.cpp +++ b/src/DataTypes/Serializations/SerializationNullable.cpp @@ -844,25 +844,52 @@ bool SerializationNullable::tryDeserializeNullJSON(DB::ReadBuffer & istr) return checkString("null", istr); } -template -ReturnType deserializeTextJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const SerializationPtr & nested, bool & is_null) +namespace +{ + +enum class Strategy : uint8_t +{ + Deserialize, + DeserializeNoEmptyCheck, + TryDeserialize +}; + +template struct ReturnTypeImpl; +template <> struct ReturnTypeImpl { using Type = void; }; +template <> struct ReturnTypeImpl { using Type = bool; }; +template <> struct ReturnTypeImpl { using Type = void; }; + +template +using ReturnType = typename ReturnTypeImpl::Type; + +template struct AlwaysFalse : std::false_type {}; + +template +ReturnType deserializeTextJSONImpl(IColumn & column, ReadBuffer & istr, const FormatSettings & settings, const SerializationPtr & nested, bool & is_null) { auto check_for_null = [](ReadBuffer & buf){ return checkStringByFirstCharacterAndAssertTheRest("null", buf); }; auto deserialize_nested = [&nested, &settings](IColumn & nested_column, ReadBuffer & buf) { - if constexpr (std::is_same_v) + if constexpr (strategy == Strategy::TryDeserialize) return nested->tryDeserializeTextJSON(nested_column, buf, settings); - nested->deserializeTextJSON(nested_column, buf, settings); + else if constexpr (strategy == Strategy::Deserialize) + nested->deserializeTextJSON(nested_column, buf, settings); + else if constexpr (strategy == Strategy::DeserializeNoEmptyCheck) + nested->deserializeTextNoEmptyCheckJSON(nested_column, buf, settings); + else + static_assert(AlwaysFalse::value); }; - return deserializeImpl(column, istr, check_for_null, deserialize_nested, is_null); + return deserializeImpl>(column, istr, check_for_null, deserialize_nested, is_null); +} + } void SerializationNullable::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { ColumnNullable & col = assert_cast(column); bool is_null; - deserializeTextJSONImpl(col.getNestedColumn(), istr, settings, nested, is_null); + deserializeTextJSONImpl(col.getNestedColumn(), istr, settings, nested, is_null); safeAppendToNullMap(col, is_null); } @@ -870,20 +897,27 @@ bool SerializationNullable::tryDeserializeTextJSON(IColumn & column, ReadBuffer { ColumnNullable & col = assert_cast(column); bool is_null; - return deserializeTextJSONImpl(col.getNestedColumn(), istr, settings, nested, is_null) && safeAppendToNullMap(col, is_null); + return deserializeTextJSONImpl(col.getNestedColumn(), istr, settings, nested, is_null) && safeAppendToNullMap(col, is_null); } bool SerializationNullable::deserializeNullAsDefaultOrNestedTextJSON(DB::IColumn & nested_column, DB::ReadBuffer & istr, const DB::FormatSettings & settings, const DB::SerializationPtr & nested_serialization) { bool is_null; - deserializeTextJSONImpl(nested_column, istr, settings, nested_serialization, is_null); + deserializeTextJSONImpl(nested_column, istr, settings, nested_serialization, is_null); + return !is_null; +} + +bool SerializationNullable::deserializeNullAsDefaultOrNestedTextNoEmptyCheckJSON(DB::IColumn & nested_column, DB::ReadBuffer & istr, const DB::FormatSettings & settings, const DB::SerializationPtr & nested_serialization) +{ + bool is_null; + deserializeTextJSONImpl(nested_column, istr, settings, nested_serialization, is_null); return !is_null; } bool SerializationNullable::tryDeserializeNullAsDefaultOrNestedTextJSON(DB::IColumn & nested_column, DB::ReadBuffer & istr, const DB::FormatSettings & settings, const DB::SerializationPtr & nested_serialization) { bool is_null; - return deserializeTextJSONImpl(nested_column, istr, settings, nested_serialization, is_null); + return deserializeTextJSONImpl(nested_column, istr, settings, nested_serialization, is_null); } void SerializationNullable::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const diff --git a/src/DataTypes/Serializations/SerializationNullable.h b/src/DataTypes/Serializations/SerializationNullable.h index f7d2d2eadf0..c5215e2a39f 100644 --- a/src/DataTypes/Serializations/SerializationNullable.h +++ b/src/DataTypes/Serializations/SerializationNullable.h @@ -88,6 +88,7 @@ public: static bool deserializeNullAsDefaultOrNestedTextQuoted(IColumn & nested_column, ReadBuffer & istr, const FormatSettings &, const SerializationPtr & nested_serialization); static bool deserializeNullAsDefaultOrNestedTextCSV(IColumn & nested_column, ReadBuffer & istr, const FormatSettings & settings, const SerializationPtr & nested_serialization); static bool deserializeNullAsDefaultOrNestedTextJSON(IColumn & nested_column, ReadBuffer & istr, const FormatSettings &, const SerializationPtr & nested_serialization); + static bool deserializeNullAsDefaultOrNestedTextNoEmptyCheckJSON(IColumn & nested_column, ReadBuffer & istr, const FormatSettings &, const SerializationPtr & nested_serialization); static bool deserializeNullAsDefaultOrNestedTextRaw(IColumn & nested_column, ReadBuffer & istr, const FormatSettings & settings, const SerializationPtr & nested_serialization); /// If Check for NULL and deserialize value into non-nullable column or insert default value of nested type. diff --git a/src/DataTypes/Serializations/SerializationUUID.cpp b/src/DataTypes/Serializations/SerializationUUID.cpp index f18466ad8ad..21a0ccf676c 100644 --- a/src/DataTypes/Serializations/SerializationUUID.cpp +++ b/src/DataTypes/Serializations/SerializationUUID.cpp @@ -94,7 +94,7 @@ void SerializationUUID::serializeTextJSON(const IColumn & column, size_t row_num writeChar('"', ostr); } -void SerializationUUID::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void SerializationUUID::deserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const { UUID x; assertChar('"', istr); @@ -103,7 +103,7 @@ void SerializationUUID::deserializeTextJSON(IColumn & column, ReadBuffer & istr, assert_cast(column).getData().push_back(x); } -bool SerializationUUID::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationUUID::tryDeserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const { UUID x; if (!checkChar('"', istr) || !tryReadText(x, istr) || !checkChar('"', istr)) diff --git a/src/DataTypes/Serializations/SerializationUUID.h b/src/DataTypes/Serializations/SerializationUUID.h index 458504f8f42..185d1d44c16 100644 --- a/src/DataTypes/Serializations/SerializationUUID.h +++ b/src/DataTypes/Serializations/SerializationUUID.h @@ -1,11 +1,12 @@ #pragma once #include +#include namespace DB { -class SerializationUUID : public SimpleTextSerialization +class SerializationUUID : public SerializationAsStringNonTrivialJSON { public: void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; @@ -15,8 +16,8 @@ public: void deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; bool tryDeserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; void serializeTextJSON(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; - void deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; - bool tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; + void deserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; + bool tryDeserializeTextNoEmptyCheckJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override; void serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override; void deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; bool tryDeserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override; diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index a78836ff63c..59131f34697 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -151,6 +151,8 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se format_settings.json.try_infer_objects_as_tuples = settings.input_format_json_try_infer_named_tuples_from_objects; format_settings.json.throw_on_bad_escape_sequence = settings.input_format_json_throw_on_bad_escape_sequence; format_settings.json.ignore_unnecessary_fields = settings.input_format_json_ignore_unnecessary_fields; + format_settings.json.case_insensitive_column_matching = settings.input_format_json_case_insensitive_column_matching; + format_settings.json.empty_as_default = settings.input_format_json_empty_as_default; format_settings.null_as_default = settings.input_format_null_as_default; format_settings.force_null_for_omitted_fields = settings.input_format_force_null_for_omitted_fields; format_settings.decimal_trailing_zeros = settings.output_format_decimal_trailing_zeros; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index f0359218775..8c247bb960c 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -234,6 +234,8 @@ struct FormatSettings bool infer_incomplete_types_as_strings = true; bool throw_on_bad_escape_sequence = true; bool ignore_unnecessary_fields = true; + bool case_insensitive_column_matching = false; + bool empty_as_default = false; } json{}; struct diff --git a/src/Formats/JSONUtils.cpp b/src/Formats/JSONUtils.cpp index 017befe5b0e..73189e81f97 100644 --- a/src/Formats/JSONUtils.cpp +++ b/src/Formats/JSONUtils.cpp @@ -286,11 +286,33 @@ namespace JSONUtils return true; } - if (as_nullable) - return SerializationNullable::deserializeNullAsDefaultOrNestedTextJSON(column, in, format_settings, serialization); + if (format_settings.json.empty_as_default && type->isNonTriviallySerializedAsStringJSON()) + { + /// We have a non-numeric non-string data type at the top level. + /// At first glance, it looks like we sort of duplicate the work done in + /// SerializationAsStringNonTrivialJSON. Actually we need to proceed as + /// done here because we want to return false if we inserted a default + /// value on purpose, which the ISerialization interface does not allow for. + if (tryMatchEmptyString(in)) + { + column.insertDefault(); + return false; + } - serialization->deserializeTextJSON(column, in, format_settings); - return true; + if (as_nullable) + return SerializationNullable::deserializeNullAsDefaultOrNestedTextNoEmptyCheckJSON(column, in, format_settings, serialization); + + serialization->deserializeTextNoEmptyCheckJSON(column, in, format_settings); + return true; + } + else + { + if (as_nullable) + return SerializationNullable::deserializeNullAsDefaultOrNestedTextJSON(column, in, format_settings, serialization); + + serialization->deserializeTextJSON(column, in, format_settings); + return true; + } } catch (Exception & e) { diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index 9559462e62b..e8ef667a4e3 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -1137,6 +1137,23 @@ template void readCSVStringInto(String & s, ReadBuffer & b template void readCSVStringInto(String & s, ReadBuffer & buf, const FormatSettings::CSV & settings); template void readCSVStringInto, false, false>(PaddedPODArray & s, ReadBuffer & buf, const FormatSettings::CSV & settings); +bool tryMatchEmptyString(ReadBuffer & buf) +{ + if (buf.eof() || *buf.position() != '"') + return false; + + ++buf.position(); + + if (buf.eof() || *buf.position() != '"') + { + --buf.position(); + return false; + } + + ++buf.position(); + + return true; +} template ReturnType readJSONStringInto(Vector & s, ReadBuffer & buf, const FormatSettings::JSON & settings) diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index ffba4fafb5c..59c1923e02a 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -665,6 +665,10 @@ void readStringUntilEOFInto(Vector & s, ReadBuffer & buf); template void readCSVStringInto(Vector & s, ReadBuffer & buf, const FormatSettings::CSV & settings); +/// Consumes the current token if it is an empty string, i.e. two consecutive double quotes, +/// Returns true if consumed. +bool tryMatchEmptyString(ReadBuffer & buf); + /// ReturnType is either bool or void. If bool, the function will return false instead of throwing an exception. template ReturnType readJSONStringInto(Vector & s, ReadBuffer & buf, const FormatSettings::JSON & settings); diff --git a/tests/queries/0_stateless/03203_json_empty_as_default.reference b/tests/queries/0_stateless/03203_json_empty_as_default.reference new file mode 100644 index 00000000000..53d7014e452 --- /dev/null +++ b/tests/queries/0_stateless/03203_json_empty_as_default.reference @@ -0,0 +1,20 @@ +1970-01-01 +1970-01-01 +1970-01-01 00:00:00 +1970-01-01 00:00:00.000 +0.0.0.0 +:: +00000000-0000-0000-0000-000000000000 +1 +:: +2001:db8:3333:4444:5555:6666:7777:8888 +:: +['00000000-0000-0000-0000-000000000000','b15f852c-c41a-4fd6-9247-1929c841715e','00000000-0000-0000-0000-000000000000'] +['::','::'] +('1970-01-01','0.0.0.0','abc') +{'abc':'::'} +00000000-0000-0000-0000-000000000000 +[['2001:db8:3333:4444:cccc:dddd:eeee:ffff','::'],['::','2001:db8:3333:4444:5555:6666:7777:8888']] +['00000000-0000-0000-0000-000000000000','b15f852c-c41a-4fd6-9247-1929c841715e'] +(['00000000-0000-0000-0000-000000000000'],('00000000-0000-0000-0000-000000000000',{'abc':'::'})) +{('1970-01-01','0.0.0.0'):'00000000-0000-0000-0000-000000000000'} diff --git a/tests/queries/0_stateless/03203_json_empty_as_default.sql b/tests/queries/0_stateless/03203_json_empty_as_default.sql new file mode 100644 index 00000000000..35652f4c751 --- /dev/null +++ b/tests/queries/0_stateless/03203_json_empty_as_default.sql @@ -0,0 +1,53 @@ +set input_format_json_empty_as_default = 1; +set allow_experimental_variant_type = 1; + +# Simple types +SELECT x FROM format(JSONEachRow, 'x Date', '{"x":""}'); +SELECT x FROM format(JSONEachRow, 'x Date32', '{"x":""}'); +SELECT toTimeZone(x, 'UTC') FROM format(JSONEachRow, 'x DateTime', '{"x":""}'); +SELECT toTimeZone(x, 'UTC') FROM format(JSONEachRow, 'x DateTime64', '{"x":""}'); +SELECT x FROM format(JSONEachRow, 'x IPv4', '{"x":""}'); +SELECT x FROM format(JSONEachRow, 'x IPv6', '{"x":""}'); +SELECT x FROM format(JSONEachRow, 'x UUID', '{"x":""}'); + +# Simple type AggregateFunction +DROP TABLE IF EXISTS table1; +CREATE TABLE table1(col AggregateFunction(uniq, UInt64)) ENGINE=Memory(); +DROP TABLE IF EXISTS table2; +CREATE TABLE table2(UserID UInt64) ENGINE=Memory(); + +INSERT INTO table1 SELECT uniqState(UserID) FROM table2; +INSERT INTO table1 SELECT x FROM format(JSONEachRow, 'x AggregateFunction(uniq, UInt64)' AS T, '{"x":""}'); +SELECT COUNT(DISTINCT col) FROM table1; + +DROP TABLE table1; +DROP TABLE table2; + +# The setting input_format_defaults_for_omitted_fields determines the default value if enabled. +CREATE TABLE table1(address IPv6 DEFAULT toIPv6('2001:db8:3333:4444:5555:6666:7777:8888')) ENGINE=Memory(); + +set input_format_defaults_for_omitted_fields = 0; +INSERT INTO table1 FORMAT JSONEachRow {"address":""}; + +set input_format_defaults_for_omitted_fields = 1; +INSERT INTO table1 FORMAT JSONEachRow {"address":""}; + +SELECT * FROM table1 ORDER BY address ASC; + +DROP TABLE table1; + +# Nullable +SELECT x FROM format(JSONEachRow, 'x Nullable(IPv6)', '{"x":""}'); + +# Compound types +SELECT x FROM format(JSONEachRow, 'x Array(UUID)', '{"x":["00000000-0000-0000-0000-000000000000","b15f852c-c41a-4fd6-9247-1929c841715e",""]}'); +SELECT x FROM format(JSONEachRow, 'x Array(Nullable(IPv6))', '{"x":["",""]}'); +SELECT x FROM format(JSONEachRow, 'x Tuple(Date, IPv4, String)', '{"x":["", "", "abc"]}'); +SELECT x FROM format(JSONEachRow, 'x Map(String, IPv6)', '{"x":{"abc": ""}}'); +SELECT x FROM format(JSONEachRow, 'x Variant(Date, UUID)', '{"x":""}'); + +# Deep composition +SELECT x FROM format(JSONEachRow, 'x Array(Array(IPv6))', '{"x":[["2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF", ""], ["", "2001:db8:3333:4444:5555:6666:7777:8888"]]}'); +SELECT x FROM format(JSONEachRow, 'x Variant(Date, Array(UUID))', '{"x":["", "b15f852c-c41a-4fd6-9247-1929c841715e"]}'); +SELECT x FROM format(JSONEachRow, 'x Tuple(Array(UUID), Tuple(UUID, Map(String, IPv6)))', '{"x":[[""], ["",{"abc":""}]]}'); +SELECT x FROM format(JSONEachRow, 'x Map(Tuple(Date,IPv4), Variant(UUID,IPv6))', '{"x":{["",""]:""}}');