From a3522072d4b38817559541353f3b70625f92b918 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 2 Sep 2024 13:08:34 +0000 Subject: [PATCH] Respect format settings in Values format during conversion from expression to the destination type --- src/Interpreters/convertFieldToType.cpp | 30 +++++++++---------- src/Interpreters/convertFieldToType.h | 7 +++-- .../Formats/Impl/ValuesBlockInputFormat.cpp | 2 +- ...at_settings_in_fields_conversion.reference | 1 + ...t_format_settings_in_fields_conversion.sql | 7 +++++ 5 files changed, 28 insertions(+), 19 deletions(-) create mode 100644 tests/queries/0_stateless/03231_values_respect_format_settings_in_fields_conversion.reference create mode 100644 tests/queries/0_stateless/03231_values_respect_format_settings_in_fields_conversion.sql diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 7e1b4e2fb0e..737353095b8 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -164,7 +164,7 @@ Field convertDecimalType(const Field & from, const To & type) } -Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const IDataType * from_type_hint) +Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const IDataType * from_type_hint, const FormatSettings & format_settings) { if (from_type_hint && from_type_hint->equals(type)) { @@ -359,7 +359,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID Array res(src_arr_size); for (size_t i = 0; i < src_arr_size; ++i) { - res[i] = convertFieldToType(src_arr[i], element_type); + res[i] = convertFieldToType(src_arr[i], element_type, nullptr, format_settings); if (res[i].isNull() && !canContainNull(element_type)) { // See the comment for Tuples below. @@ -387,7 +387,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID for (size_t i = 0; i < dst_tuple_size; ++i) { const auto & element_type = *(type_tuple->getElements()[i]); - res[i] = convertFieldToType(src_tuple[i], element_type); + res[i] = convertFieldToType(src_tuple[i], element_type, nullptr, format_settings); if (res[i].isNull() && !canContainNull(element_type)) { /* @@ -435,12 +435,12 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID Tuple updated_entry(2); - updated_entry[0] = convertFieldToType(key, key_type); + updated_entry[0] = convertFieldToType(key, key_type, nullptr, format_settings); if (updated_entry[0].isNull() && !canContainNull(key_type)) have_unconvertible_element = true; - updated_entry[1] = convertFieldToType(value, value_type); + updated_entry[1] = convertFieldToType(value, value_type, nullptr, format_settings); if (updated_entry[1].isNull() && !canContainNull(value_type)) have_unconvertible_element = true; @@ -551,7 +551,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID ReadBufferFromString in_buffer(src.safeGet()); try { - type_to_parse->getDefaultSerialization()->deserializeWholeText(*col, in_buffer, FormatSettings{}); + type_to_parse->getDefaultSerialization()->deserializeWholeText(*col, in_buffer, format_settings); } catch (Exception & e) { @@ -563,7 +563,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID } Field parsed = (*col)[0]; - return convertFieldToType(parsed, type, from_type_hint); + return convertFieldToType(parsed, type, from_type_hint, format_settings); } throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch in IN or VALUES section. Expected: {}. Got: {}", @@ -573,7 +573,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID } -Field convertFieldToType(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint) +Field convertFieldToType(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint, const FormatSettings & format_settings) { if (from_value.isNull()) return from_value; @@ -582,7 +582,7 @@ Field convertFieldToType(const Field & from_value, const IDataType & to_type, co return from_value; if (const auto * low_cardinality_type = typeid_cast(&to_type)) - return convertFieldToType(from_value, *low_cardinality_type->getDictionaryType(), from_type_hint); + return convertFieldToType(from_value, *low_cardinality_type->getDictionaryType(), from_type_hint, format_settings); else if (const auto * nullable_type = typeid_cast(&to_type)) { const IDataType & nested_type = *nullable_type->getNestedType(); @@ -593,20 +593,20 @@ Field convertFieldToType(const Field & from_value, const IDataType & to_type, co if (from_type_hint && from_type_hint->equals(nested_type)) return from_value; - return convertFieldToTypeImpl(from_value, nested_type, from_type_hint); + return convertFieldToTypeImpl(from_value, nested_type, from_type_hint, format_settings); } else - return convertFieldToTypeImpl(from_value, to_type, from_type_hint); + return convertFieldToTypeImpl(from_value, to_type, from_type_hint, format_settings); } -Field convertFieldToTypeOrThrow(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint) +Field convertFieldToTypeOrThrow(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint, const FormatSettings & format_settings) { bool is_null = from_value.isNull(); if (is_null && !canContainNull(to_type)) throw Exception(ErrorCodes::TYPE_MISMATCH, "Cannot convert NULL to {}", to_type.getName()); - Field converted = convertFieldToType(from_value, to_type, from_type_hint); + Field converted = convertFieldToType(from_value, to_type, from_type_hint, format_settings); if (!is_null && converted.isNull()) throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, @@ -626,9 +626,9 @@ static bool decimalEqualsFloat(Field field, Float64 float_value) return decimal_to_float == float_value; } -std::optional convertFieldToTypeStrict(const Field & from_value, const IDataType & from_type, const IDataType & to_type) +std::optional convertFieldToTypeStrict(const Field & from_value, const IDataType & from_type, const IDataType & to_type, const FormatSettings & format_settings) { - Field result_value = convertFieldToType(from_value, to_type, &from_type); + Field result_value = convertFieldToType(from_value, to_type, &from_type, format_settings); if (Field::isDecimal(from_value.getType()) && Field::isDecimal(result_value.getType())) { diff --git a/src/Interpreters/convertFieldToType.h b/src/Interpreters/convertFieldToType.h index 4aa09f8619e..c3c6271a157 100644 --- a/src/Interpreters/convertFieldToType.h +++ b/src/Interpreters/convertFieldToType.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB @@ -15,13 +16,13 @@ class IDataType; * Checks for the compatibility of types, checks values fall in the range of valid values of the type, makes type conversion. * If the value does not fall into the range - returns Null. */ -Field convertFieldToType(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint = nullptr); +Field convertFieldToType(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint = nullptr, const FormatSettings & format_settings = {}); /// Does the same, but throws ARGUMENT_OUT_OF_BOUND if value does not fall into the range. -Field convertFieldToTypeOrThrow(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint = nullptr); +Field convertFieldToTypeOrThrow(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint = nullptr, const FormatSettings & format_settings = {}); /// Applies stricter rules than convertFieldToType, doesn't allow loss of precision converting to Decimal. /// Returns `Field` if the conversion was successful and the result is equal to the original value, otherwise returns nullopt. -std::optional convertFieldToTypeStrict(const Field & from_value, const IDataType & from_type, const IDataType & to_type); +std::optional convertFieldToTypeStrict(const Field & from_value, const IDataType & from_type, const IDataType & to_type, const FormatSettings & format_settings = {}); } diff --git a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp index 10d0e051665..16b88d0b8dc 100644 --- a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp @@ -542,7 +542,7 @@ bool ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx if (format_settings.null_as_default) tryToReplaceNullFieldsInComplexTypesWithDefaultValues(expression_value, type); - Field value = convertFieldToType(expression_value, type, value_raw.second.get()); + Field value = convertFieldToType(expression_value, type, value_raw.second.get(), format_settings); /// Check that we are indeed allowed to insert a NULL. if (value.isNull() && !type.isNullable() && !type.isLowCardinalityNullable()) diff --git a/tests/queries/0_stateless/03231_values_respect_format_settings_in_fields_conversion.reference b/tests/queries/0_stateless/03231_values_respect_format_settings_in_fields_conversion.reference new file mode 100644 index 00000000000..8dbf92d6590 --- /dev/null +++ b/tests/queries/0_stateless/03231_values_respect_format_settings_in_fields_conversion.reference @@ -0,0 +1 @@ +{'Hello':'2020-01-01 00:00:00'} diff --git a/tests/queries/0_stateless/03231_values_respect_format_settings_in_fields_conversion.sql b/tests/queries/0_stateless/03231_values_respect_format_settings_in_fields_conversion.sql new file mode 100644 index 00000000000..484a16bb22f --- /dev/null +++ b/tests/queries/0_stateless/03231_values_respect_format_settings_in_fields_conversion.sql @@ -0,0 +1,7 @@ +drop table if exists test; +create table test (map Map(String, DateTime)) engine=Memory; +set date_time_input_format='best_effort'; +insert into test values (map('Hello', '01/01/2020')); +select * from test; +drop table test; +