thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 15:42:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #18445 from ClickHouse/fix_to_nullable_conversion

Browse Source 
Try fix 'value is too short' when converting from String to Nullable(T)
This commit is contained in:
tavplubix 2020-12-24 13:38:43 +03:00 committed by GitHub
commit 5fbfc1935c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 181 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
src/Core/callOnTypeIndex.h
View File

@ -162,43 +162,43 @@ template <typename T> class DataTypeNumber;
template <typename T> class DataTypeDecimal; template <typename T> class DataTypeDecimal;
template <typename T, typename F> template <typename T, typename F, typename... ExtraArgs>
bool callOnIndexAndDataType(TypeIndex number, F && f) bool callOnIndexAndDataType(TypeIndex number, F && f, ExtraArgs && ... args)
{ {
switch (number) switch (number)
{ {
case TypeIndex::UInt8: return f(TypePair<DataTypeNumber<UInt8>, T>()); case TypeIndex::UInt8: return f(TypePair<DataTypeNumber<UInt8>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::UInt16: return f(TypePair<DataTypeNumber<UInt16>, T>()); case TypeIndex::UInt16: return f(TypePair<DataTypeNumber<UInt16>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::UInt32: return f(TypePair<DataTypeNumber<UInt32>, T>()); case TypeIndex::UInt32: return f(TypePair<DataTypeNumber<UInt32>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::UInt64: return f(TypePair<DataTypeNumber<UInt64>, T>()); case TypeIndex::UInt64: return f(TypePair<DataTypeNumber<UInt64>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::UInt256: return f(TypePair<DataTypeNumber<UInt256>, T>()); case TypeIndex::UInt256: return f(TypePair<DataTypeNumber<UInt256>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::Int8: return f(TypePair<DataTypeNumber<Int8>, T>()); case TypeIndex::Int8: return f(TypePair<DataTypeNumber<Int8>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::Int16: return f(TypePair<DataTypeNumber<Int16>, T>()); case TypeIndex::Int16: return f(TypePair<DataTypeNumber<Int16>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::Int32: return f(TypePair<DataTypeNumber<Int32>, T>()); case TypeIndex::Int32: return f(TypePair<DataTypeNumber<Int32>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::Int64: return f(TypePair<DataTypeNumber<Int64>, T>()); case TypeIndex::Int64: return f(TypePair<DataTypeNumber<Int64>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::Int128: return f(TypePair<DataTypeNumber<Int128>, T>()); case TypeIndex::Int128: return f(TypePair<DataTypeNumber<Int128>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::Int256: return f(TypePair<DataTypeNumber<Int256>, T>()); case TypeIndex::Int256: return f(TypePair<DataTypeNumber<Int256>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::Float32: return f(TypePair<DataTypeNumber<Float32>, T>()); case TypeIndex::Float32: return f(TypePair<DataTypeNumber<Float32>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::Float64: return f(TypePair<DataTypeNumber<Float64>, T>()); case TypeIndex::Float64: return f(TypePair<DataTypeNumber<Float64>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::Decimal32: return f(TypePair<DataTypeDecimal<Decimal32>, T>()); case TypeIndex::Decimal32: return f(TypePair<DataTypeDecimal<Decimal32>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::Decimal64: return f(TypePair<DataTypeDecimal<Decimal64>, T>()); case TypeIndex::Decimal64: return f(TypePair<DataTypeDecimal<Decimal64>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::Decimal128: return f(TypePair<DataTypeDecimal<Decimal128>, T>()); case TypeIndex::Decimal128: return f(TypePair<DataTypeDecimal<Decimal128>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::Decimal256: return f(TypePair<DataTypeDecimal<Decimal256>, T>()); case TypeIndex::Decimal256: return f(TypePair<DataTypeDecimal<Decimal256>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::Date: return f(TypePair<DataTypeDate, T>()); case TypeIndex::Date: return f(TypePair<DataTypeDate, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::DateTime: return f(TypePair<DataTypeDateTime, T>()); case TypeIndex::DateTime: return f(TypePair<DataTypeDateTime, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::DateTime64: return f(TypePair<DataTypeDateTime64, T>()); case TypeIndex::DateTime64: return f(TypePair<DataTypeDateTime64, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::String: return f(TypePair<DataTypeString, T>()); case TypeIndex::String: return f(TypePair<DataTypeString, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::FixedString: return f(TypePair<DataTypeFixedString, T>()); case TypeIndex::FixedString: return f(TypePair<DataTypeFixedString, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::Enum8: return f(TypePair<DataTypeEnum<Int8>, T>()); case TypeIndex::Enum8: return f(TypePair<DataTypeEnum<Int8>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::Enum16: return f(TypePair<DataTypeEnum<Int16>, T>()); case TypeIndex::Enum16: return f(TypePair<DataTypeEnum<Int16>, T>(), std::forward<ExtraArgs>(args)...);
case TypeIndex::UUID: return f(TypePair<DataTypeUUID, T>()); case TypeIndex::UUID: return f(TypePair<DataTypeUUID, T>(), std::forward<ExtraArgs>(args)...);
default: default:
break; break;

110
src/Functions/FunctionsConversion.h
View File

@ -108,10 +108,14 @@ struct AccurateOrNullConvertStrategyAdditions
UInt32 scale { 0 }; UInt32 scale { 0 };
}; };
struct ConvertDefaultBehaviorTag {};
struct ConvertReturnNullOnErrorTag {};
/** Conversion of number types to each other, enums to numbers, dates and datetimes to numbers and back: done by straight assignment. /** Conversion of number types to each other, enums to numbers, dates and datetimes to numbers and back: done by straight assignment.
* (Date is represented internally as number of days from some day; DateTime - as unix timestamp) * (Date is represented internally as number of days from some day; DateTime - as unix timestamp)
*/ */
template <typename FromDataType, typename ToDataType, typename Name> template <typename FromDataType, typename ToDataType, typename Name, typename SpecialTag = ConvertDefaultBehaviorTag>
struct ConvertImpl struct ConvertImpl
{ {
using FromFieldType = typename FromDataType::FieldType; using FromFieldType = typename FromDataType::FieldType;
@ -279,7 +283,7 @@ struct ConvertImpl
/** Conversion of DateTime to Date: throw off time component. /** Conversion of DateTime to Date: throw off time component.
*/ */
template <typename Name> struct ConvertImpl<DataTypeDateTime, DataTypeDate, Name> template <typename Name> struct ConvertImpl<DataTypeDateTime, DataTypeDate, Name, ConvertDefaultBehaviorTag>
: DateTimeTransformImpl<DataTypeDateTime, DataTypeDate, ToDateImpl> {}; : DateTimeTransformImpl<DataTypeDateTime, DataTypeDate, ToDateImpl> {};
@ -301,7 +305,7 @@ struct ToDateTimeImpl
} }
}; };
template <typename Name> struct ConvertImpl<DataTypeDate, DataTypeDateTime, Name> template <typename Name> struct ConvertImpl<DataTypeDate, DataTypeDateTime, Name, ConvertDefaultBehaviorTag>
: DateTimeTransformImpl<DataTypeDate, DataTypeDateTime, ToDateTimeImpl> {}; : DateTimeTransformImpl<DataTypeDate, DataTypeDateTime, ToDateTimeImpl> {};
/// Implementation of toDate function. /// Implementation of toDate function.
@ -357,21 +361,21 @@ struct ToDateTransform8Or16Signed
* when user write toDate(UInt32), expecting conversion of unix timestamp to Date. * when user write toDate(UInt32), expecting conversion of unix timestamp to Date.
* (otherwise such usage would be frequent mistake). * (otherwise such usage would be frequent mistake).
*/ */
template <typename Name> struct ConvertImpl<DataTypeUInt32, DataTypeDate, Name> template <typename Name> struct ConvertImpl<DataTypeUInt32, DataTypeDate, Name, ConvertDefaultBehaviorTag>
: DateTimeTransformImpl<DataTypeUInt32, DataTypeDate, ToDateTransform32Or64<UInt32, UInt16>> {}; : DateTimeTransformImpl<DataTypeUInt32, DataTypeDate, ToDateTransform32Or64<UInt32, UInt16>> {};
template <typename Name> struct ConvertImpl<DataTypeUInt64, DataTypeDate, Name> template <typename Name> struct ConvertImpl<DataTypeUInt64, DataTypeDate, Name, ConvertDefaultBehaviorTag>
: DateTimeTransformImpl<DataTypeUInt64, DataTypeDate, ToDateTransform32Or64<UInt64, UInt16>> {}; : DateTimeTransformImpl<DataTypeUInt64, DataTypeDate, ToDateTransform32Or64<UInt64, UInt16>> {};
template <typename Name> struct ConvertImpl<DataTypeInt8, DataTypeDate, Name> template <typename Name> struct ConvertImpl<DataTypeInt8, DataTypeDate, Name, ConvertDefaultBehaviorTag>
: DateTimeTransformImpl<DataTypeInt8, DataTypeDate, ToDateTransform8Or16Signed<Int8, UInt16>> {}; : DateTimeTransformImpl<DataTypeInt8, DataTypeDate, ToDateTransform8Or16Signed<Int8, UInt16>> {};
template <typename Name> struct ConvertImpl<DataTypeInt16, DataTypeDate, Name> template <typename Name> struct ConvertImpl<DataTypeInt16, DataTypeDate, Name, ConvertDefaultBehaviorTag>
: DateTimeTransformImpl<DataTypeInt16, DataTypeDate, ToDateTransform8Or16Signed<Int16, UInt16>> {}; : DateTimeTransformImpl<DataTypeInt16, DataTypeDate, ToDateTransform8Or16Signed<Int16, UInt16>> {};
template <typename Name> struct ConvertImpl<DataTypeInt32, DataTypeDate, Name> template <typename Name> struct ConvertImpl<DataTypeInt32, DataTypeDate, Name, ConvertDefaultBehaviorTag>
: DateTimeTransformImpl<DataTypeInt32, DataTypeDate, ToDateTransform32Or64Signed<Int32, UInt16>> {}; : DateTimeTransformImpl<DataTypeInt32, DataTypeDate, ToDateTransform32Or64Signed<Int32, UInt16>> {};
template <typename Name> struct ConvertImpl<DataTypeInt64, DataTypeDate, Name> template <typename Name> struct ConvertImpl<DataTypeInt64, DataTypeDate, Name, ConvertDefaultBehaviorTag>
: DateTimeTransformImpl<DataTypeInt64, DataTypeDate, ToDateTransform32Or64Signed<Int64, UInt16>> {}; : DateTimeTransformImpl<DataTypeInt64, DataTypeDate, ToDateTransform32Or64Signed<Int64, UInt16>> {};
template <typename Name> struct ConvertImpl<DataTypeFloat32, DataTypeDate, Name> template <typename Name> struct ConvertImpl<DataTypeFloat32, DataTypeDate, Name, ConvertDefaultBehaviorTag>
: DateTimeTransformImpl<DataTypeFloat32, DataTypeDate, ToDateTransform32Or64Signed<Float32, UInt16>> {}; : DateTimeTransformImpl<DataTypeFloat32, DataTypeDate, ToDateTransform32Or64Signed<Float32, UInt16>> {};
template <typename Name> struct ConvertImpl<DataTypeFloat64, DataTypeDate, Name> template <typename Name> struct ConvertImpl<DataTypeFloat64, DataTypeDate, Name, ConvertDefaultBehaviorTag>
: DateTimeTransformImpl<DataTypeFloat64, DataTypeDate, ToDateTransform32Or64Signed<Float64, UInt16>> {}; : DateTimeTransformImpl<DataTypeFloat64, DataTypeDate, ToDateTransform32Or64Signed<Float64, UInt16>> {};
@ -456,9 +460,9 @@ struct ToDateTime64Transform
} }
}; };
template <typename Name> struct ConvertImpl<DataTypeDate, DataTypeDateTime64, Name> template <typename Name> struct ConvertImpl<DataTypeDate, DataTypeDateTime64, Name, ConvertDefaultBehaviorTag>
: DateTimeTransformImpl<DataTypeDate, DataTypeDateTime64, ToDateTime64Transform> {}; : DateTimeTransformImpl<DataTypeDate, DataTypeDateTime64, ToDateTime64Transform> {};
template <typename Name> struct ConvertImpl<DataTypeDateTime, DataTypeDateTime64, Name> template <typename Name> struct ConvertImpl<DataTypeDateTime, DataTypeDateTime64, Name, ConvertDefaultBehaviorTag>
: DateTimeTransformImpl<DataTypeDateTime, DataTypeDateTime64, ToDateTime64Transform> {}; : DateTimeTransformImpl<DataTypeDateTime, DataTypeDateTime64, ToDateTime64Transform> {};
/** Conversion of DateTime64 to Date or DateTime: discards fractional part. /** Conversion of DateTime64 to Date or DateTime: discards fractional part.
@ -481,9 +485,9 @@ struct FromDateTime64Transform
} }
}; };
template <typename Name> struct ConvertImpl<DataTypeDateTime64, DataTypeDate, Name> template <typename Name> struct ConvertImpl<DataTypeDateTime64, DataTypeDate, Name, ConvertDefaultBehaviorTag>
: DateTimeTransformImpl<DataTypeDateTime64, DataTypeDate, FromDateTime64Transform<ToDateImpl>> {}; : DateTimeTransformImpl<DataTypeDateTime64, DataTypeDate, FromDateTime64Transform<ToDateImpl>> {};
template <typename Name> struct ConvertImpl<DataTypeDateTime64, DataTypeDateTime, Name> template <typename Name> struct ConvertImpl<DataTypeDateTime64, DataTypeDateTime, Name, ConvertDefaultBehaviorTag>
: DateTimeTransformImpl<DataTypeDateTime64, DataTypeDateTime, FromDateTime64Transform<ToDateTimeImpl>> {}; : DateTimeTransformImpl<DataTypeDateTime64, DataTypeDateTime, FromDateTime64Transform<ToDateTimeImpl>> {};
@ -547,7 +551,7 @@ struct FormatImpl<DataTypeDecimal<FieldType>>
/// DataTypeEnum<T> to DataType<T> free conversion /// DataTypeEnum<T> to DataType<T> free conversion
template <typename FieldType, typename Name> template <typename FieldType, typename Name>
struct ConvertImpl<DataTypeEnum<FieldType>, DataTypeNumber<FieldType>, Name> struct ConvertImpl<DataTypeEnum<FieldType>, DataTypeNumber<FieldType>, Name, ConvertDefaultBehaviorTag>
{ {
static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/)
{ {
@ -557,7 +561,7 @@ struct ConvertImpl<DataTypeEnum<FieldType>, DataTypeNumber<FieldType>, Name>
template <typename FromDataType, typename Name> template <typename FromDataType, typename Name>
struct ConvertImpl<FromDataType, std::enable_if_t<!std::is_same_v<FromDataType, DataTypeString>, DataTypeString>, Name> struct ConvertImpl<FromDataType, std::enable_if_t<!std::is_same_v<FromDataType, DataTypeString>, DataTypeString>, Name, ConvertDefaultBehaviorTag>
{ {
using FromFieldType = typename FromDataType::FieldType; using FromFieldType = typename FromDataType::FieldType;
using ColVecType = std::conditional_t<IsDecimalNumber<FromFieldType>, ColumnDecimal<FromFieldType>, ColumnVector<FromFieldType>>; using ColVecType = std::conditional_t<IsDecimalNumber<FromFieldType>, ColumnDecimal<FromFieldType>, ColumnVector<FromFieldType>>;
@ -974,13 +978,21 @@ struct ConvertThroughParsing
template <typename ToDataType, typename Name> template <typename ToDataType, typename Name>
struct ConvertImpl<std::enable_if_t<!std::is_same_v<ToDataType, DataTypeString>, DataTypeString>, ToDataType, Name> struct ConvertImpl<std::enable_if_t<!std::is_same_v<ToDataType, DataTypeString>, DataTypeString>, ToDataType, Name, ConvertDefaultBehaviorTag>
: ConvertThroughParsing<DataTypeString, ToDataType, Name, ConvertFromStringExceptionMode::Throw, ConvertFromStringParsingMode::Normal> {}; : ConvertThroughParsing<DataTypeString, ToDataType, Name, ConvertFromStringExceptionMode::Throw, ConvertFromStringParsingMode::Normal> {};
template <typename ToDataType, typename Name> template <typename ToDataType, typename Name>
struct ConvertImpl<std::enable_if_t<!std::is_same_v<ToDataType, DataTypeFixedString>, DataTypeFixedString>, ToDataType, Name> struct ConvertImpl<std::enable_if_t<!std::is_same_v<ToDataType, DataTypeFixedString>, DataTypeFixedString>, ToDataType, Name, ConvertDefaultBehaviorTag>
: ConvertThroughParsing<DataTypeFixedString, ToDataType, Name, ConvertFromStringExceptionMode::Throw, ConvertFromStringParsingMode::Normal> {}; : ConvertThroughParsing<DataTypeFixedString, ToDataType, Name, ConvertFromStringExceptionMode::Throw, ConvertFromStringParsingMode::Normal> {};
template <typename ToDataType, typename Name>
struct ConvertImpl<std::enable_if_t<!std::is_same_v<ToDataType, DataTypeString>, DataTypeString>, ToDataType, Name, ConvertReturnNullOnErrorTag>
: ConvertThroughParsing<DataTypeString, ToDataType, Name, ConvertFromStringExceptionMode::Null, ConvertFromStringParsingMode::Normal> {};
template <typename ToDataType, typename Name>
struct ConvertImpl<std::enable_if_t<!std::is_same_v<ToDataType, DataTypeFixedString>, DataTypeFixedString>, ToDataType, Name, ConvertReturnNullOnErrorTag>
: ConvertThroughParsing<DataTypeFixedString, ToDataType, Name, ConvertFromStringExceptionMode::Null, ConvertFromStringParsingMode::Normal> {};
/// Generic conversion of any type from String. Used for complex types: Array and Tuple. /// Generic conversion of any type from String. Used for complex types: Array and Tuple.
struct ConvertImplGenericFromString struct ConvertImplGenericFromString
{ {
@ -1027,14 +1039,17 @@ struct ConvertImplGenericFromString
template <> template <>
struct ConvertImpl<DataTypeString, DataTypeUInt32, NameToUnixTimestamp> struct ConvertImpl<DataTypeString, DataTypeUInt32, NameToUnixTimestamp, ConvertDefaultBehaviorTag>
: ConvertImpl<DataTypeString, DataTypeDateTime, NameToUnixTimestamp> {}; : ConvertImpl<DataTypeString, DataTypeDateTime, NameToUnixTimestamp, ConvertDefaultBehaviorTag> {};
template <>
struct ConvertImpl<DataTypeString, DataTypeUInt32, NameToUnixTimestamp, ConvertReturnNullOnErrorTag>
: ConvertImpl<DataTypeString, DataTypeDateTime, NameToUnixTimestamp, ConvertReturnNullOnErrorTag> {};
/** If types are identical, just take reference to column. /** If types are identical, just take reference to column.
*/ */
template <typename T, typename Name> template <typename T, typename Name>
struct ConvertImpl<std::enable_if_t<!T::is_parametric, T>, T, Name> struct ConvertImpl<std::enable_if_t<!T::is_parametric, T>, T, Name, ConvertDefaultBehaviorTag>
{ {
template <typename Additions = void *> template <typename Additions = void *>
static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/, static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/,
@ -1049,7 +1064,7 @@ struct ConvertImpl<std::enable_if_t<!T::is_parametric, T>, T, Name>
* Cutting sequences of zero bytes from end of strings. * Cutting sequences of zero bytes from end of strings.
*/ */
template <typename Name> template <typename Name>
struct ConvertImpl<DataTypeFixedString, DataTypeString, Name> struct ConvertImpl<DataTypeFixedString, DataTypeString, Name, ConvertDefaultBehaviorTag>
{ {
static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/)
{ {
@ -1153,6 +1168,9 @@ public:
static constexpr bool to_datetime64 = std::is_same_v<ToDataType, DataTypeDateTime64>; static constexpr bool to_datetime64 = std::is_same_v<ToDataType, DataTypeDateTime64>;
static constexpr bool to_string_or_fixed_string = std::is_same_v<ToDataType, DataTypeFixedString> ||
std::is_same_v<ToDataType, DataTypeString>;
static FunctionPtr create(const Context &) { return std::make_shared<FunctionConvert>(); } static FunctionPtr create(const Context &) { return std::make_shared<FunctionConvert>(); }
static FunctionPtr create() { return std::make_shared<FunctionConvert>(); } static FunctionPtr create() { return std::make_shared<FunctionConvert>(); }
@ -1166,6 +1184,15 @@ public:
bool isInjective(const ColumnsWithTypeAndName &) const override { return std::is_same_v<Name, NameToString>; } bool isInjective(const ColumnsWithTypeAndName &) const override { return std::is_same_v<Name, NameToString>; }
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
auto getter = [&] (const auto & args) { return getReturnTypeImplRemovedNullable(args); };
auto res = FunctionOverloadResolverAdaptor::getReturnTypeDefaultImplementationForNulls(arguments, getter);
to_nullable = res->isNullable();
checked_return_type = true;
return res;
}
DataTypePtr getReturnTypeImplRemovedNullable(const ColumnsWithTypeAndName & arguments) const
{ {
FunctionArgumentDescriptors mandatory_args = {{"Value", nullptr, nullptr, nullptr}}; FunctionArgumentDescriptors mandatory_args = {{"Value", nullptr, nullptr, nullptr}};
FunctionArgumentDescriptors optional_args; FunctionArgumentDescriptors optional_args;
@ -1243,6 +1270,11 @@ public:
} }
} }
/// Function actually uses default implementation for nulls,
/// but we need to know if return type is Nullable or not,
/// so we use checked_return_type only to intercept the first call to getReturnTypeImpl(...).
bool useDefaultImplementationForNulls() const override { return checked_return_type; }
bool useDefaultImplementationForConstants() const override { return true; } bool useDefaultImplementationForConstants() const override { return true; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
bool canBeExecutedOnDefaultArguments() const override { return false; } bool canBeExecutedOnDefaultArguments() const override { return false; }
@ -1292,6 +1324,9 @@ public:
} }
private: private:
mutable bool checked_return_type = false;
mutable bool to_nullable = false;
ColumnPtr executeInternal(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const ColumnPtr executeInternal(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const
{ {
if (arguments.empty()) if (arguments.empty())
@ -1301,11 +1336,12 @@ private:
const IDataType * from_type = arguments[0].type.get(); const IDataType * from_type = arguments[0].type.get();
ColumnPtr result_column; ColumnPtr result_column;
auto call = [&](const auto & types) -> bool auto call = [&](const auto & types, const auto & tag) -> bool
{ {
using Types = std::decay_t<decltype(types)>; using Types = std::decay_t<decltype(types)>;
using LeftDataType = typename Types::LeftType; using LeftDataType = typename Types::LeftType;
using RightDataType = typename Types::RightType; using RightDataType = typename Types::RightType;
using SpecialTag = std::decay_t<decltype(tag)>;
if constexpr (IsDataTypeDecimal<RightDataType>) if constexpr (IsDataTypeDecimal<RightDataType>)
{ {
@ -1325,12 +1361,12 @@ private:
const ColumnWithTypeAndName & scale_column = arguments[1]; const ColumnWithTypeAndName & scale_column = arguments[1];
UInt32 scale = extractToDecimalScale(scale_column); UInt32 scale = extractToDecimalScale(scale_column);
result_column = ConvertImpl<LeftDataType, RightDataType, Name>::execute(arguments, result_type, input_rows_count, scale); result_column = ConvertImpl<LeftDataType, RightDataType, Name, SpecialTag>::execute(arguments, result_type, input_rows_count, scale);
} }
else if constexpr (IsDataTypeDateOrDateTime<RightDataType> && std::is_same_v<LeftDataType, DataTypeDateTime64>) else if constexpr (IsDataTypeDateOrDateTime<RightDataType> && std::is_same_v<LeftDataType, DataTypeDateTime64>)
{ {
const auto * dt64 = assert_cast<const DataTypeDateTime64 *>(arguments[0].type.get()); const auto * dt64 = assert_cast<const DataTypeDateTime64 *>(arguments[0].type.get());
result_column = ConvertImpl<LeftDataType, RightDataType, Name>::execute(arguments, result_type, input_rows_count, dt64->getScale()); result_column = ConvertImpl<LeftDataType, RightDataType, Name, SpecialTag>::execute(arguments, result_type, input_rows_count, dt64->getScale());
} }
else if constexpr (IsDataTypeDecimalOrNumber<LeftDataType> && IsDataTypeDecimalOrNumber<RightDataType>) else if constexpr (IsDataTypeDecimalOrNumber<LeftDataType> && IsDataTypeDecimalOrNumber<RightDataType>)
{ {
@ -1349,10 +1385,10 @@ private:
throw Exception("Wrong UUID conversion", ErrorCodes::CANNOT_CONVERT_TYPE); throw Exception("Wrong UUID conversion", ErrorCodes::CANNOT_CONVERT_TYPE);
} }
else else
result_column = ConvertImpl<LeftDataType, RightDataType, Name>::execute(arguments, result_type, input_rows_count); result_column = ConvertImpl<LeftDataType, RightDataType, Name, SpecialTag>::execute(arguments, result_type, input_rows_count);
} }
else else
result_column = ConvertImpl<LeftDataType, RightDataType, Name>::execute(arguments, result_type, input_rows_count); result_column = ConvertImpl<LeftDataType, RightDataType, Name, SpecialTag>::execute(arguments, result_type, input_rows_count);
return true; return true;
}; };
@ -1365,7 +1401,7 @@ private:
if (to_datetime64 || scale != 0) /// When scale = 0, the data type is DateTime otherwise the data type is DateTime64 if (to_datetime64 || scale != 0) /// When scale = 0, the data type is DateTime otherwise the data type is DateTime64
{ {
if (!callOnIndexAndDataType<DataTypeDateTime64>(from_type->getTypeId(), call)) if (!callOnIndexAndDataType<DataTypeDateTime64>(from_type->getTypeId(), call, ConvertDefaultBehaviorTag{}))
throw Exception("Illegal type " + arguments[0].type->getName() + " of argument of function " + getName(), throw Exception("Illegal type " + arguments[0].type->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
@ -1373,7 +1409,21 @@ private:
} }
} }
bool done = callOnIndexAndDataType<ToDataType>(from_type->getTypeId(), call); bool done;
if constexpr (to_string_or_fixed_string)
{
done = callOnIndexAndDataType<ToDataType>(from_type->getTypeId(), call, ConvertDefaultBehaviorTag{});
}
else
{
/// We should use ConvertFromStringExceptionMode::Null mode when converting from String (or FixedString)
/// to Nullable type, to avoid 'value is too short' error on attempt to parse empty string from NULL values.
if (to_nullable && WhichDataType(from_type).isStringOrFixedString())
done = callOnIndexAndDataType<ToDataType>(from_type->getTypeId(), call, ConvertReturnNullOnErrorTag{});
else
done = callOnIndexAndDataType<ToDataType>(from_type->getTypeId(), call, ConvertDefaultBehaviorTag{});
}
if (!done) if (!done)
{ {
/// Generic conversion of any type to String. /// Generic conversion of any type to String.

35
src/Functions/IFunction.cpp
View File

@ -465,25 +465,32 @@ void FunctionOverloadResolverAdaptor::checkNumberOfArguments(size_t number_of_ar
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
} }
DataTypePtr FunctionOverloadResolverAdaptor::getReturnTypeDefaultImplementationForNulls(const ColumnsWithTypeAndName & arguments,
const DefaultReturnTypeGetter & getter)
{
NullPresence null_presence = getNullPresense(arguments);
if (null_presence.has_null_constant)
{
return makeNullable(std::make_shared<DataTypeNothing>());
}
if (null_presence.has_nullable)
{
Block nested_columns = createBlockWithNestedColumns(arguments);
auto return_type = getter(ColumnsWithTypeAndName(nested_columns.begin(), nested_columns.end()));
return makeNullable(return_type);
}
return getter(arguments);
}
DataTypePtr FunctionOverloadResolverAdaptor::getReturnTypeWithoutLowCardinality(const ColumnsWithTypeAndName & arguments) const DataTypePtr FunctionOverloadResolverAdaptor::getReturnTypeWithoutLowCardinality(const ColumnsWithTypeAndName & arguments) const
{ {
checkNumberOfArguments(arguments.size()); checkNumberOfArguments(arguments.size());
if (!arguments.empty() && impl->useDefaultImplementationForNulls()) if (!arguments.empty() && impl->useDefaultImplementationForNulls())
{ return getReturnTypeDefaultImplementationForNulls(arguments, [&](const auto & args) { return impl->getReturnType(args); });
NullPresence null_presence = getNullPresense(arguments);
if (null_presence.has_null_constant)
{
return makeNullable(std::make_shared<DataTypeNothing>());
}
if (null_presence.has_nullable)
{
Block nested_columns = createBlockWithNestedColumns(arguments);
auto return_type = impl->getReturnType(ColumnsWithTypeAndName(nested_columns.begin(), nested_columns.end()));
return makeNullable(return_type);
}
}
return impl->getReturnType(arguments); return impl->getReturnType(arguments);
} }

2
src/Functions/IFunctionAdaptors.h
View File

@ -129,6 +129,8 @@ public:
return impl->getArgumentsThatDontImplyNullableReturnType(number_of_arguments); return impl->getArgumentsThatDontImplyNullableReturnType(number_of_arguments);
} }
using DefaultReturnTypeGetter = std::function<DataTypePtr(const ColumnsWithTypeAndName &)>;
static DataTypePtr getReturnTypeDefaultImplementationForNulls(const ColumnsWithTypeAndName & arguments, const DefaultReturnTypeGetter & getter);
private: private:
FunctionOverloadResolverImplPtr impl; FunctionOverloadResolverImplPtr impl;

38
tests/queries/0_stateless/01186_conversion_to_nullable.reference Normal file
View File

@ -0,0 +1,38 @@
42
\N
0
\N
\N
42
\N
0
\N
256
2020-12-24
\N
1970-01-01
\N
1970-01-01
2020-12-24 01:02:03
\N
1970-01-01 03:00:00
\N
2020-12-24 01:02:03.00
\N
1970-01-01 03:00:00.00
1970-01-01 03:00:00.00
946721532
\N
\N
42.00
\N
\N
42.00000000
\N
3.14159000
42
\N
test
42\0\0\0\0\0\0
\N
test\0\0\0\0

13
tests/queries/0_stateless/01186_conversion_to_nullable.sql Normal file
View File

@ -0,0 +1,13 @@
select toUInt8(x) from values('x Nullable(String)', '42', NULL, '0', '', '256');
select toInt64(x) from values('x Nullable(String)', '42', NULL, '0', '', '256');
select toDate(x) from values('x Nullable(String)', '2020-12-24', NULL, '0000-00-00', '', '9999-01-01');
select toDateTime(x) from values('x Nullable(String)', '2020-12-24 01:02:03', NULL, '0000-00-00 00:00:00', '');
select toDateTime64(x, 2) from values('x Nullable(String)', '2020-12-24 01:02:03', NULL, '0000-00-00 00:00:00', '');
select toUnixTimestamp(x) from values ('x Nullable(String)', '2000-01-01 13:12:12', NULL, '');
select toDecimal32(x, 2) from values ('x Nullable(String)', '42', NULL, '3.14159');
select toDecimal64(x, 8) from values ('x Nullable(String)', '42', NULL, '3.14159');
select toString(x) from values ('x Nullable(String)', '42', NULL, 'test');
select toFixedString(x, 8) from values ('x Nullable(String)', '42', NULL, 'test');