From 408fc241d685d4aaab4b8b3aeb3512fe0baed124 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 20 Jun 2020 17:43:01 +0300 Subject: [PATCH] Make reading of Decimal more compatible with other DBMS --- src/Common/intExp.h | 11 +++++ src/Functions/intExp10.cpp | 9 +++- src/IO/readDecimalText.h | 47 ++++++++++++------- ...mal_cut_extra_digits_after_point.reference | 14 ++++++ ...7_decimal_cut_extra_digits_after_point.sql | 24 ++++++++++ 5 files changed, 87 insertions(+), 18 deletions(-) create mode 100644 tests/queries/0_stateless/01327_decimal_cut_extra_digits_after_point.reference create mode 100644 tests/queries/0_stateless/01327_decimal_cut_extra_digits_after_point.sql diff --git a/src/Common/intExp.h b/src/Common/intExp.h index 0212eb4c084..a021d9660ff 100644 --- a/src/Common/intExp.h +++ b/src/Common/intExp.h @@ -134,3 +134,14 @@ inline __int128 exp10_i128(int x) } } + + +/// intExp10 returning the type T. +template +inline T intExp10OfSize(int x) +{ + if constexpr (sizeof(T) <= 8) + return intExp10(x); + else + return common::exp10_i128(x); +} diff --git a/src/Functions/intExp10.cpp b/src/Functions/intExp10.cpp index 112e21a51b4..3192c9e2f79 100644 --- a/src/Functions/intExp10.cpp +++ b/src/Functions/intExp10.cpp @@ -31,8 +31,13 @@ template <> struct FunctionUnaryArithmeticMonotonicity static bool has() { return true; } static IFunction::Monotonicity get(const Field & left, const Field & right) { - Float64 left_float = left.isNull() ? -std::numeric_limits::infinity() : applyVisitor(FieldVisitorConvertToNumber(), left); - Float64 right_float = right.isNull() ? std::numeric_limits::infinity() : applyVisitor(FieldVisitorConvertToNumber(), right); + Float64 left_float = left.isNull() + ? -std::numeric_limits::infinity() + : applyVisitor(FieldVisitorConvertToNumber(), left); + + Float64 right_float = right.isNull() + ? std::numeric_limits::infinity() + : applyVisitor(FieldVisitorConvertToNumber(), right); if (left_float < 0 || right_float > 19) return {}; diff --git a/src/IO/readDecimalText.h b/src/IO/readDecimalText.h index 5a6a63529e2..fd8aae8023e 100644 --- a/src/IO/readDecimalText.h +++ b/src/IO/readDecimalText.h @@ -1,4 +1,5 @@ #include +#include namespace DB @@ -80,21 +81,33 @@ inline bool readDigits(ReadBuffer & buf, T & x, uint32_t & digits, int32_t & exp ++places; // num zeroes before + current digit if (digits + places > max_digits) { - if constexpr (_throw_on_error) - throw Exception("Too many digits (" + std::to_string(digits + places) + " > " + std::to_string(max_digits) - + ") in decimal value", ErrorCodes::ARGUMENT_OUT_OF_BOUND); - return false; + if (after_point) + { + /// Simply cut excessive digits. + break; + } + else + { + if constexpr (_throw_on_error) + throw Exception("Too many digits (" + std::to_string(digits + places) + " > " + std::to_string(max_digits) + + ") in decimal value", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + return false; + } } + else + { + digits += places; + if (after_point) + exponent -= places; - digits += places; - if (after_point) - exponent -= places; + // TODO: accurate shift10 for big integers + x *= intExp10OfSize(places); + places = 0; - // TODO: accurate shift10 for big integers - for (; places; --places) - x *= 10; - x += (byte - '0'); - break; + x += (byte - '0'); + break; + } } case 'e': [[fallthrough]]; case 'E': @@ -144,10 +157,12 @@ inline void readDecimalText(ReadBuffer & buf, T & x, uint32_t precision, uint32_ digits, x, exponent, scale, precision), ErrorCodes::ARGUMENT_OUT_OF_BOUND); if (static_cast(scale) + exponent < 0) - throw Exception(fmt::format( - "Decimal value has too large number of digits after point: {} digits were read: {}e{}." - " Expected to read decimal with scale {} and precision {}", - digits, x, exponent, scale, precision), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + { + /// Too many digits after point. Just cut off excessive digits. + x.value /= intExp10OfSize(-exponent - scale); + scale = 0; + return; + } scale += exponent; } diff --git a/tests/queries/0_stateless/01327_decimal_cut_extra_digits_after_point.reference b/tests/queries/0_stateless/01327_decimal_cut_extra_digits_after_point.reference new file mode 100644 index 00000000000..6006f9981b4 --- /dev/null +++ b/tests/queries/0_stateless/01327_decimal_cut_extra_digits_after_point.reference @@ -0,0 +1,14 @@ +1.10000 +1.12345 +1.12345 +1.12345 +1.12345 +1.12345 +12345.10000 +123456789123.10000 +1234567891234.10000 +1234567891234.12345 +1.12345 +1.12345 +1.12344 +1.12345 diff --git a/tests/queries/0_stateless/01327_decimal_cut_extra_digits_after_point.sql b/tests/queries/0_stateless/01327_decimal_cut_extra_digits_after_point.sql new file mode 100644 index 00000000000..4d456296d8f --- /dev/null +++ b/tests/queries/0_stateless/01327_decimal_cut_extra_digits_after_point.sql @@ -0,0 +1,24 @@ +SELECT CAST('1.1' AS Decimal(10, 5)); +SELECT CAST('1.12345' AS Decimal(10, 5)); +SELECT CAST('1.123451' AS Decimal(10, 5)); +SELECT CAST('1.1234511111' AS Decimal(10, 5)); +SELECT CAST('1.12345111111' AS Decimal(10, 5)); +SELECT CAST('1.12345111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111' AS Decimal(10, 5)); +SELECT CAST('12345.1' AS Decimal(10, 5)); + +-- Actually our decimal can contain more than 10 digits for free. +SELECT CAST('123456789123.1' AS Decimal(10, 5)); +SELECT CAST('1234567891234.1' AS Decimal(10, 5)); +SELECT CAST('1234567891234.12345111' AS Decimal(10, 5)); +-- But it's just Decimal64, so there is the limit. +SELECT CAST('12345678912345.1' AS Decimal(10, 5)); -- { serverError 69 } + +-- The rounding may work in unexpected way: this is just integer rounding. +-- We can improve it but here is the current behaviour: +SELECT CAST('1.123455' AS Decimal(10, 5)); +SELECT CAST('1.123456' AS Decimal(10, 5)); +SELECT CAST('1.123445' AS Decimal(10, 5)); -- Check if suddenly banker's rounding will be implemented. + +CREATE TEMPORARY TABLE test (x Decimal(10, 5)); +INSERT INTO test VALUES (1.12345111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111); +SELECT * FROM test;