From 582f76c83bd62b11a12252b564701fc2665775cb Mon Sep 17 00:00:00 2001 From: chertus Date: Mon, 30 Jul 2018 21:10:38 +0300 Subject: [PATCH] string to decimal, decimal to string, check decimal bounds --- dbms/src/DataTypes/DataTypesDecimal.cpp | 29 +++++- dbms/src/DataTypes/DataTypesDecimal.h | 2 + dbms/src/IO/WriteIntText.h | 2 +- dbms/src/IO/readFloatText.h | 15 +-- dbms/src/Interpreters/convertFieldToType.cpp | 28 ++++-- .../00700_decimal_bounds.reference | 21 +++++ .../0_stateless/00700_decimal_bounds.sql | 93 +++++++++++++++++++ 7 files changed, 173 insertions(+), 17 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00700_decimal_bounds.reference create mode 100644 dbms/tests/queries/0_stateless/00700_decimal_bounds.sql diff --git a/dbms/src/DataTypes/DataTypesDecimal.cpp b/dbms/src/DataTypes/DataTypesDecimal.cpp index 2eccf459ec8..854918d54d6 100644 --- a/dbms/src/DataTypes/DataTypesDecimal.cpp +++ b/dbms/src/DataTypes/DataTypesDecimal.cpp @@ -42,12 +42,22 @@ bool DataTypeDecimal::equals(const IDataType & rhs) const template void DataTypeDecimal::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const { - const T & value = static_cast(column).getData()[row_num]; + T value = static_cast(column).getData()[row_num]; + if (value < 0) + { + value *= -1; + writeChar('-', ostr); /// avoid crop leading minus when whole part is zero + } writeIntText(wholePart(value), ostr); - writeChar('.', ostr); if (scale) - writeIntText(fractionalPart(value), ostr); + { + writeChar('.', ostr); + String str_fractional(scale, '0'); + for (Int32 pos = scale - 1; pos >= 0; --pos, value /= 10) + str_fractional[pos] += value % 10; + ostr.write(str_fractional.data(), scale); + } } @@ -62,6 +72,18 @@ void DataTypeDecimal::deserializeText(IColumn & column, ReadBuffer & istr, co } +template +T DataTypeDecimal::parseFromString(const String & str) const +{ + ReadBufferFromMemory buf(str.data(), str.size()); + T x; + UInt32 unread_scale = scale; + readDecimalText(buf, x, precision, unread_scale, true); + x *= getScaleMultiplier(unread_scale); + return x; +} + + template void DataTypeDecimal::serializeBinary(const Field & field, WriteBuffer & ostr) const { @@ -166,6 +188,7 @@ static DataTypePtr create(const ASTPtr & arguments) void registerDataTypeDecimal(DataTypeFactory & factory) { factory.registerDataType("Decimal", create, DataTypeFactory::CaseInsensitive); + factory.registerAlias("DEC", "Decimal", DataTypeFactory::CaseInsensitive); } diff --git a/dbms/src/DataTypes/DataTypesDecimal.h b/dbms/src/DataTypes/DataTypesDecimal.h index b89bf9cc2b2..672ec06d740 100644 --- a/dbms/src/DataTypes/DataTypesDecimal.h +++ b/dbms/src/DataTypes/DataTypesDecimal.h @@ -198,6 +198,8 @@ public: return DataTypeDecimal::getScaleMultiplier(scale_delta); } + T parseFromString(const String & str) const; + private: const UInt32 precision; const UInt32 scale; /// TODO: should we support scales out of [0, precision]? diff --git a/dbms/src/IO/WriteIntText.h b/dbms/src/IO/WriteIntText.h index 265eb1e3984..c961181882c 100644 --- a/dbms/src/IO/WriteIntText.h +++ b/dbms/src/IO/WriteIntText.h @@ -188,7 +188,7 @@ namespace detail #if 1 inline void writeSIntText(__int128 x, WriteBuffer & buf) { - if (unlikely((x - 1) == 0)) + if (unlikely((x - 1) > x)) { buf.write("-170141183460469231731687303715884105728", 40); return; diff --git a/dbms/src/IO/readFloatText.h b/dbms/src/IO/readFloatText.h index 10beb9c32ee..1aaa2a7d5c0 100644 --- a/dbms/src/IO/readFloatText.h +++ b/dbms/src/IO/readFloatText.h @@ -96,6 +96,7 @@ namespace DB namespace ErrorCodes { extern const int CANNOT_PARSE_NUMBER; + extern const int ARGUMENT_OUT_OF_BOUND; } @@ -553,7 +554,7 @@ ReturnType readFloatTextSimpleImpl(T & x, ReadBuffer & buf) template -inline void readDecimalText(ReadBuffer & buf, T & x, unsigned int precision, unsigned int & scale) +inline void readDecimalText(ReadBuffer & buf, T & x, unsigned int precision, unsigned int & scale, bool digits_only = false) { x = 0; int sign = 1; @@ -598,28 +599,30 @@ inline void readDecimalText(ReadBuffer & buf, T & x, unsigned int precision, uns case '9': leading_zeores = false; if (trailing_zeores || precision == 0) - throw Exception("Cannot read decimal value", ErrorCodes::CANNOT_PARSE_NUMBER); + throw Exception("Cannot read decimal value", ErrorCodes::ARGUMENT_OUT_OF_BOUND); [[fallthrough]]; case '0': { /// ignore leading and trailing zeroes if (likely(!leading_zeores && !trailing_zeores)) { - if (precision == 0 || precision < scale) - throw Exception("Cannot read decimal value", ErrorCodes::CANNOT_PARSE_NUMBER); + if (precision == 0 || precision < scale || ((precision == scale) && !after_point)) + throw Exception("Cannot read decimal value", ErrorCodes::ARGUMENT_OUT_OF_BOUND); --precision; x = x * 10 + (byte - '0'); } - if (after_point) + if (after_point && scale) { --scale; - if (scale == 0) + if (!scale) trailing_zeores = true; } break; } default: + if (digits_only) + throw Exception("Unexpected symbol while reading decimal", ErrorCodes::CANNOT_PARSE_NUMBER); x *= sign; return; } diff --git a/dbms/src/Interpreters/convertFieldToType.cpp b/dbms/src/Interpreters/convertFieldToType.cpp index 268e05df988..3a188140f6f 100644 --- a/dbms/src/Interpreters/convertFieldToType.cpp +++ b/dbms/src/Interpreters/convertFieldToType.cpp @@ -74,7 +74,7 @@ static Field convertNumericType(const Field & from, const IDataType & type) template -static Field convertIntToDecimalTypeImpl(const Field & from, const To & type) +static Field convertIntToDecimalType(const Field & from, const To & type) { using FieldType = typename To::FieldType; @@ -86,13 +86,27 @@ static Field convertIntToDecimalTypeImpl(const Field & from, const To & type) return Field(FieldType(scaled_value)); } + +template +static Field convertStringToDecimalType(const Field & from, const DataTypeDecimal & type) +{ + using FieldType = typename DataTypeDecimal::FieldType; + + const String & str_value = from.get(); + T value = type.parseFromString(str_value); + return Field(FieldType(value)); +} + + template -static Field convertIntToDecimalType(const Field & from, const To & type) +static Field convertDecimalType(const Field & from, const To & type) { if (from.getType() == Field::Types::UInt64) - return convertIntToDecimalTypeImpl(from, type); + return convertIntToDecimalType(from, type); if (from.getType() == Field::Types::Int64) - return convertIntToDecimalTypeImpl(from, type); + return convertIntToDecimalType(from, type); + if (from.getType() == Field::Types::String) + return convertStringToDecimalType(from, type); throw Exception("Type mismatch in IN or VALUES section. Expected: " + type.getName() + ". Got: " + Field::Types::toString(from.getType()), ErrorCodes::TYPE_MISMATCH); @@ -150,9 +164,9 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type) if (typeid_cast(&type)) return convertNumericType(src, type); if (typeid_cast(&type)) return convertNumericType(src, type); if (typeid_cast(&type)) return convertNumericType(src, type); - if (auto * ptype = typeid_cast *>(&type)) return convertIntToDecimalType(src, *ptype); - if (auto * ptype = typeid_cast *>(&type)) return convertIntToDecimalType(src, *ptype); - if (auto * ptype = typeid_cast *>(&type)) return convertIntToDecimalType(src, *ptype); + if (auto * ptype = typeid_cast *>(&type)) return convertDecimalType(src, *ptype); + if (auto * ptype = typeid_cast *>(&type)) return convertDecimalType(src, *ptype); + if (auto * ptype = typeid_cast *>(&type)) return convertDecimalType(src, *ptype); const bool is_date = typeid_cast(&type); bool is_datetime = false; diff --git a/dbms/tests/queries/0_stateless/00700_decimal_bounds.reference b/dbms/tests/queries/0_stateless/00700_decimal_bounds.reference new file mode 100644 index 00000000000..a7951527a5b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00700_decimal_bounds.reference @@ -0,0 +1,21 @@ +999999999 999999999999999999 0 0.999999999 0.000000000000000000 0.00000000000000000000000000000000000000 9999.99999 0.000000000 0.000000000000000000 0 +1 1 1 0.000000001 0.000000000000000000 0.00000000000000000000000000000000000000 0.00001 0.000000001 0.000000000000000000 1 +-999999999 -999999999999999999 0 -0.999999999 0.000000000000000000 0.00000000000000000000000000000000000000 -9999.99999 0.000000000 0.000000000000000000 0 +-1 -1 -1 -0.000000001 0.000000000000000000 0.00000000000000000000000000000000000000 -0.00001 -0.000000001 0.000000000000000000 -1 +0 0 99999999999999999999999999999999999999 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 +0 0 0 0.000000000 0.000000000000000001 0.00000000000000000000000000000000000001 0.00000 0.000000000 0.000000000000000000 0 +0 0 -99999999999999999999999999999999999999 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 +0 0 0 0.000000000 -0.000000000000000001 -0.00000000000000000000000000000000000001 0.00000 0.000000000 0.000000000000000000 0 +0 0 0 0.000000000 0.000000000000000000 0.99999999999999999999999999999999999999 0.00000 0.000000000 0.000000000000000000 0 +0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000001 0 +0 0 0 0.000000000 0.000000000000000000 -0.99999999999999999999999999999999999999 0.00000 0.000000000 0.000000000000000000 0 +0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 -0.000000000000000001 0 +0 0 0 0.000000000 0.999999999999999999 0.00000000000000000000000000000000000000 0.00000 999999999.999999999 0.000000000000000000 0 +0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 +0 0 0 0.000000000 -0.999999999999999999 0.00000000000000000000000000000000000000 0.00000 -999999999.999999999 0.000000000000000000 0 +0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 +0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 99999999999999999999.999999999999999999 0 +0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 +0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 -99999999999999999999.999999999999999999 0 +0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 +42 42 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.99999 0.000000000 0.000000000000000000 0 diff --git a/dbms/tests/queries/0_stateless/00700_decimal_bounds.sql b/dbms/tests/queries/0_stateless/00700_decimal_bounds.sql new file mode 100644 index 00000000000..657f68762a3 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00700_decimal_bounds.sql @@ -0,0 +1,93 @@ +CREATE DATABASE IF NOT EXISTS test; +DROP TABLE IF EXISTS test.decimal; + +CREATE TABLE IF NOT EXISTS test.decimal (x DECIMAL(10, -2)) ENGINE = Memory; -- { serverError 69 } +CREATE TABLE IF NOT EXISTS test.decimal (x DECIMAL(10, 15)) ENGINE = Memory; -- { serverError 69 } +CREATE TABLE IF NOT EXISTS test.decimal (x DECIMAL(0, 0)) ENGINE = Memory; -- { serverError 69 } + +CREATE TABLE IF NOT EXISTS test.decimal +( + a DECIMAL(9,0), + b DECIMAL(18,0), + c DECIMAL(38,0), + d DECIMAL(9, 9), + e DECIMAL(18, 18), + f DECIMAL(38, 38), + g Decimal(9, 5), + h decimal(18, 9), + i deciMAL(38, 18), + j DECIMAL(1,0) +) ENGINE = Memory; + +INSERT INTO test.decimal (a) VALUES (1000000000); -- { clientError 69 } +INSERT INTO test.decimal (a) VALUES (-1000000000); -- { clientError 69 } +INSERT INTO test.decimal (b) VALUES (1000000000000000000); -- { clientError 69 } +INSERT INTO test.decimal (b) VALUES (-1000000000000000000); -- { clientError 69 } +INSERT INTO test.decimal (c) VALUES (100000000000000000000000000000000000000); -- { clientError 69 } +INSERT INTO test.decimal (c) VALUES (-100000000000000000000000000000000000000); -- { clientError 69 } +INSERT INTO test.decimal (d) VALUES (1); -- { clientError 69 } +INSERT INTO test.decimal (d) VALUES (-1); -- { clientError 69 } +INSERT INTO test.decimal (e) VALUES (1000000000000000000); -- { clientError 69 } +INSERT INTO test.decimal (e) VALUES (-1000000000000000000); -- { clientError 69 } +INSERT INTO test.decimal (f) VALUES (1); -- { clientError 69 } +INSERT INTO test.decimal (f) VALUES (-1); -- { clientError 69 } +INSERT INTO test.decimal (g) VALUES (10000); -- { clientError 69 } +INSERT INTO test.decimal (g) VALUES (-10000); -- { clientError 69 } +INSERT INTO test.decimal (h) VALUES (1000000000); -- { clientError 69 } +INSERT INTO test.decimal (h) VALUES (-1000000000); -- { clientError 69 } +INSERT INTO test.decimal (i) VALUES (100000000000000000000); -- { clientError 69 } +INSERT INTO test.decimal (i) VALUES (-100000000000000000000); -- { clientError 69 } +INSERT INTO test.decimal (j) VALUES (10); -- { clientError 69 } +INSERT INTO test.decimal (j) VALUES (-10); -- { clientError 69 } + +INSERT INTO test.decimal (a) VALUES (0.1); -- { clientError 69 } +INSERT INTO test.decimal (a) VALUES (-0.1); -- { clientError 69 } +INSERT INTO test.decimal (b) VALUES (0.1); -- { clientError 69 } +INSERT INTO test.decimal (b) VALUES (-0.1); -- { clientError 69 } +INSERT INTO test.decimal (c) VALUES (0.1); -- { clientError 69 } +INSERT INTO test.decimal (c) VALUES (-0.1); -- { clientError 69 } +INSERT INTO test.decimal (d) VALUES (0.0000000001); -- { clientError 69 } +INSERT INTO test.decimal (d) VALUES (-0.0000000001); -- { clientError 69 } +INSERT INTO test.decimal (e) VALUES (0.0000000000000000001); -- { clientError 69 } +INSERT INTO test.decimal (e) VALUES (-0.0000000000000000001); -- { clientError 69 } +INSERT INTO test.decimal (f) VALUES (0.000000000000000000000000000000000000001); -- { clientError 69 } +INSERT INTO test.decimal (f) VALUES (-0.000000000000000000000000000000000000001); -- { clientError 69 } +INSERT INTO test.decimal (g) VALUES (0.000001); -- { clientError 69 } +INSERT INTO test.decimal (g) VALUES (-0.000001); -- { clientError 69 } +INSERT INTO test.decimal (h) VALUES (0.0000000001); -- { clientError 69 } +INSERT INTO test.decimal (h) VALUES (-0.0000000001); -- { clientError 69 } +INSERT INTO test.decimal (i) VALUES (0.0000000000000000001); -- { clientError 69 } +INSERT INTO test.decimal (i) VALUES (-0.0000000000000000001); -- { clientError 69 } +INSERT INTO test.decimal (j) VALUES (0.1); -- { clientError 69 } +INSERT INTO test.decimal (j) VALUES (-0.1); -- { clientError 69 } + +INSERT INTO test.decimal (a, b, d, g) VALUES (999999999, 999999999999999999, 0.999999999, 9999.99999); +INSERT INTO test.decimal (a, b, d, g) VALUES (-999999999, -999999999999999999, -0.999999999, -9999.99999); +INSERT INTO test.decimal (c) VALUES (99999999999999999999999999999999999999); +INSERT INTO test.decimal (c) VALUES (-99999999999999999999999999999999999999); +INSERT INTO test.decimal (f) VALUES (0.99999999999999999999999999999999999999); +INSERT INTO test.decimal (f) VALUES (-0.99999999999999999999999999999999999999); +INSERT INTO test.decimal (e, h) VALUES (0.999999999999999999, 999999999.999999999); +INSERT INTO test.decimal (e, h) VALUES (-0.999999999999999999, -999999999.999999999); +INSERT INTO test.decimal (i) VALUES (99999999999999999999.999999999999999999); +INSERT INTO test.decimal (i) VALUES (-99999999999999999999.999999999999999999); + +INSERT INTO test.decimal (a, b, c, d, g, j, h) VALUES (1, 1, 1, 0.000000001, 0.00001, 1, 0.000000001); +INSERT INTO test.decimal (a, b, c, d, g, j, h) VALUES (-1, -1, -1, -0.000000001, -0.00001, -1, -0.000000001); +INSERT INTO test.decimal (e, f) VALUES (0.000000000000000001, 0.00000000000000000000000000000000000001); +INSERT INTO test.decimal (e, f) VALUES (-0.000000000000000001, -0.00000000000000000000000000000000000001); +INSERT INTO test.decimal (i) VALUES (0.000000000000000001); +INSERT INTO test.decimal (i) VALUES (-0.000000000000000001); + +INSERT INTO test.decimal (a, b, c, d, e, f, g, h, i, j) VALUES (0, 0, 0, 0, 0, 0, 0, 0, 0, 0); +INSERT INTO test.decimal (a, b, c, d, e, f, g, h, i, j) VALUES (-0, -0, -0, -0, -0, -0, -0, -0, -0, -0); +INSERT INTO test.decimal (a, b, c, d, e, f, g, h, i, j) VALUES (0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0); +INSERT INTO test.decimal (a, b, c, d, e, f, g, h, i, j) VALUES (-0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0); + +INSERT INTO test.decimal (a, b, g) VALUES ('42.00000', 42.0000000000000000000000000000000, '0.999990'); +INSERT INTO test.decimal (a) VALUES ('-9x'); -- { clientError 72 } +INSERT INTO test.decimal (a) VALUES ('0x1'); -- { clientError 72 } +INSERT INTO test.decimal (a) VALUES ('1e2'); -- { clientError 72 } + +SELECT * FROM test.decimal; +--DROP TABLE IF EXISTS test.defaults;