Merge pull request #14151 from 4ertus2/some

Add functions: isDecimalOverflow(), countDigits()
This commit is contained in:
alexey-milovidov 2020-08-28 22:01:54 +03:00 committed by GitHub
commit c6f55bbe6a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 472 additions and 0 deletions

View File

@ -77,6 +77,19 @@ inline UInt32 getDecimalScale(const IDataType & data_type, UInt32 default_value
return default_value;
}
inline UInt32 getDecimalPrecision(const IDataType & data_type)
{
if (auto * decimal_type = checkDecimal<Decimal32>(data_type))
return decimal_type->getPrecision();
if (auto * decimal_type = checkDecimal<Decimal64>(data_type))
return decimal_type->getPrecision();
if (auto * decimal_type = checkDecimal<Decimal128>(data_type))
return decimal_type->getPrecision();
if (auto * decimal_type = checkDecimal<Decimal256>(data_type))
return decimal_type->getPrecision();
return 0;
}
template <typename T>
inline UInt32 getDecimalScale(const DataTypeDecimal<T> & data_type)
{

View File

@ -0,0 +1,145 @@
#include <Functions/IFunctionImpl.h>
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionHelpers.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypesDecimal.h>
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnDecimal.h>
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int ILLEGAL_COLUMN;
}
/// Returns number of decimal digits you need to represent the value.
/// For Decimal values takes in account their scales: calculates result over underlying int type which is (value * scale).
/// countDigits(42) = 2, countDigits(42.000) = 5, countDigits(0.04200) = 4.
/// I.e. you may check decimal overflow for Decimal64 with 'countDecimal(x) > 18'. It's a slow variant of isDecimalOverflow().
class FunctionCountDigits : public IFunction
{
public:
static constexpr auto name = "countDigits";
static FunctionPtr create(const Context &)
{
return std::make_shared<FunctionCountDigits>();
}
String getName() const override { return name; }
bool useDefaultImplementationForConstants() const override { return true; }
size_t getNumberOfArguments() const override { return 1; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
WhichDataType which_first(arguments[0]->getTypeId());
if (!which_first.isInt() && !which_first.isUInt() && !which_first.isDecimal())
throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return std::make_shared<DataTypeUInt8>(); /// Up to 255 decimal digits.
}
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result_pos, size_t input_rows_count) const override
{
const auto & src_column = block.getByPosition(arguments[0]);
if (!src_column.column)
throw Exception("Illegal column while execute function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
auto result_column = ColumnUInt8::create();
auto call = [&](const auto & types) -> bool
{
using Types = std::decay_t<decltype(types)>;
using Type = typename Types::RightType;
using ColVecType = std::conditional_t<IsDecimalNumber<Type>, ColumnDecimal<Type>, ColumnVector<Type>>;
if (const ColVecType * col_vec = checkAndGetColumn<ColVecType>(src_column.column.get()))
{
execute<Type>(*col_vec, *result_column, input_rows_count);
return true;
}
throw Exception("Illegal column while execute function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
};
TypeIndex dec_type_idx = src_column.type->getTypeId();
if (!callOnBasicType<void, true, false, true, false>(dec_type_idx, call))
throw Exception("Wrong call for " + getName() + " with " + src_column.type->getName(),
ErrorCodes::ILLEGAL_COLUMN);
block.getByPosition(result_pos).column = std::move(result_column);
}
private:
template <typename T, typename ColVecType>
static void execute(const ColVecType & col, ColumnUInt8 & result_column, size_t rows_count)
{
using NativeT = typename NativeType<T>::Type;
const auto & src_data = col.getData();
auto & dst_data = result_column.getData();
dst_data.resize(rows_count);
for (size_t i = 0; i < rows_count; ++i)
{
if constexpr (IsDecimalNumber<T>)
dst_data[i] = digits<NativeT>(src_data[i].value);
else
dst_data[i] = digits<NativeT>(src_data[i]);
}
}
template <typename T>
static UInt32 digits(T value)
{
static_assert(!IsDecimalNumber<T>);
using DivT = std::conditional_t<is_signed_v<T>, Int32, UInt32>;
UInt32 res = 0;
T tmp;
if constexpr (sizeof(T) > sizeof(Int32))
{
static constexpr const DivT e9 = 1000000000;
tmp = value / e9;
while (tmp != 0)
{
value = tmp;
tmp /= e9;
res += 9;
}
}
static constexpr const DivT e3 = 1000;
tmp = value / e3;
while (tmp != 0)
{
value = tmp;
tmp /= e3;
res += 3;
}
while (value != 0)
{
value /= 10;
++res;
}
return res;
}
};
void registerFunctionCountDigits(FunctionFactory & factory)
{
factory.registerFunction<FunctionCountDigits>();
}
}

View File

@ -0,0 +1,151 @@
#include <Functions/IFunctionImpl.h>
#include <Functions/FunctionFactory.h>
#include <Functions/FunctionHelpers.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypesDecimal.h>
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnDecimal.h>
#include <Columns/ColumnConst.h>
#include <Common/intExp.h>
namespace DB
{
namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int ILLEGAL_COLUMN;
}
/// Returns 1 if and Decimal value has more digits then it's Precision allow, 0 otherwise.
/// Precision could be set as second argument or omitted. If ommited function uses Decimal presicion of the first argument.
class FunctionIsDecimalOverflow : public IFunction
{
public:
static constexpr auto name = "isDecimalOverflow";
static FunctionPtr create(const Context &)
{
return std::make_shared<FunctionIsDecimalOverflow>();
}
String getName() const override { return name; }
bool isVariadic() const override { return true; }
size_t getNumberOfArguments() const override { return 0; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (arguments.empty() || arguments.size() > 2)
throw Exception("Number of arguments for function " + getName() + " doesn't match: passed " +
toString(arguments.size()) + ", should be 1 or 2.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
WhichDataType which_first(arguments[0]->getTypeId());
if (!which_first.isDecimal())
throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
if (arguments.size() == 2)
{
WhichDataType which_second(arguments[1]->getTypeId());
if (!which_second.isUInt8())
throw Exception("Illegal type " + arguments[1]->getName() + " of argument of function " + getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
return std::make_shared<DataTypeUInt8>();
}
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result_pos, size_t input_rows_count) const override
{
const auto & src_column = block.getByPosition(arguments[0]);
if (!src_column.column)
throw Exception("Illegal column while execute function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
UInt32 precision = 0;
if (arguments.size() == 2)
{
const auto & precision_column = block.getByPosition(arguments[1]);
if (!precision_column.column)
throw Exception("Illegal column while execute function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
const ColumnConst * const_column = checkAndGetColumnConst<ColumnUInt8>(precision_column.column.get());
if (!const_column)
throw Exception("Second argument for function " + getName() + " must be constant UInt8: precision.",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
precision = const_column->getValue<UInt8>();
}
else
precision = getDecimalPrecision(*src_column.type);
auto result_column = ColumnUInt8::create();
auto call = [&](const auto & types) -> bool
{
using Types = std::decay_t<decltype(types)>;
using Type = typename Types::RightType;
using ColVecType = ColumnDecimal<Type>;
if (const ColumnConst * const_column = checkAndGetColumnConst<ColVecType>(src_column.column.get()))
{
Type const_decimal = checkAndGetColumn<ColVecType>(const_column->getDataColumnPtr().get())->getData()[0];
UInt8 res_value = outOfDigits<Type>(const_decimal, precision);
result_column->getData().resize_fill(input_rows_count, res_value);
return true;
}
else if (const ColVecType * col_vec = checkAndGetColumn<ColVecType>(src_column.column.get()))
{
execute<Type>(*col_vec, *result_column, input_rows_count, precision);
return true;
}
throw Exception("Illegal column while execute function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
};
TypeIndex dec_type_idx = src_column.type->getTypeId();
if (!callOnBasicType<void, false, false, true, false>(dec_type_idx, call))
throw Exception("Wrong call for " + getName() + " with " + src_column.type->getName(),
ErrorCodes::ILLEGAL_COLUMN);
block.getByPosition(result_pos).column = std::move(result_column);
}
private:
template <typename T>
static void execute(const ColumnDecimal<T> & col, ColumnUInt8 & result_column, size_t rows_count, UInt32 precision)
{
const auto & src_data = col.getData();
auto & dst_data = result_column.getData();
dst_data.resize(rows_count);
for (size_t i = 0; i < rows_count; ++i)
dst_data[i] = outOfDigits<T>(src_data[i], precision);
}
template <typename T>
static bool outOfDigits(T dec, UInt32 precision)
{
static_assert(IsDecimalNumber<T>);
using NativeT = typename T::NativeType;
if (precision > DecimalUtils::maxPrecision<T>())
return false;
NativeT pow10 = intExp10OfSize<NativeT>(precision);
if (dec.value < 0)
return dec.value <= -pow10;
return dec.value >= pow10;
}
};
void registerFunctionIsDecimalOverflow(FunctionFactory & factory)
{
factory.registerFunction<FunctionIsDecimalOverflow>();
}
}

View File

@ -59,6 +59,8 @@ void registerFunctionGetMacro(FunctionFactory &);
void registerFunctionGetScalar(FunctionFactory &);
void registerFunctionGetSetting(FunctionFactory &);
void registerFunctionIsConstant(FunctionFactory &);
void registerFunctionIsDecimalOverflow(FunctionFactory &);
void registerFunctionCountDigits(FunctionFactory &);
void registerFunctionGlobalVariable(FunctionFactory &);
void registerFunctionHasThreadFuzzer(FunctionFactory &);
void registerFunctionInitializeAggregation(FunctionFactory &);
@ -121,6 +123,8 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory)
registerFunctionGetScalar(factory);
registerFunctionGetSetting(factory);
registerFunctionIsConstant(factory);
registerFunctionIsDecimalOverflow(factory);
registerFunctionCountDigits(factory);
registerFunctionGlobalVariable(factory);
registerFunctionHasThreadFuzzer(factory);
registerFunctionInitializeAggregation(factory);

View File

@ -132,6 +132,7 @@ SRCS(
concat.cpp
convertCharset.cpp
cos.cpp
countDigits.cpp
CRC.cpp
currentDatabase.cpp
currentUser.cpp
@ -243,6 +244,7 @@ SRCS(
intExp10.cpp
intExp2.cpp
isConstant.cpp
isDecimalOverflow.cpp
isFinite.cpp
isInfinite.cpp
isNaN.cpp

View File

@ -0,0 +1,7 @@
0 2 2 0 2 3 0 2 4
2 3 4
10 10 19 19 39 39
2 2 2 2 2 2 2 2 2 2 2 2
0 0 0 0 0 0 0 0 0 0 0 0
3 3 3 5 5 5 10 10 10 19 19 20
2 3 4 5 6 7

View File

@ -0,0 +1,30 @@
SELECT countDigits(toDecimal32(0, 0)), countDigits(toDecimal32(42, 0)), countDigits(toDecimal32(4.2, 1)),
countDigits(toDecimal64(0, 0)), countDigits(toDecimal64(42, 0)), countDigits(toDecimal64(4.2, 2)),
countDigits(toDecimal128(0, 0)), countDigits(toDecimal128(42, 0)), countDigits(toDecimal128(4.2, 3));
SELECT countDigits(materialize(toDecimal32(4.2, 1))),
countDigits(materialize(toDecimal64(4.2, 2))),
countDigits(materialize(toDecimal128(4.2, 3)));
SELECT countDigits(toDecimal32(1, 9)), countDigits(toDecimal32(-1, 9)),
countDigits(toDecimal64(1, 18)), countDigits(toDecimal64(-1, 18)),
countDigits(toDecimal128(1, 38)), countDigits(toDecimal128(-1, 38));
SELECT countDigits(toInt8(42)), countDigits(toInt8(-42)), countDigits(toUInt8(42)),
countDigits(toInt16(42)), countDigits(toInt16(-42)), countDigits(toUInt16(42)),
countDigits(toInt32(42)), countDigits(toInt32(-42)), countDigits(toUInt32(42)),
countDigits(toInt64(42)), countDigits(toInt64(-42)), countDigits(toUInt64(42));
SELECT countDigits(toInt8(0)), countDigits(toInt8(0)), countDigits(toUInt8(0)),
countDigits(toInt16(0)), countDigits(toInt16(0)), countDigits(toUInt16(0)),
countDigits(toInt32(0)), countDigits(toInt32(0)), countDigits(toUInt32(0)),
countDigits(toInt64(0)), countDigits(toInt64(0)), countDigits(toUInt64(0));
SELECT countDigits(toInt8(127)), countDigits(toInt8(-128)), countDigits(toUInt8(255)),
countDigits(toInt16(32767)), countDigits(toInt16(-32768)), countDigits(toUInt16(65535)),
countDigits(toInt32(2147483647)), countDigits(toInt32(-2147483648)), countDigits(toUInt32(4294967295)),
countDigits(toInt64(9223372036854775807)), countDigits(toInt64(-9223372036854775808)), countDigits(toUInt64(18446744073709551615));
SELECT countDigits(toNullable(toDecimal32(4.2, 1))), countDigits(materialize(toNullable(toDecimal32(4.2, 2)))),
countDigits(toNullable(toDecimal64(4.2, 3))), countDigits(materialize(toNullable(toDecimal64(4.2, 4)))),
countDigits(toNullable(toDecimal128(4.2, 5))), countDigits(materialize(toNullable(toDecimal128(4.2, 6))));

View File

@ -0,0 +1,20 @@
0 0 0
1 1 1 1
0 0 0 0
1 1 1 1 1 1
1 1 1 1
0 0 0 0
1 1 1 1 1 1
1 1 1 1
0 0 0 0
1 1 1 1 1 1
1 1 1 1
0 0 0 0
1 1 1 1 1 1
1 1 1 1
0 0 0 0
1 1 1 1 1 1
1 1 1 1
0 0 0 0
1 1 1 1 1 1
1 0 1 0 1 0

View File

@ -0,0 +1,100 @@
SELECT isDecimalOverflow(toDecimal32(0, 0), 0),
isDecimalOverflow(toDecimal64(0, 0), 0),
isDecimalOverflow(toDecimal128(0, 0), 0);
SELECT isDecimalOverflow(toDecimal32(1000000000, 0), 9),
isDecimalOverflow(toDecimal32(1000000000, 0)),
isDecimalOverflow(toDecimal32(-1000000000, 0), 9),
isDecimalOverflow(toDecimal32(-1000000000, 0));
SELECT isDecimalOverflow(toDecimal32(999999999, 0), 9),
isDecimalOverflow(toDecimal32(999999999, 0)),
isDecimalOverflow(toDecimal32(-999999999, 0), 9),
isDecimalOverflow(toDecimal32(-999999999, 0));
SELECT isDecimalOverflow(toDecimal32(999999999, 0), 8),
isDecimalOverflow(toDecimal32(10, 0), 1),
isDecimalOverflow(toDecimal32(1, 0), 0),
isDecimalOverflow(toDecimal32(-999999999, 0), 8),
isDecimalOverflow(toDecimal32(-10, 0), 1),
isDecimalOverflow(toDecimal32(-1, 0), 0);
SELECT isDecimalOverflow(materialize(toDecimal32(1000000000, 0)), 9),
isDecimalOverflow(materialize(toDecimal32(1000000000, 0))),
isDecimalOverflow(materialize(toDecimal32(-1000000000, 0)), 9),
isDecimalOverflow(materialize(toDecimal32(-1000000000, 0)));
SELECT isDecimalOverflow(materialize(toDecimal32(999999999, 0)), 9),
isDecimalOverflow(materialize(toDecimal32(999999999, 0))),
isDecimalOverflow(materialize(toDecimal32(-999999999, 0)), 9),
isDecimalOverflow(materialize(toDecimal32(-999999999, 0)));
SELECT isDecimalOverflow(materialize(toDecimal32(999999999, 0)), 8),
isDecimalOverflow(materialize(toDecimal32(10, 0)), 1),
isDecimalOverflow(materialize(toDecimal32(1, 0)), 0),
isDecimalOverflow(materialize(toDecimal32(-999999999, 0)), 8),
isDecimalOverflow(materialize(toDecimal32(-10, 0)), 1),
isDecimalOverflow(materialize(toDecimal32(-1, 0)), 0);
SELECT isDecimalOverflow(toDecimal64(1000000000000000000, 0), 18),
isDecimalOverflow(toDecimal64(1000000000000000000, 0)),
isDecimalOverflow(toDecimal64(-1000000000000000000, 0), 18),
isDecimalOverflow(toDecimal64(-1000000000000000000, 0));
SELECT isDecimalOverflow(toDecimal64(999999999999999999, 0), 18),
isDecimalOverflow(toDecimal64(999999999999999999, 0)),
isDecimalOverflow(toDecimal64(-999999999999999999, 0), 18),
isDecimalOverflow(toDecimal64(-999999999999999999, 0));
SELECT isDecimalOverflow(toDecimal64(999999999999999999, 0), 17),
isDecimalOverflow(toDecimal64(10, 0), 1),
isDecimalOverflow(toDecimal64(1, 0), 0),
isDecimalOverflow(toDecimal64(-999999999999999999, 0), 17),
isDecimalOverflow(toDecimal64(-10, 0), 1),
isDecimalOverflow(toDecimal64(-1, 0), 0);
SELECT isDecimalOverflow(materialize(toDecimal64(1000000000000000000, 0)), 18),
isDecimalOverflow(materialize(toDecimal64(1000000000000000000, 0))),
isDecimalOverflow(materialize(toDecimal64(-1000000000000000000, 0)), 18),
isDecimalOverflow(materialize(toDecimal64(-1000000000000000000, 0)));
SELECT isDecimalOverflow(materialize(toDecimal64(999999999999999999, 0)), 18),
isDecimalOverflow(materialize(toDecimal64(999999999999999999, 0))),
isDecimalOverflow(materialize(toDecimal64(-999999999999999999, 0)), 18),
isDecimalOverflow(materialize(toDecimal64(-999999999999999999, 0)));
SELECT isDecimalOverflow(materialize(toDecimal64(999999999999999999, 0)), 17),
isDecimalOverflow(materialize(toDecimal64(10, 0)), 1),
isDecimalOverflow(materialize(toDecimal64(1, 0)), 0),
isDecimalOverflow(materialize(toDecimal64(-999999999999999999, 0)), 17),
isDecimalOverflow(materialize(toDecimal64(-10, 0)), 1),
isDecimalOverflow(materialize(toDecimal64(-1, 0)), 0);
SELECT isDecimalOverflow(toDecimal128('99999999999999999999999999999999999999', 0) + 1, 38),
isDecimalOverflow(toDecimal128('99999999999999999999999999999999999999', 0) + 1),
isDecimalOverflow(toDecimal128('-99999999999999999999999999999999999999', 0) - 1, 38),
isDecimalOverflow(toDecimal128('-99999999999999999999999999999999999999', 0) - 1);
SELECT isDecimalOverflow(toDecimal128('99999999999999999999999999999999999999', 0), 38),
isDecimalOverflow(toDecimal128('99999999999999999999999999999999999999', 0)),
isDecimalOverflow(toDecimal128('-99999999999999999999999999999999999999', 0), 38),
isDecimalOverflow(toDecimal128('-99999999999999999999999999999999999999', 0));
SELECT isDecimalOverflow(toDecimal128('99999999999999999999999999999999999999', 0), 37),
isDecimalOverflow(toDecimal128('10', 0), 1),
isDecimalOverflow(toDecimal128('1', 0), 0),
isDecimalOverflow(toDecimal128('-99999999999999999999999999999999999999', 0), 37),
isDecimalOverflow(toDecimal128('-10', 0), 1),
isDecimalOverflow(toDecimal128('-1', 0), 0);
SELECT isDecimalOverflow(materialize(toDecimal128('99999999999999999999999999999999999999', 0)) + 1, 38),
isDecimalOverflow(materialize(toDecimal128('99999999999999999999999999999999999999', 0)) + 1),
isDecimalOverflow(materialize(toDecimal128('-99999999999999999999999999999999999999', 0)) - 1, 38),
isDecimalOverflow(materialize(toDecimal128('-99999999999999999999999999999999999999', 0)) - 1);
SELECT isDecimalOverflow(materialize(toDecimal128('99999999999999999999999999999999999999', 0)), 38),
isDecimalOverflow(materialize(toDecimal128('99999999999999999999999999999999999999', 0))),
isDecimalOverflow(materialize(toDecimal128('-99999999999999999999999999999999999999', 0)), 38),
isDecimalOverflow(materialize(toDecimal128('-99999999999999999999999999999999999999', 0)));
SELECT isDecimalOverflow(materialize(toDecimal128('99999999999999999999999999999999999999', 0)), 37),
isDecimalOverflow(materialize(toDecimal128('10', 0)), 1),
isDecimalOverflow(materialize(toDecimal128('1', 0)), 0),
isDecimalOverflow(materialize(toDecimal128('-99999999999999999999999999999999999999', 0)), 37),
isDecimalOverflow(materialize(toDecimal128('-10', 0)), 1),
isDecimalOverflow(materialize(toDecimal128('-1', 0)), 0);
SELECT isDecimalOverflow(toNullable(toDecimal32(42, 0)), 1),
isDecimalOverflow(materialize(toNullable(toDecimal32(42, 0))), 2),
isDecimalOverflow(toNullable(toDecimal64(42, 0)), 1),
isDecimalOverflow(materialize(toNullable(toDecimal64(42, 0))), 2),
isDecimalOverflow(toNullable(toDecimal128(42, 0)), 1),
isDecimalOverflow(materialize(toNullable(toDecimal128(42, 0))), 2);