#pragma once #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if USE_EMBEDDED_COMPILER #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #include // Y_IGNORE #pragma GCC diagnostic pop #endif namespace DB { namespace ErrorCodes { extern const int ILLEGAL_DIVISION; extern const int ILLEGAL_COLUMN; extern const int LOGICAL_ERROR; extern const int TOO_LESS_ARGUMENTS_FOR_FUNCTION; extern const int DECIMAL_OVERFLOW; } /** Arithmetic operations: +, -, *, /, %, * intDiv (integer division), unary minus. * Bitwise operations: |, &, ^, ~. * Etc. */ template struct BinaryOperationImplBase { using ResultType = ResultType_; static void NO_INLINE vector_vector(const PaddedPODArray & a, const PaddedPODArray & b, PaddedPODArray & c) { size_t size = a.size(); for (size_t i = 0; i < size; ++i) c[i] = Op::template apply(a[i], b[i]); } static void NO_INLINE vector_constant(const PaddedPODArray & a, B b, PaddedPODArray & c) { size_t size = a.size(); for (size_t i = 0; i < size; ++i) c[i] = Op::template apply(a[i], b); } static void NO_INLINE constant_vector(A a, const PaddedPODArray & b, PaddedPODArray & c) { size_t size = b.size(); for (size_t i = 0; i < size; ++i) c[i] = Op::template apply(a, b[i]); } static ResultType constant_constant(A a, B b) { return Op::template apply(a, b); } }; template struct BinaryOperationImpl : BinaryOperationImplBase { }; template struct UnaryOperationImpl { using ResultType = typename Op::ResultType; using ArrayA = typename ColumnVector::Container; using ArrayC = typename ColumnVector::Container; static void NO_INLINE vector(const ArrayA & a, ArrayC & c) { size_t size = a.size(); for (size_t i = 0; i < size; ++i) c[i] = Op::apply(a[i]); } static void constant(A a, ResultType & c) { c = Op::apply(a); } }; template struct PlusImpl { using ResultType = typename NumberTraits::ResultOfAdditionMultiplication::Type; static const constexpr bool allow_decimal = true; template static inline Result apply(A a, B b) { /// Next everywhere, static_cast - so that there is no wrong result in expressions of the form Int64 c = UInt32(a) * Int32(-1). return static_cast(a) + b; } template static inline bool apply(A a, B b, Result & c) { return common::addOverflow(static_cast(a), b, c); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { return left->getType()->isIntegerTy() ? b.CreateAdd(left, right) : b.CreateFAdd(left, right); } #endif }; template struct MultiplyImpl { using ResultType = typename NumberTraits::ResultOfAdditionMultiplication::Type; static const constexpr bool allow_decimal = true; template static inline Result apply(A a, B b) { return static_cast(a) * b; } template static inline bool apply(A a, B b, Result & c) { return common::mulOverflow(static_cast(a), b, c); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { return left->getType()->isIntegerTy() ? b.CreateMul(left, right) : b.CreateFMul(left, right); } #endif }; template struct MinusImpl { using ResultType = typename NumberTraits::ResultOfSubtraction::Type; static const constexpr bool allow_decimal = true; template static inline Result apply(A a, B b) { return static_cast(a) - b; } template static inline bool apply(A a, B b, Result & c) { return common::subOverflow(static_cast(a), b, c); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { return left->getType()->isIntegerTy() ? b.CreateSub(left, right) : b.CreateFSub(left, right); } #endif }; template struct DivideFloatingImpl { using ResultType = typename NumberTraits::ResultOfFloatingPointDivision::Type; static const constexpr bool allow_decimal = true; template static inline Result apply(A a, B b) { return static_cast(a) / b; } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { if (left->getType()->isIntegerTy()) throw Exception("DivideFloatingImpl expected a floating-point type", ErrorCodes::LOGICAL_ERROR); return b.CreateFDiv(left, right); } #endif }; #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wsign-compare" template inline void throwIfDivisionLeadsToFPE(A a, B b) { /// Is it better to use siglongjmp instead of checks? if (unlikely(b == 0)) throw Exception("Division by zero", ErrorCodes::ILLEGAL_DIVISION); /// http://avva.livejournal.com/2548306.html if (unlikely(std::is_signed_v && std::is_signed_v && a == std::numeric_limits::min() && b == -1)) throw Exception("Division of minimal signed number by minus one", ErrorCodes::ILLEGAL_DIVISION); } template inline bool divisionLeadsToFPE(A a, B b) { if (unlikely(b == 0)) return true; if (unlikely(std::is_signed_v && std::is_signed_v && a == std::numeric_limits::min() && b == -1)) return true; return false; } #pragma GCC diagnostic pop template struct DivideIntegralImpl { using ResultType = typename NumberTraits::ResultOfIntegerDivision::Type; template static inline Result apply(A a, B b) { throwIfDivisionLeadsToFPE(a, b); return a / b; } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// don't know how to throw from LLVM IR #endif }; template struct DivideIntegralOrZeroImpl { using ResultType = typename NumberTraits::ResultOfIntegerDivision::Type; template static inline Result apply(A a, B b) { return unlikely(divisionLeadsToFPE(a, b)) ? 0 : a / b; } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// TODO implement the checks #endif }; template struct ModuloImpl { using ResultType = typename NumberTraits::ResultOfModulo::Type; template static inline Result apply(A a, B b) { throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger::Type(a), typename NumberTraits::ToInteger::Type(b)); return typename NumberTraits::ToInteger::Type(a) % typename NumberTraits::ToInteger::Type(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// don't know how to throw from LLVM IR #endif }; template struct BitAndImpl { using ResultType = typename NumberTraits::ResultOfBit::Type; template static inline Result apply(A a, B b) { return static_cast(a) & static_cast(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { if (!left->getType()->isIntegerTy()) throw Exception("BitAndImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); return b.CreateAnd(left, right); } #endif }; template struct BitOrImpl { using ResultType = typename NumberTraits::ResultOfBit::Type; template static inline Result apply(A a, B b) { return static_cast(a) | static_cast(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { if (!left->getType()->isIntegerTy()) throw Exception("BitOrImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); return b.CreateOr(left, right); } #endif }; template struct BitXorImpl { using ResultType = typename NumberTraits::ResultOfBit::Type; template static inline Result apply(A a, B b) { return static_cast(a) ^ static_cast(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { if (!left->getType()->isIntegerTy()) throw Exception("BitXorImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); return b.CreateXor(left, right); } #endif }; template struct BitShiftLeftImpl { using ResultType = typename NumberTraits::ResultOfBit::Type; template static inline Result apply(A a, B b) { return static_cast(a) << static_cast(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { if (!left->getType()->isIntegerTy()) throw Exception("BitShiftLeftImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); return b.CreateShl(left, right); } #endif }; template struct BitShiftRightImpl { using ResultType = typename NumberTraits::ResultOfBit::Type; template static inline Result apply(A a, B b) { return static_cast(a) >> static_cast(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed) { if (!left->getType()->isIntegerTy()) throw Exception("BitShiftRightImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); return is_signed ? b.CreateAShr(left, right) : b.CreateLShr(left, right); } #endif }; template struct BitRotateLeftImpl { using ResultType = typename NumberTraits::ResultOfBit::Type; template static inline Result apply(A a, B b) { return (static_cast(a) << static_cast(b)) | (static_cast(a) >> ((sizeof(Result) * 8) - static_cast(b))); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { if (!left->getType()->isIntegerTy()) throw Exception("BitRotateLeftImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); auto * size = llvm::ConstantInt::get(left->getType(), left->getType()->getPrimitiveSizeInBits()); /// XXX how is this supposed to behave in signed mode? return b.CreateOr(b.CreateShl(left, right), b.CreateLShr(left, b.CreateSub(size, right))); } #endif }; template struct BitRotateRightImpl { using ResultType = typename NumberTraits::ResultOfBit::Type; template static inline Result apply(A a, B b) { return (static_cast(a) >> static_cast(b)) | (static_cast(a) << ((sizeof(Result) * 8) - static_cast(b))); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { if (!left->getType()->isIntegerTy()) throw Exception("BitRotateRightImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); auto * size = llvm::ConstantInt::get(left->getType(), left->getType()->getPrimitiveSizeInBits()); return b.CreateOr(b.CreateLShr(left, right), b.CreateShl(left, b.CreateSub(size, right))); } #endif }; template struct BitTestImpl { using ResultType = UInt8; template static inline Result apply(A a, B b) { return (typename NumberTraits::ToInteger::Type(a) >> typename NumberTraits::ToInteger::Type(b)) & 1; } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// TODO #endif }; template struct LeastBaseImpl { using ResultType = NumberTraits::ResultOfLeast; template static inline Result apply(A a, B b) { /** gcc 4.9.2 successfully vectorizes a loop from this function. */ return static_cast(a) < static_cast(b) ? static_cast(a) : static_cast(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed) { if (!left->getType()->isIntegerTy()) /// XXX minnum is basically fmin(), it may or may not match whatever apply() does return b.CreateMinNum(left, right); return b.CreateSelect(is_signed ? b.CreateICmpSLT(left, right) : b.CreateICmpULT(left, right), left, right); } #endif }; template struct LeastSpecialImpl { using ResultType = std::make_signed_t; template static inline Result apply(A a, B b) { static_assert(std::is_same_v, "ResultType != Result"); return accurate::lessOp(a, b) ? static_cast(a) : static_cast(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// ??? #endif }; template using LeastImpl = std::conditional_t, LeastBaseImpl, LeastSpecialImpl>; template struct GreatestBaseImpl { using ResultType = NumberTraits::ResultOfGreatest; template static inline Result apply(A a, B b) { return static_cast(a) > static_cast(b) ? static_cast(a) : static_cast(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed) { if (!left->getType()->isIntegerTy()) /// XXX maxnum is basically fmax(), it may or may not match whatever apply() does /// XXX CreateMaxNum is broken on LLVM 5.0 and 6.0 (generates minnum instead; fixed in 7) return b.CreateBinaryIntrinsic(llvm::Intrinsic::maxnum, left, right); return b.CreateSelect(is_signed ? b.CreateICmpSGT(left, right) : b.CreateICmpUGT(left, right), left, right); } #endif }; template struct GreatestSpecialImpl { using ResultType = std::make_unsigned_t; template static inline Result apply(A a, B b) { static_assert(std::is_same_v, "ResultType != Result"); return accurate::greaterOp(a, b) ? static_cast(a) : static_cast(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// ??? #endif }; template using GreatestImpl = std::conditional_t, GreatestBaseImpl, GreatestSpecialImpl>; template struct NegateImpl { using ResultType = std::conditional_t, A, typename NumberTraits::ResultOfNegate::Type>; static inline ResultType apply(A a) { return -static_cast(a); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * arg, bool) { return arg->getType()->isIntegerTy() ? b.CreateNeg(arg) : b.CreateFNeg(arg); } #endif }; template struct BitNotImpl { using ResultType = typename NumberTraits::ResultOfBitNot::Type; static inline ResultType apply(A a) { return ~static_cast(a); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * arg, bool) { if (!arg->getType()->isIntegerTy()) throw Exception("BitNotImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); return b.CreateNot(arg); } #endif }; template struct AbsImpl { using ResultType = std::conditional_t, A, typename NumberTraits::ResultOfAbs::Type>; static inline ResultType apply(A a) { if constexpr (IsDecimalNumber) return a < 0 ? A(-a) : a; else if constexpr (std::is_integral_v && std::is_signed_v) return a < 0 ? static_cast(~a) + 1 : a; else if constexpr (std::is_integral_v && std::is_unsigned_v) return static_cast(a); else if constexpr (std::is_floating_point_v) return static_cast(std::abs(a)); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// special type handling, some other time #endif }; template struct GCDImpl { using ResultType = typename NumberTraits::ResultOfAdditionMultiplication::Type; template static inline Result apply(A a, B b) { throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger::Type(a), typename NumberTraits::ToInteger::Type(b)); throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger::Type(b), typename NumberTraits::ToInteger::Type(a)); return boost::integer::gcd( typename NumberTraits::ToInteger::Type(a), typename NumberTraits::ToInteger::Type(b)); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// exceptions (and a non-trivial algorithm) #endif }; template struct LCMImpl { using ResultType = typename NumberTraits::ResultOfAdditionMultiplication::Type; template static inline Result apply(A a, B b) { throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger::Type(a), typename NumberTraits::ToInteger::Type(b)); throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger::Type(b), typename NumberTraits::ToInteger::Type(a)); return boost::integer::lcm( typename NumberTraits::ToInteger::Type(a), typename NumberTraits::ToInteger::Type(b)); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// exceptions (and a non-trivial algorithm) #endif }; template struct IntExp2Impl { using ResultType = UInt64; static inline ResultType apply(A a) { return intExp2(a); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * arg, bool) { if (!arg->getType()->isIntegerTy()) throw Exception("IntExp2Impl expected an integral type", ErrorCodes::LOGICAL_ERROR); return b.CreateShl(llvm::ConstantInt::get(arg->getType(), 1), arg); } #endif }; template struct IntExp10Impl { using ResultType = UInt64; static inline ResultType apply(A a) { return intExp10(a); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// library function #endif }; template struct NativeType { using Type = T; }; template <> struct NativeType { using Type = Int32; }; template <> struct NativeType { using Type = Int64; }; template <> struct NativeType { using Type = Int128; }; /// Binary operations for Decimals need scale args /// +|- scale one of args (which scale factor is not 1). ScaleR = oneof(Scale1, Scale2); /// * no agrs scale. ScaleR = Scale1 + Scale2; /// / first arg scale. ScaleR = Scale1 (scale_a = DecimalType::getScale()). template typename Operation, typename ResultType_> struct DecimalBinaryOperation { using ResultType = ResultType_; using NativeResultType = typename NativeType::Type; using Op = Operation; using ArrayA = typename ColumnVector::Container; using ArrayB = typename ColumnVector::Container; using ArrayC = typename ColumnVector::Container; static constexpr bool is_plus_minus = std::is_same_v, PlusImpl> || std::is_same_v, MinusImpl>; static constexpr bool is_multiply = std::is_same_v, MultiplyImpl>; static constexpr bool is_division = std::is_same_v, DivideFloatingImpl>; static constexpr bool is_compare = std::is_same_v, LeastBaseImpl> || std::is_same_v, GreatestBaseImpl>; static constexpr bool is_plus_minus_compare = is_plus_minus || is_compare; static constexpr bool can_overflow = is_plus_minus || is_multiply; static void NO_INLINE vector_vector(const ArrayA & a, const ArrayB & b, ArrayC & c, ResultType scale_a [[maybe_unused]], ResultType scale_b [[maybe_unused]]) { size_t size = a.size(); if constexpr (is_plus_minus_compare) { if (scale_a != 1) { for (size_t i = 0; i < size; ++i) c[i] = applyScaled(a[i], b[i], scale_a); return; } else if (scale_b != 1) { for (size_t i = 0; i < size; ++i) c[i] = applyScaled(a[i], b[i], scale_b); return; } } else if constexpr (is_division && IsDecimalNumber) { for (size_t i = 0; i < size; ++i) c[i] = applyScaledDiv(a[i], b[i], scale_a); return; } /// default: use it if no return before for (size_t i = 0; i < size; ++i) c[i] = apply(a[i], b[i]); } static void NO_INLINE vector_constant(const ArrayA & a, B b, ArrayC & c, ResultType scale_a [[maybe_unused]], ResultType scale_b [[maybe_unused]]) { size_t size = a.size(); if constexpr (is_plus_minus_compare) { if (scale_a != 1) { for (size_t i = 0; i < size; ++i) c[i] = applyScaled(a[i], b, scale_a); return; } else if (scale_b != 1) { for (size_t i = 0; i < size; ++i) c[i] = applyScaled(a[i], b, scale_b); return; } } else if constexpr (is_division && IsDecimalNumber) { for (size_t i = 0; i < size; ++i) c[i] = applyScaledDiv(a[i], b, scale_a); return; } /// default: use it if no return before for (size_t i = 0; i < size; ++i) c[i] = apply(a[i], b); } static void NO_INLINE constant_vector(A a, const ArrayB & b, ArrayC & c, ResultType scale_a [[maybe_unused]], ResultType scale_b [[maybe_unused]]) { size_t size = b.size(); if constexpr (is_plus_minus_compare) { if (scale_a != 1) { for (size_t i = 0; i < size; ++i) c[i] = applyScaled(a, b[i], scale_a); return; } else if (scale_b != 1) { for (size_t i = 0; i < size; ++i) c[i] = applyScaled(a, b[i], scale_b); return; } } else if constexpr (is_division && IsDecimalNumber) { for (size_t i = 0; i < size; ++i) c[i] = applyScaledDiv(a, b[i], scale_a); return; } /// default: use it if no return before for (size_t i = 0; i < size; ++i) c[i] = apply(a, b[i]); } static ResultType constant_constant(A a, B b, ResultType scale_a [[maybe_unused]], ResultType scale_b [[maybe_unused]]) { if constexpr (is_plus_minus_compare) { if (scale_a != 1) return applyScaled(a, b, scale_a); else if (scale_b != 1) return applyScaled(a, b, scale_b); } else if constexpr (is_division && IsDecimalNumber) return applyScaledDiv(a, b, scale_a); return apply(a, b); } private: /// there's implicit type convertion here static NativeResultType apply(NativeResultType a, NativeResultType b) { if constexpr (can_overflow) { NativeResultType res; if (Op::template apply(a, b, res)) throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); return res; } else return Op::template apply(a, b); } template static NativeResultType applyScaled(NativeResultType a, NativeResultType b, NativeResultType scale) { if constexpr (is_plus_minus_compare) { NativeResultType res; bool overflow = false; if constexpr (scale_left) overflow |= common::mulOverflow(a, scale, a); else overflow |= common::mulOverflow(b, scale, b); if constexpr (can_overflow) overflow |= Op::template apply(a, b, res); else res = Op::template apply(a, b); if (overflow) throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); return res; } } static NativeResultType applyScaledDiv(NativeResultType a, NativeResultType b, NativeResultType scale) { if constexpr (is_division) { bool overflow = false; if constexpr (!IsDecimalNumber) overflow |= common::mulOverflow(scale, scale, scale); overflow |= common::mulOverflow(a, scale, a); if (overflow) throw Exception("Decimal math overflow", ErrorCodes::DECIMAL_OVERFLOW); return Op::template apply(a, b); } } }; /// Used to indicate undefined operation struct InvalidType; template struct Case : std::bool_constant { using type = T; }; /// Switch, ...> -- select the first Ti for which Ci is true; InvalidType if none. template using Switch = typename std::disjunction>::type; template constexpr bool IsIntegral = false; template <> constexpr bool IsIntegral = true; template <> constexpr bool IsIntegral = true; template <> constexpr bool IsIntegral = true; template <> constexpr bool IsIntegral = true; template <> constexpr bool IsIntegral = true; template <> constexpr bool IsIntegral = true; template <> constexpr bool IsIntegral = true; template <> constexpr bool IsIntegral = true; template constexpr bool IsDateOrDateTime = false; template <> constexpr bool IsDateOrDateTime = true; template <> constexpr bool IsDateOrDateTime = true; template constexpr bool IsDecimal = false; template <> constexpr bool IsDecimal> = true; template <> constexpr bool IsDecimal> = true; template <> constexpr bool IsDecimal> = true; template constexpr bool UseLeftDecimal = false; template <> constexpr bool UseLeftDecimal, DataTypeDecimal> = true; template <> constexpr bool UseLeftDecimal, DataTypeDecimal> = true; template <> constexpr bool UseLeftDecimal, DataTypeDecimal> = true; template using DataTypeFromFieldType = std::conditional_t, InvalidType, DataTypeNumber>; template