#pragma once #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if USE_EMBEDDED_COMPILER #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #include #pragma GCC diagnostic pop #endif namespace DB { namespace ErrorCodes { extern const int ILLEGAL_DIVISION; extern const int ILLEGAL_COLUMN; extern const int LOGICAL_ERROR; extern const int TOO_LESS_ARGUMENTS_FOR_FUNCTION; } /** Arithmetic operations: +, -, *, /, %, * intDiv (integer division), unary minus. * Bitwise operations: |, &, ^, ~. * Etc. */ template struct BinaryOperationImplBase { using ResultType = ResultType_; static void NO_INLINE vector_vector(const PaddedPODArray & a, const PaddedPODArray & b, PaddedPODArray & c) { size_t size = a.size(); for (size_t i = 0; i < size; ++i) c[i] = Op::template apply(a[i], b[i]); } static void NO_INLINE vector_constant(const PaddedPODArray & a, B b, PaddedPODArray & c) { size_t size = a.size(); for (size_t i = 0; i < size; ++i) c[i] = Op::template apply(a[i], b); } static void NO_INLINE constant_vector(A a, const PaddedPODArray & b, PaddedPODArray & c) { size_t size = b.size(); for (size_t i = 0; i < size; ++i) c[i] = Op::template apply(a, b[i]); } static ResultType constant_constant(A a, B b) { return Op::template apply(a, b); } }; template struct BinaryOperationImpl : BinaryOperationImplBase { }; template struct UnaryOperationImpl { using ResultType = typename Op::ResultType; static void NO_INLINE vector(const PaddedPODArray & a, PaddedPODArray & c) { size_t size = a.size(); for (size_t i = 0; i < size; ++i) c[i] = Op::apply(a[i]); } static void constant(A a, ResultType & c) { c = Op::apply(a); } }; template struct PlusImpl { using ResultType = typename NumberTraits::ResultOfAdditionMultiplication::Type; template static inline Result apply(A a, B b) { /// Next everywhere, static_cast - so that there is no wrong result in expressions of the form Int64 c = UInt32(a) * Int32(-1). return static_cast(a) + b; } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { return left->getType()->isIntegerTy() ? b.CreateAdd(left, right) : b.CreateFAdd(left, right); } #endif }; template struct MultiplyImpl { using ResultType = typename NumberTraits::ResultOfAdditionMultiplication::Type; template static inline Result apply(A a, B b) { return static_cast(a) * b; } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { return left->getType()->isIntegerTy() ? b.CreateMul(left, right) : b.CreateFMul(left, right); } #endif }; template struct MinusImpl { using ResultType = typename NumberTraits::ResultOfSubtraction::Type; template static inline Result apply(A a, B b) { return static_cast(a) - b; } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { return left->getType()->isIntegerTy() ? b.CreateSub(left, right) : b.CreateFSub(left, right); } #endif }; template struct DivideFloatingImpl { using ResultType = typename NumberTraits::ResultOfFloatingPointDivision::Type; template static inline Result apply(A a, B b) { return static_cast(a) / b; } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { if (left->getType()->isIntegerTy()) throw Exception("DivideFloatingImpl expected a floating-point type", ErrorCodes::LOGICAL_ERROR); return b.CreateFDiv(left, right); } #endif }; #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wsign-compare" template inline void throwIfDivisionLeadsToFPE(A a, B b) { /// Is it better to use siglongjmp instead of checks? if (unlikely(b == 0)) throw Exception("Division by zero", ErrorCodes::ILLEGAL_DIVISION); /// http://avva.livejournal.com/2548306.html if (unlikely(std::is_signed_v && std::is_signed_v && a == std::numeric_limits::min() && b == -1)) throw Exception("Division of minimal signed number by minus one", ErrorCodes::ILLEGAL_DIVISION); } template inline bool divisionLeadsToFPE(A a, B b) { if (unlikely(b == 0)) return true; if (unlikely(std::is_signed_v && std::is_signed_v && a == std::numeric_limits::min() && b == -1)) return true; return false; } #pragma GCC diagnostic pop template struct DivideIntegralImpl { using ResultType = typename NumberTraits::ResultOfIntegerDivision::Type; template static inline Result apply(A a, B b) { throwIfDivisionLeadsToFPE(a, b); return a / b; } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// don't know how to throw from LLVM IR #endif }; template struct DivideIntegralOrZeroImpl { using ResultType = typename NumberTraits::ResultOfIntegerDivision::Type; template static inline Result apply(A a, B b) { return unlikely(divisionLeadsToFPE(a, b)) ? 0 : a / b; } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// TODO implement the checks #endif }; template struct ModuloImpl { using ResultType = typename NumberTraits::ResultOfModulo::Type; template static inline Result apply(A a, B b) { throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger::Type(a), typename NumberTraits::ToInteger::Type(b)); return typename NumberTraits::ToInteger::Type(a) % typename NumberTraits::ToInteger::Type(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// don't know how to throw from LLVM IR #endif }; template struct BitAndImpl { using ResultType = typename NumberTraits::ResultOfBit::Type; template static inline Result apply(A a, B b) { return static_cast(a) & static_cast(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { if (!left->getType()->isIntegerTy()) throw Exception("BitAndImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); return b.CreateAnd(left, right); } #endif }; template struct BitOrImpl { using ResultType = typename NumberTraits::ResultOfBit::Type; template static inline Result apply(A a, B b) { return static_cast(a) | static_cast(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { if (!left->getType()->isIntegerTy()) throw Exception("BitOrImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); return b.CreateOr(left, right); } #endif }; template struct BitXorImpl { using ResultType = typename NumberTraits::ResultOfBit::Type; template static inline Result apply(A a, B b) { return static_cast(a) ^ static_cast(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { if (!left->getType()->isIntegerTy()) throw Exception("BitXorImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); return b.CreateXor(left, right); } #endif }; template struct BitShiftLeftImpl { using ResultType = typename NumberTraits::ResultOfBit::Type; template static inline Result apply(A a, B b) { return static_cast(a) << static_cast(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { if (!left->getType()->isIntegerTy()) throw Exception("BitShiftLeftImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); return b.CreateShl(left, right); } #endif }; template struct BitShiftRightImpl { using ResultType = typename NumberTraits::ResultOfBit::Type; template static inline Result apply(A a, B b) { return static_cast(a) >> static_cast(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed) { if (!left->getType()->isIntegerTy()) throw Exception("BitShiftRightImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); return is_signed ? b.CreateAShr(left, right) : b.CreateLShr(left, right); } #endif }; template struct BitRotateLeftImpl { using ResultType = typename NumberTraits::ResultOfBit::Type; template static inline Result apply(A a, B b) { return (static_cast(a) << static_cast(b)) | (static_cast(a) >> ((sizeof(Result) * 8) - static_cast(b))); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { if (!left->getType()->isIntegerTy()) throw Exception("BitRotateLeftImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); auto * size = llvm::ConstantInt::get(left->getType(), left->getType()->getPrimitiveSizeInBits()); /// XXX how is this supposed to behave in signed mode? return b.CreateOr(b.CreateShl(left, right), b.CreateLShr(left, b.CreateSub(size, right))); } #endif }; template struct BitRotateRightImpl { using ResultType = typename NumberTraits::ResultOfBit::Type; template static inline Result apply(A a, B b) { return (static_cast(a) >> static_cast(b)) | (static_cast(a) << ((sizeof(Result) * 8) - static_cast(b))); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) { if (!left->getType()->isIntegerTy()) throw Exception("BitRotateRightImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); auto * size = llvm::ConstantInt::get(left->getType(), left->getType()->getPrimitiveSizeInBits()); return b.CreateOr(b.CreateLShr(left, right), b.CreateShl(left, b.CreateSub(size, right))); } #endif }; template struct BitTestImpl { using ResultType = UInt8; template static inline Result apply(A a, B b) { return (typename NumberTraits::ToInteger::Type(a) >> typename NumberTraits::ToInteger::Type(b)) & 1; } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// TODO #endif }; template struct LeastBaseImpl { using ResultType = NumberTraits::ResultOfLeast; template static inline Result apply(A a, B b) { /** gcc 4.9.2 successfully vectorizes a loop from this function. */ return static_cast(a) < static_cast(b) ? static_cast(a) : static_cast(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed) { if (!left->getType()->isIntegerTy()) /// XXX minnum is basically fmin(), it may or may not match whatever apply() does return b.CreateMinNum(left, right); return b.CreateSelect(is_signed ? b.CreateICmpSLT(left, right) : b.CreateICmpULT(left, right), left, right); } #endif }; template struct LeastSpecialImpl { using ResultType = std::make_signed_t; template static inline Result apply(A a, B b) { static_assert(std::is_same_v, "ResultType != Result"); return accurate::lessOp(a, b) ? static_cast(a) : static_cast(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// ??? #endif }; template using LeastImpl = std::conditional_t, LeastBaseImpl, LeastSpecialImpl>; template struct GreatestBaseImpl { using ResultType = NumberTraits::ResultOfGreatest; template static inline Result apply(A a, B b) { return static_cast(a) > static_cast(b) ? static_cast(a) : static_cast(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed) { if (!left->getType()->isIntegerTy()) /// XXX maxnum is basically fmax(), it may or may not match whatever apply() does /// XXX CreateMaxNum is broken on LLVM 5.0 and 6.0 (generates minnum instead; fixed in 7) return b.CreateBinaryIntrinsic(llvm::Intrinsic::maxnum, left, right); return b.CreateSelect(is_signed ? b.CreateICmpSGT(left, right) : b.CreateICmpUGT(left, right), left, right); } #endif }; template struct GreatestSpecialImpl { using ResultType = std::make_unsigned_t; template static inline Result apply(A a, B b) { static_assert(std::is_same_v, "ResultType != Result"); return accurate::greaterOp(a, b) ? static_cast(a) : static_cast(b); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// ??? #endif }; template using GreatestImpl = std::conditional_t, GreatestBaseImpl, GreatestSpecialImpl>; template struct NegateImpl { using ResultType = typename NumberTraits::ResultOfNegate::Type; static inline ResultType apply(A a) { return -static_cast(a); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * arg, bool) { return arg->getType()->isIntegerTy() ? b.CreateNeg(arg) : b.CreateFNeg(arg); } #endif }; template struct BitNotImpl { using ResultType = typename NumberTraits::ResultOfBitNot::Type; static inline ResultType apply(A a) { return ~static_cast(a); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * arg, bool) { if (!arg->getType()->isIntegerTy()) throw Exception("BitNotImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); return b.CreateNot(arg); } #endif }; template struct AbsImpl { using ResultType = typename NumberTraits::ResultOfAbs::Type; static inline ResultType apply(A a) { if constexpr (std::is_integral_v && std::is_signed_v) return a < 0 ? static_cast(~a) + 1 : a; else if constexpr (std::is_integral_v && std::is_unsigned_v) return static_cast(a); else if constexpr (std::is_floating_point_v) return static_cast(std::abs(a)); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// special type handling, some other time #endif }; template struct GCDImpl { using ResultType = typename NumberTraits::ResultOfAdditionMultiplication::Type; template static inline Result apply(A a, B b) { throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger::Type(a), typename NumberTraits::ToInteger::Type(b)); throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger::Type(b), typename NumberTraits::ToInteger::Type(a)); return boost::integer::gcd( typename NumberTraits::ToInteger::Type(a), typename NumberTraits::ToInteger::Type(b)); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// exceptions (and a non-trivial algorithm) #endif }; template struct LCMImpl { using ResultType = typename NumberTraits::ResultOfAdditionMultiplication::Type; template static inline Result apply(A a, B b) { throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger::Type(a), typename NumberTraits::ToInteger::Type(b)); throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger::Type(b), typename NumberTraits::ToInteger::Type(a)); return boost::integer::lcm( typename NumberTraits::ToInteger::Type(a), typename NumberTraits::ToInteger::Type(b)); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// exceptions (and a non-trivial algorithm) #endif }; template struct IntExp2Impl { using ResultType = UInt64; static inline ResultType apply(A a) { return intExp2(a); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = true; static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * arg, bool) { if (!arg->getType()->isIntegerTy()) throw Exception("IntExp2Impl expected an integral type", ErrorCodes::LOGICAL_ERROR); return b.CreateShl(llvm::ConstantInt::get(arg->getType(), 1), arg); } #endif }; template struct IntExp10Impl { using ResultType = UInt64; static inline ResultType apply(A a) { return intExp10(a); } #if USE_EMBEDDED_COMPILER static constexpr bool compilable = false; /// library function #endif }; /// Used to indicate undefined operation struct InvalidType; template struct Case : std::bool_constant { using type = T; }; /// Switch, ...> -- select the first Ti for which Ci is true; InvalidType if none. template using Switch = typename std::disjunction>::type; template constexpr bool IsIntegral = false; template <> constexpr bool IsIntegral = true; template <> constexpr bool IsIntegral = true; template <> constexpr bool IsIntegral = true; template <> constexpr bool IsIntegral = true; template <> constexpr bool IsIntegral = true; template <> constexpr bool IsIntegral = true; template <> constexpr bool IsIntegral = true; template <> constexpr bool IsIntegral = true; template constexpr bool IsDateOrDateTime = false; template <> constexpr bool IsDateOrDateTime = true; template <> constexpr bool IsDateOrDateTime = true; template using DataTypeFromFieldType = std::conditional_t, InvalidType, DataTypeNumber>; template