#pragma once // Include this first, because `#define _asan_poison_address` from // llvm/Support/Compiler.h conflicts with its forward declaration in // sanitizer/asan_interface.h #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "IFunctionImpl.h" #include "FunctionHelpers.h" #include "IsOperation.h" #include "DivisionUtils.h" #include "castTypeToEither.h" #include "FunctionFactory.h" #include #include #include #include #include #if !defined(ARCADIA_BUILD) # include #endif #if USE_EMBEDDED_COMPILER # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Wunused-parameter" # include # pragma GCC diagnostic pop #endif namespace DB { namespace ErrorCodes { extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int LOGICAL_ERROR; extern const int DECIMAL_OVERFLOW; extern const int CANNOT_ADD_DIFFERENT_AGGREGATE_STATES; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } /** Arithmetic operations: +, -, *, /, %, * intDiv (integer division) * Bitwise operations: |, &, ^, ~. * Etc. */ template struct BinaryOperation { using ResultType = ResultType_; static const constexpr bool allow_fixed_string = false; static void NO_INLINE vectorVector(const A * __restrict a, const B * __restrict b, ResultType * __restrict c, size_t size) { for (size_t i = 0; i < size; ++i) c[i] = Op::template apply(a[i], b[i]); } static void NO_INLINE vectorConstant(const A * __restrict a, B b, ResultType * __restrict c, size_t size) { for (size_t i = 0; i < size; ++i) c[i] = Op::template apply(a[i], b); } static void NO_INLINE constantVector(A a, const B * __restrict b, ResultType * __restrict c, size_t size) { for (size_t i = 0; i < size; ++i) c[i] = Op::template apply(a, b[i]); } static ResultType constantConstant(A a, B b) { return Op::template apply(a, b); } }; template struct FixedStringOperationImpl { static void NO_INLINE vectorVector(const UInt8 * __restrict a, const UInt8 * __restrict b, UInt8 * __restrict c, size_t size) { for (size_t i = 0; i < size; ++i) c[i] = Op::template apply(a[i], b[i]); } template static void NO_INLINE vector_constant_impl(const UInt8 * __restrict a, const UInt8 * __restrict b, UInt8 * __restrict c, size_t size, size_t N) { /// These complications are needed to avoid integer division in inner loop. /// Create a pattern of repeated values of b with at least 16 bytes, /// so we can read 16 bytes of this repeated pattern starting from any offset inside b. /// /// Example: /// /// N = 6 /// ------ /// [abcdefabcdefabcdefabc] /// ^^^^^^^^^^^^^^^^ /// 16 bytes starting from the last offset inside b. const size_t b_repeated_size = N + 15; UInt8 b_repeated[b_repeated_size]; for (size_t i = 0; i < b_repeated_size; ++i) b_repeated[i] = b[i % N]; size_t b_offset = 0; size_t b_increment = 16 % N; /// Example: /// /// At first iteration we copy 16 bytes at offset 0 from b_repeated: /// [abcdefabcdefabcdefabc] /// ^^^^^^^^^^^^^^^^ /// At second iteration we copy 16 bytes at offset 4 = 16 % 6 from b_repeated: /// [abcdefabcdefabcdefabc] /// ^^^^^^^^^^^^^^^^ /// At third iteration we copy 16 bytes at offset 2 = (16 * 2) % 6 from b_repeated: /// [abcdefabcdefabcdefabc] /// ^^^^^^^^^^^^^^^^ /// PaddedPODArray allows overflow for 15 bytes. for (size_t i = 0; i < size; i += 16) { /// This loop is formed in a way to be vectorized into two SIMD mov. for (size_t j = 0; j < 16; ++j) c[i + j] = inverted ? Op::template apply(a[i + j], b_repeated[b_offset + j]) : Op::template apply(b_repeated[b_offset + j], a[i + j]); b_offset += b_increment; if (b_offset >= N) /// This condition is easily predictable. b_offset -= N; } } static void vectorConstant(const UInt8 * __restrict a, const UInt8 * __restrict b, UInt8 * __restrict c, size_t size, size_t N) { vector_constant_impl(a, b, c, size, N); } static void constantVector(const UInt8 * __restrict a, const UInt8 * __restrict b, UInt8 * __restrict c, size_t size, size_t N) { vector_constant_impl(b, a, c, size, N); } }; template struct BinaryOperationImpl : BinaryOperation { }; template inline constexpr const auto & undec(const T & x) { if constexpr (IsDecimalNumber) return x.value; else return x; } /// Binary operations for Decimals need scale args /// +|- scale one of args (which scale factor is not 1). ScaleR = oneof(Scale1, Scale2); /// * no agrs scale. ScaleR = Scale1 + Scale2; /// / first arg scale. ScaleR = Scale1 (scale_a = DecimalType::getScale()). template