diff --git a/base/common/setTerminalEcho.cpp b/base/common/setTerminalEcho.cpp index 658f27705ba..66db216a235 100644 --- a/base/common/setTerminalEcho.cpp +++ b/base/common/setTerminalEcho.cpp @@ -1,45 +1,28 @@ -// https://stackoverflow.com/questions/1413445/reading-a-password-from-stdcin - #include #include #include #include #include - -#ifdef WIN32 -#include -#else #include #include -#include -#endif + void setTerminalEcho(bool enable) { -#ifdef WIN32 - auto handle = GetStdHandle(STD_INPUT_HANDLE); - DWORD mode; - if (!GetConsoleMode(handle, &mode)) - throw std::runtime_error(std::string("setTerminalEcho failed get: ") + std::to_string(GetLastError())); + /// Obtain terminal attributes, + /// toggle the ECHO flag + /// and set them back. - if (!enable) - mode &= ~ENABLE_ECHO_INPUT; - else - mode |= ENABLE_ECHO_INPUT; + struct termios tty{}; - if (!SetConsoleMode(handle, mode)) - throw std::runtime_error(std::string("setTerminalEcho failed set: ") + std::to_string(GetLastError())); -#else - struct termios tty; - if (tcgetattr(STDIN_FILENO, &tty)) + if (0 != tcgetattr(STDIN_FILENO, &tty)) throw std::runtime_error(std::string("setTerminalEcho failed get: ") + errnoToString(errno)); - if (!enable) - tty.c_lflag &= ~ECHO; - else - tty.c_lflag |= ECHO; - auto ret = tcsetattr(STDIN_FILENO, TCSANOW, &tty); - if (ret) + if (enable) + tty.c_lflag |= ECHO; + else + tty.c_lflag &= ~ECHO; + + if (0 != tcsetattr(STDIN_FILENO, TCSANOW, &tty)) throw std::runtime_error(std::string("setTerminalEcho failed set: ") + errnoToString(errno)); -#endif } diff --git a/contrib/libdivide/libdivide.h b/contrib/libdivide/libdivide.h index 81057b7b43d..33d210310a1 100644 --- a/contrib/libdivide/libdivide.h +++ b/contrib/libdivide/libdivide.h @@ -18,78 +18,79 @@ #include #if defined(__cplusplus) - #include - #include - #include +#include +#include +#include #else - #include - #include +#include +#include #endif -#if defined(LIBDIVIDE_AVX512) - #include -#elif defined(LIBDIVIDE_AVX2) - #include -#elif defined(LIBDIVIDE_SSE2) - #include +#if defined(LIBDIVIDE_SSE2) +#include +#endif +#if defined(LIBDIVIDE_AVX2) || defined(LIBDIVIDE_AVX512) +#include +#endif +#if defined(LIBDIVIDE_NEON) +#include #endif #if defined(_MSC_VER) - #include - // disable warning C4146: unary minus operator applied - // to unsigned type, result still unsigned - #pragma warning(disable: 4146) - #define LIBDIVIDE_VC +#include +// disable warning C4146: unary minus operator applied +// to unsigned type, result still unsigned +#pragma warning(disable : 4146) +#define LIBDIVIDE_VC #endif #if !defined(__has_builtin) - #define __has_builtin(x) 0 +#define __has_builtin(x) 0 #endif #if defined(__SIZEOF_INT128__) - #define HAS_INT128_T - // clang-cl on Windows does not yet support 128-bit division - #if !(defined(__clang__) && defined(LIBDIVIDE_VC)) - #define HAS_INT128_DIV - #endif +#define HAS_INT128_T +// clang-cl on Windows does not yet support 128-bit division +#if !(defined(__clang__) && defined(LIBDIVIDE_VC)) +#define HAS_INT128_DIV +#endif #endif #if defined(__x86_64__) || defined(_M_X64) - #define LIBDIVIDE_X86_64 +#define LIBDIVIDE_X86_64 #endif #if defined(__i386__) - #define LIBDIVIDE_i386 +#define LIBDIVIDE_i386 #endif #if defined(__GNUC__) || defined(__clang__) - #define LIBDIVIDE_GCC_STYLE_ASM +#define LIBDIVIDE_GCC_STYLE_ASM #endif #if defined(__cplusplus) || defined(LIBDIVIDE_VC) - #define LIBDIVIDE_FUNCTION __FUNCTION__ +#define LIBDIVIDE_FUNCTION __FUNCTION__ #else - #define LIBDIVIDE_FUNCTION __func__ +#define LIBDIVIDE_FUNCTION __func__ #endif -#define LIBDIVIDE_ERROR(msg) \ - do { \ - fprintf(stderr, "libdivide.h:%d: %s(): Error: %s\n", \ - __LINE__, LIBDIVIDE_FUNCTION, msg); \ - abort(); \ +#define LIBDIVIDE_ERROR(msg) \ + do { \ + fprintf(stderr, "libdivide.h:%d: %s(): Error: %s\n", __LINE__, LIBDIVIDE_FUNCTION, msg); \ + abort(); \ } while (0) #if defined(LIBDIVIDE_ASSERTIONS_ON) - #define LIBDIVIDE_ASSERT(x) \ - do { \ - if (!(x)) { \ - fprintf(stderr, "libdivide.h:%d: %s(): Assertion failed: %s\n", \ - __LINE__, LIBDIVIDE_FUNCTION, #x); \ - abort(); \ - } \ - } while (0) +#define LIBDIVIDE_ASSERT(x) \ + do { \ + if (!(x)) { \ + fprintf(stderr, "libdivide.h:%d: %s(): Assertion failed: %s\n", __LINE__, \ + LIBDIVIDE_FUNCTION, #x); \ + abort(); \ + } \ + } while (0) #else - #define LIBDIVIDE_ASSERT(x) +#define LIBDIVIDE_ASSERT(x) #endif #ifdef __cplusplus @@ -193,25 +194,33 @@ static inline struct libdivide_u32_branchfree_t libdivide_u32_branchfree_gen(uin static inline struct libdivide_s64_branchfree_t libdivide_s64_branchfree_gen(int64_t d); static inline struct libdivide_u64_branchfree_t libdivide_u64_branchfree_gen(uint64_t d); -static inline int32_t libdivide_s32_do(int32_t numer, const struct libdivide_s32_t *denom); +static inline int32_t libdivide_s32_do(int32_t numer, const struct libdivide_s32_t *denom); static inline uint32_t libdivide_u32_do(uint32_t numer, const struct libdivide_u32_t *denom); -static inline int64_t libdivide_s64_do(int64_t numer, const struct libdivide_s64_t *denom); +static inline int64_t libdivide_s64_do(int64_t numer, const struct libdivide_s64_t *denom); static inline uint64_t libdivide_u64_do(uint64_t numer, const struct libdivide_u64_t *denom); -static inline int32_t libdivide_s32_branchfree_do(int32_t numer, const struct libdivide_s32_branchfree_t *denom); -static inline uint32_t libdivide_u32_branchfree_do(uint32_t numer, const struct libdivide_u32_branchfree_t *denom); -static inline int64_t libdivide_s64_branchfree_do(int64_t numer, const struct libdivide_s64_branchfree_t *denom); -static inline uint64_t libdivide_u64_branchfree_do(uint64_t numer, const struct libdivide_u64_branchfree_t *denom); +static inline int32_t libdivide_s32_branchfree_do( + int32_t numer, const struct libdivide_s32_branchfree_t *denom); +static inline uint32_t libdivide_u32_branchfree_do( + uint32_t numer, const struct libdivide_u32_branchfree_t *denom); +static inline int64_t libdivide_s64_branchfree_do( + int64_t numer, const struct libdivide_s64_branchfree_t *denom); +static inline uint64_t libdivide_u64_branchfree_do( + uint64_t numer, const struct libdivide_u64_branchfree_t *denom); -static inline int32_t libdivide_s32_recover(const struct libdivide_s32_t *denom); +static inline int32_t libdivide_s32_recover(const struct libdivide_s32_t *denom); static inline uint32_t libdivide_u32_recover(const struct libdivide_u32_t *denom); -static inline int64_t libdivide_s64_recover(const struct libdivide_s64_t *denom); +static inline int64_t libdivide_s64_recover(const struct libdivide_s64_t *denom); static inline uint64_t libdivide_u64_recover(const struct libdivide_u64_t *denom); -static inline int32_t libdivide_s32_branchfree_recover(const struct libdivide_s32_branchfree_t *denom); -static inline uint32_t libdivide_u32_branchfree_recover(const struct libdivide_u32_branchfree_t *denom); -static inline int64_t libdivide_s64_branchfree_recover(const struct libdivide_s64_branchfree_t *denom); -static inline uint64_t libdivide_u64_branchfree_recover(const struct libdivide_u64_branchfree_t *denom); +static inline int32_t libdivide_s32_branchfree_recover( + const struct libdivide_s32_branchfree_t *denom); +static inline uint32_t libdivide_u32_branchfree_recover( + const struct libdivide_u32_branchfree_t *denom); +static inline int64_t libdivide_s64_branchfree_recover( + const struct libdivide_s64_branchfree_t *denom); +static inline uint64_t libdivide_u64_branchfree_recover( + const struct libdivide_u64_branchfree_t *denom); //////// Internal Utility Functions @@ -229,8 +238,7 @@ static inline int32_t libdivide_mullhi_s32(int32_t x, int32_t y) { } static inline uint64_t libdivide_mullhi_u64(uint64_t x, uint64_t y) { -#if defined(LIBDIVIDE_VC) && \ - defined(LIBDIVIDE_X86_64) +#if defined(LIBDIVIDE_VC) && defined(LIBDIVIDE_X86_64) return __umulh(x, y); #elif defined(HAS_INT128_T) __uint128_t xl = x, yl = y; @@ -256,8 +264,7 @@ static inline uint64_t libdivide_mullhi_u64(uint64_t x, uint64_t y) { } static inline int64_t libdivide_mullhi_s64(int64_t x, int64_t y) { -#if defined(LIBDIVIDE_VC) && \ - defined(LIBDIVIDE_X86_64) +#if defined(LIBDIVIDE_VC) && defined(LIBDIVIDE_X86_64) return __mulh(x, y); #elif defined(HAS_INT128_T) __int128_t xl = x, yl = y; @@ -279,8 +286,7 @@ static inline int64_t libdivide_mullhi_s64(int64_t x, int64_t y) { } static inline int32_t libdivide_count_leading_zeros32(uint32_t val) { -#if defined(__GNUC__) || \ - __has_builtin(__builtin_clz) +#if defined(__GNUC__) || __has_builtin(__builtin_clz) // Fast way to count leading zeros return __builtin_clz(val); #elif defined(LIBDIVIDE_VC) @@ -290,8 +296,7 @@ static inline int32_t libdivide_count_leading_zeros32(uint32_t val) { } return 0; #else - if (val == 0) - return 32; + if (val == 0) return 32; int32_t result = 8; uint32_t hi = 0xFFU << 24; while ((val & hi) == 0) { @@ -307,8 +312,7 @@ static inline int32_t libdivide_count_leading_zeros32(uint32_t val) { } static inline int32_t libdivide_count_leading_zeros64(uint64_t val) { -#if defined(__GNUC__) || \ - __has_builtin(__builtin_clzll) +#if defined(__GNUC__) || __has_builtin(__builtin_clzll) // Fast way to count leading zeros return __builtin_clzll(val); #elif defined(LIBDIVIDE_VC) && defined(_WIN64) @@ -328,14 +332,11 @@ static inline int32_t libdivide_count_leading_zeros64(uint64_t val) { // libdivide_64_div_32_to_32: divides a 64-bit uint {u1, u0} by a 32-bit // uint {v}. The result must fit in 32 bits. // Returns the quotient directly and the remainder in *r -static inline uint32_t libdivide_64_div_32_to_32(uint32_t u1, uint32_t u0, uint32_t v, uint32_t *r) { -#if (defined(LIBDIVIDE_i386) || defined(LIBDIVIDE_X86_64)) && \ - defined(LIBDIVIDE_GCC_STYLE_ASM) +static inline uint32_t libdivide_64_div_32_to_32( + uint32_t u1, uint32_t u0, uint32_t v, uint32_t *r) { +#if (defined(LIBDIVIDE_i386) || defined(LIBDIVIDE_X86_64)) && defined(LIBDIVIDE_GCC_STYLE_ASM) uint32_t result; - __asm__("divl %[v]" - : "=a"(result), "=d"(*r) - : [v] "r"(v), "a"(u0), "d"(u1) - ); + __asm__("divl %[v]" : "=a"(result), "=d"(*r) : [v] "r"(v), "a"(u0), "d"(u1)); return result; #else uint64_t n = ((uint64_t)u1 << 32) | u0; @@ -349,19 +350,13 @@ static inline uint32_t libdivide_64_div_32_to_32(uint32_t u1, uint32_t u0, uint3 // uint {v}. The result must fit in 64 bits. // Returns the quotient directly and the remainder in *r static uint64_t libdivide_128_div_64_to_64(uint64_t u1, uint64_t u0, uint64_t v, uint64_t *r) { -#if defined(LIBDIVIDE_X86_64) && \ - defined(LIBDIVIDE_GCC_STYLE_ASM) + // N.B. resist the temptation to use __uint128_t here. + // In LLVM compiler-rt, it performs a 128/128 -> 128 division which is many times slower than + // necessary. In gcc it's better but still slower than the divlu implementation, perhaps because + // it's not inlined. +#if defined(LIBDIVIDE_X86_64) && defined(LIBDIVIDE_GCC_STYLE_ASM) uint64_t result; - __asm__("divq %[v]" - : "=a"(result), "=d"(*r) - : [v] "r"(v), "a"(u0), "d"(u1) - ); - return result; -#elif defined(HAS_INT128_T) && \ - defined(HAS_INT128_DIV) - __uint128_t n = ((__uint128_t)u1 << 64) | u0; - uint64_t result = (uint64_t)(n / v); - *r = (uint64_t)(n - result * (__uint128_t)v); + __asm__("divq %[v]" : "=a"(result), "=d"(*r) : [v] "r"(v), "a"(u0), "d"(u1)); return result; #else // Code taken from Hacker's Delight: @@ -369,19 +364,19 @@ static uint64_t libdivide_128_div_64_to_64(uint64_t u1, uint64_t u0, uint64_t v, // License permits inclusion here per: // http://www.hackersdelight.org/permissions.htm - const uint64_t b = (1ULL << 32); // Number base (32 bits) - uint64_t un1, un0; // Norm. dividend LSD's - uint64_t vn1, vn0; // Norm. divisor digits - uint64_t q1, q0; // Quotient digits - uint64_t un64, un21, un10; // Dividend digit pairs - uint64_t rhat; // A remainder - int32_t s; // Shift amount for norm + const uint64_t b = (1ULL << 32); // Number base (32 bits) + uint64_t un1, un0; // Norm. dividend LSD's + uint64_t vn1, vn0; // Norm. divisor digits + uint64_t q1, q0; // Quotient digits + uint64_t un64, un21, un10; // Dividend digit pairs + uint64_t rhat; // A remainder + int32_t s; // Shift amount for norm // If overflow, set rem. to an impossible value, // and return the largest possible quotient if (u1 >= v) { - *r = (uint64_t) -1; - return (uint64_t) -1; + *r = (uint64_t)-1; + return (uint64_t)-1; } // count leading zeros @@ -390,7 +385,7 @@ static uint64_t libdivide_128_div_64_to_64(uint64_t u1, uint64_t u0, uint64_t v, // Normalize divisor v = v << s; un64 = (u1 << s) | (u0 >> (64 - s)); - un10 = u0 << s; // Shift dividend left + un10 = u0 << s; // Shift dividend left } else { // Avoid undefined behavior of (u0 >> 64). // The behavior is undefined if the right operand is @@ -415,11 +410,10 @@ static uint64_t libdivide_128_div_64_to_64(uint64_t u1, uint64_t u0, uint64_t v, while (q1 >= b || q1 * vn0 > b * rhat + un1) { q1 = q1 - 1; rhat = rhat + vn1; - if (rhat >= b) - break; + if (rhat >= b) break; } - // Multiply and subtract + // Multiply and subtract un21 = un64 * b + un1 - q1 * v; // Compute the second quotient digit @@ -429,8 +423,7 @@ static uint64_t libdivide_128_div_64_to_64(uint64_t u1, uint64_t u0, uint64_t v, while (q0 >= b || q0 * vn0 > b * rhat + un0) { q0 = q0 - 1; rhat = rhat + vn1; - if (rhat >= b) - break; + if (rhat >= b) break; } *r = (un21 * b + un0 - q0 * v) >> s; @@ -445,8 +438,7 @@ static inline void libdivide_u128_shift(uint64_t *u1, uint64_t *u0, int32_t sign *u1 <<= shift; *u1 |= *u0 >> (64 - shift); *u0 <<= shift; - } - else if (signed_shift < 0) { + } else if (signed_shift < 0) { uint32_t shift = -signed_shift; *u0 >>= shift; *u0 |= *u1 << (64 - shift); @@ -455,9 +447,9 @@ static inline void libdivide_u128_shift(uint64_t *u1, uint64_t *u0, int32_t sign } // Computes a 128 / 128 -> 64 bit division, with a 128 bit remainder. -static uint64_t libdivide_128_div_128_to_64(uint64_t u_hi, uint64_t u_lo, uint64_t v_hi, uint64_t v_lo, uint64_t *r_hi, uint64_t *r_lo) { -#if defined(HAS_INT128_T) && \ - defined(HAS_INT128_DIV) +static uint64_t libdivide_128_div_128_to_64( + uint64_t u_hi, uint64_t u_lo, uint64_t v_hi, uint64_t v_lo, uint64_t *r_hi, uint64_t *r_lo) { +#if defined(HAS_INT128_T) && defined(HAS_INT128_DIV) __uint128_t ufull = u_hi; __uint128_t vfull = v_hi; ufull = (ufull << 64) | u_lo; @@ -470,7 +462,10 @@ static uint64_t libdivide_128_div_128_to_64(uint64_t u_hi, uint64_t u_lo, uint64 #else // Adapted from "Unsigned Doubleword Division" in Hacker's Delight // We want to compute u / v - typedef struct { uint64_t hi; uint64_t lo; } u128_t; + typedef struct { + uint64_t hi; + uint64_t lo; + } u128_t; u128_t u = {u_hi, u_lo}; u128_t v = {v_hi, v_lo}; @@ -490,7 +485,7 @@ static uint64_t libdivide_128_div_128_to_64(uint64_t u_hi, uint64_t u_lo, uint64 // Normalize the divisor so its MSB is 1 u128_t v1t = v; libdivide_u128_shift(&v1t.hi, &v1t.lo, n); - uint64_t v1 = v1t.hi; // i.e. v1 = v1t >> 64 + uint64_t v1 = v1t.hi; // i.e. v1 = v1t >> 64 // To ensure no overflow u128_t u1 = u; @@ -508,7 +503,7 @@ static uint64_t libdivide_128_div_128_to_64(uint64_t u_hi, uint64_t u_lo, uint64 // Make q0 correct or too small by 1 // Equivalent to `if (q0 != 0) q0 = q0 - 1;` if (q0.hi != 0 || q0.lo != 0) { - q0.hi -= (q0.lo == 0); // borrow + q0.hi -= (q0.lo == 0); // borrow q0.lo -= 1; } @@ -520,22 +515,21 @@ static uint64_t libdivide_128_div_128_to_64(uint64_t u_hi, uint64_t u_lo, uint64 // Each term is 128 bit // High half of full product (upper 128 bits!) are dropped u128_t q0v = {0, 0}; - q0v.hi = q0.hi*v.lo + q0.lo*v.hi + libdivide_mullhi_u64(q0.lo, v.lo); - q0v.lo = q0.lo*v.lo; + q0v.hi = q0.hi * v.lo + q0.lo * v.hi + libdivide_mullhi_u64(q0.lo, v.lo); + q0v.lo = q0.lo * v.lo; // Compute u - q0v as u_q0v // This is the remainder u128_t u_q0v = u; - u_q0v.hi -= q0v.hi + (u.lo < q0v.lo); // second term is borrow + u_q0v.hi -= q0v.hi + (u.lo < q0v.lo); // second term is borrow u_q0v.lo -= q0v.lo; // Check if u_q0v >= v // This checks if our remainder is larger than the divisor - if ((u_q0v.hi > v.hi) || - (u_q0v.hi == v.hi && u_q0v.lo >= v.lo)) { + if ((u_q0v.hi > v.hi) || (u_q0v.hi == v.hi && u_q0v.lo >= v.lo)) { // Increment q0 q0.lo += 1; - q0.hi += (q0.lo == 0); // carry + q0.hi += (q0.lo == 0); // carry // Subtract v from remainder u_q0v.hi -= v.hi + (u_q0v.lo < v.lo); @@ -611,7 +605,8 @@ struct libdivide_u32_branchfree_t libdivide_u32_branchfree_gen(uint32_t d) { LIBDIVIDE_ERROR("branchfree divider must be != 1"); } struct libdivide_u32_t tmp = libdivide_internal_u32_gen(d, 1); - struct libdivide_u32_branchfree_t ret = {tmp.magic, (uint8_t)(tmp.more & LIBDIVIDE_32_SHIFT_MASK)}; + struct libdivide_u32_branchfree_t ret = { + tmp.magic, (uint8_t)(tmp.more & LIBDIVIDE_32_SHIFT_MASK)}; return ret; } @@ -619,14 +614,12 @@ uint32_t libdivide_u32_do(uint32_t numer, const struct libdivide_u32_t *denom) { uint8_t more = denom->more; if (!denom->magic) { return numer >> more; - } - else { + } else { uint32_t q = libdivide_mullhi_u32(denom->magic, numer); if (more & LIBDIVIDE_ADD_MARKER) { uint32_t t = ((numer - q) >> 1) + q; return t >> (more & LIBDIVIDE_32_SHIFT_MASK); - } - else { + } else { // All upper bits are 0, // don't need to mask them off. return q >> more; @@ -634,7 +627,8 @@ uint32_t libdivide_u32_do(uint32_t numer, const struct libdivide_u32_t *denom) { } } -uint32_t libdivide_u32_branchfree_do(uint32_t numer, const struct libdivide_u32_branchfree_t *denom) { +uint32_t libdivide_u32_branchfree_do( + uint32_t numer, const struct libdivide_u32_branchfree_t *denom) { uint32_t q = libdivide_mullhi_u32(denom->magic, numer); uint32_t t = ((numer - q) >> 1) + q; return t >> denom->more; @@ -671,7 +665,7 @@ uint32_t libdivide_u32_recover(const struct libdivide_u32_t *denom) { // Need to double it, and then add 1 to the quotient if doubling th // remainder would increase the quotient. // Note that rem<<1 cannot overflow, since rem < d and d is 33 bits - uint32_t full_q = half_q + half_q + ((rem<<1) >= d); + uint32_t full_q = half_q + half_q + ((rem << 1) >= d); // We rounded down in gen (hence +1) return full_q + 1; @@ -700,7 +694,7 @@ uint32_t libdivide_u32_branchfree_recover(const struct libdivide_u32_branchfree_ // Need to double it, and then add 1 to the quotient if doubling th // remainder would increase the quotient. // Note that rem<<1 cannot overflow, since rem < d and d is 33 bits - uint32_t full_q = half_q + half_q + ((rem<<1) >= d); + uint32_t full_q = half_q + half_q + ((rem << 1) >= d); // We rounded down in gen (hence +1) return full_q + 1; @@ -747,7 +741,7 @@ static inline struct libdivide_u64_t libdivide_internal_u64_gen(uint64_t d, int proposed_m += proposed_m; const uint64_t twice_rem = rem + rem; if (twice_rem >= d || twice_rem < rem) proposed_m += 1; - more = floor_log_2_d | LIBDIVIDE_ADD_MARKER; + more = floor_log_2_d | LIBDIVIDE_ADD_MARKER; } result.magic = 1 + proposed_m; result.more = more; @@ -770,7 +764,8 @@ struct libdivide_u64_branchfree_t libdivide_u64_branchfree_gen(uint64_t d) { LIBDIVIDE_ERROR("branchfree divider must be != 1"); } struct libdivide_u64_t tmp = libdivide_internal_u64_gen(d, 1); - struct libdivide_u64_branchfree_t ret = {tmp.magic, (uint8_t)(tmp.more & LIBDIVIDE_64_SHIFT_MASK)}; + struct libdivide_u64_branchfree_t ret = { + tmp.magic, (uint8_t)(tmp.more & LIBDIVIDE_64_SHIFT_MASK)}; return ret; } @@ -778,22 +773,21 @@ uint64_t libdivide_u64_do(uint64_t numer, const struct libdivide_u64_t *denom) { uint8_t more = denom->more; if (!denom->magic) { return numer >> more; - } - else { + } else { uint64_t q = libdivide_mullhi_u64(denom->magic, numer); if (more & LIBDIVIDE_ADD_MARKER) { uint64_t t = ((numer - q) >> 1) + q; return t >> (more & LIBDIVIDE_64_SHIFT_MASK); - } - else { - // All upper bits are 0, - // don't need to mask them off. + } else { + // All upper bits are 0, + // don't need to mask them off. return q >> more; } } } -uint64_t libdivide_u64_branchfree_do(uint64_t numer, const struct libdivide_u64_branchfree_t *denom) { +uint64_t libdivide_u64_branchfree_do( + uint64_t numer, const struct libdivide_u64_branchfree_t *denom) { uint64_t q = libdivide_mullhi_u64(denom->magic, numer); uint64_t t = ((numer - q) >> 1) + q; return t >> denom->more; @@ -829,13 +823,14 @@ uint64_t libdivide_u64_recover(const struct libdivide_u64_t *denom) { // Note that the quotient is guaranteed <= 64 bits, // but the remainder may need 65! uint64_t r_hi, r_lo; - uint64_t half_q = libdivide_128_div_128_to_64(half_n_hi, half_n_lo, d_hi, d_lo, &r_hi, &r_lo); + uint64_t half_q = + libdivide_128_div_128_to_64(half_n_hi, half_n_lo, d_hi, d_lo, &r_hi, &r_lo); // We computed 2^(64+shift)/(m+2^64) // Double the remainder ('dr') and check if that is larger than d // Note that d is a 65 bit value, so r1 is small and so r1 + r1 // cannot overflow uint64_t dr_lo = r_lo + r_lo; - uint64_t dr_hi = r_hi + r_hi + (dr_lo < r_lo); // last term is carry + uint64_t dr_hi = r_hi + r_hi + (dr_lo < r_lo); // last term is carry int dr_exceeds_d = (dr_hi > d_hi) || (dr_hi == d_hi && dr_lo >= d_lo); uint64_t full_q = half_q + half_q + (dr_exceeds_d ? 1 : 0); return full_q + 1; @@ -863,13 +858,14 @@ uint64_t libdivide_u64_branchfree_recover(const struct libdivide_u64_branchfree_ // Note that the quotient is guaranteed <= 64 bits, // but the remainder may need 65! uint64_t r_hi, r_lo; - uint64_t half_q = libdivide_128_div_128_to_64(half_n_hi, half_n_lo, d_hi, d_lo, &r_hi, &r_lo); + uint64_t half_q = + libdivide_128_div_128_to_64(half_n_hi, half_n_lo, d_hi, d_lo, &r_hi, &r_lo); // We computed 2^(64+shift)/(m+2^64) // Double the remainder ('dr') and check if that is larger than d // Note that d is a 65 bit value, so r1 is small and so r1 + r1 // cannot overflow uint64_t dr_lo = r_lo + r_lo; - uint64_t dr_hi = r_hi + r_hi + (dr_lo < r_lo); // last term is carry + uint64_t dr_hi = r_hi + r_hi + (dr_lo < r_lo); // last term is carry int dr_exceeds_d = (dr_hi > d_hi) || (dr_hi == d_hi && dr_lo >= d_lo); uint64_t full_q = half_q + half_q + (dr_exceeds_d ? 1 : 0); return full_q + 1; @@ -1023,8 +1019,7 @@ int32_t libdivide_s32_recover(const struct libdivide_s32_t *denom) { // the magic number's sign is opposite that of the divisor. // We want to compute the positive magic number. int negative_divisor = (more & LIBDIVIDE_NEGATIVE_DIVISOR); - int magic_was_negated = (more & LIBDIVIDE_ADD_MARKER) - ? denom->magic > 0 : denom->magic < 0; + int magic_was_negated = (more & LIBDIVIDE_ADD_MARKER) ? denom->magic > 0 : denom->magic < 0; // Handle the power of 2 case (including branchfree) if (denom->magic == 0) { @@ -1033,7 +1028,7 @@ int32_t libdivide_s32_recover(const struct libdivide_s32_t *denom) { } uint32_t d = (uint32_t)(magic_was_negated ? -denom->magic : denom->magic); - uint64_t n = 1ULL << (32 + shift); // this shift cannot exceed 30 + uint64_t n = 1ULL << (32 + shift); // this shift cannot exceed 30 uint32_t q = (uint32_t)(n / d); int32_t result = (int32_t)q; result += 1; @@ -1126,7 +1121,7 @@ int64_t libdivide_s64_do(int64_t numer, const struct libdivide_s64_t *denom) { uint8_t more = denom->more; uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; - if (!denom->magic) { // shift path + if (!denom->magic) { // shift path uint64_t mask = (1ULL << shift) - 1; uint64_t uq = numer + ((numer >> 63) & mask); int64_t q = (int64_t)uq; @@ -1178,7 +1173,7 @@ int64_t libdivide_s64_branchfree_do(int64_t numer, const struct libdivide_s64_br int64_t libdivide_s64_recover(const struct libdivide_s64_t *denom) { uint8_t more = denom->more; uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; - if (denom->magic == 0) { // shift path + if (denom->magic == 0) { // shift path uint64_t absD = 1ULL << shift; if (more & LIBDIVIDE_NEGATIVE_DIVISOR) { absD = -absD; @@ -1187,8 +1182,7 @@ int64_t libdivide_s64_recover(const struct libdivide_s64_t *denom) { } else { // Unsigned math is much easier int negative_divisor = (more & LIBDIVIDE_NEGATIVE_DIVISOR); - int magic_was_negated = (more & LIBDIVIDE_ADD_MARKER) - ? denom->magic > 0 : denom->magic < 0; + int magic_was_negated = (more & LIBDIVIDE_ADD_MARKER) ? denom->magic > 0 : denom->magic < 0; uint64_t d = (uint64_t)(magic_was_negated ? -denom->magic : denom->magic); uint64_t n_hi = 1ULL << shift, n_lo = 0; @@ -1206,30 +1200,305 @@ int64_t libdivide_s64_branchfree_recover(const struct libdivide_s64_branchfree_t return libdivide_s64_recover((const struct libdivide_s64_t *)denom); } -#if defined(LIBDIVIDE_AVX512) +#if defined(LIBDIVIDE_NEON) -static inline __m512i libdivide_u32_do_vector(__m512i numers, const struct libdivide_u32_t *denom); -static inline __m512i libdivide_s32_do_vector(__m512i numers, const struct libdivide_s32_t *denom); -static inline __m512i libdivide_u64_do_vector(__m512i numers, const struct libdivide_u64_t *denom); -static inline __m512i libdivide_s64_do_vector(__m512i numers, const struct libdivide_s64_t *denom); +static inline uint32x4_t libdivide_u32_do_vec128( + uint32x4_t numers, const struct libdivide_u32_t *denom); +static inline int32x4_t libdivide_s32_do_vec128( + int32x4_t numers, const struct libdivide_s32_t *denom); +static inline uint64x2_t libdivide_u64_do_vec128( + uint64x2_t numers, const struct libdivide_u64_t *denom); +static inline int64x2_t libdivide_s64_do_vec128( + int64x2_t numers, const struct libdivide_s64_t *denom); -static inline __m512i libdivide_u32_branchfree_do_vector(__m512i numers, const struct libdivide_u32_branchfree_t *denom); -static inline __m512i libdivide_s32_branchfree_do_vector(__m512i numers, const struct libdivide_s32_branchfree_t *denom); -static inline __m512i libdivide_u64_branchfree_do_vector(__m512i numers, const struct libdivide_u64_branchfree_t *denom); -static inline __m512i libdivide_s64_branchfree_do_vector(__m512i numers, const struct libdivide_s64_branchfree_t *denom); +static inline uint32x4_t libdivide_u32_branchfree_do_vec128( + uint32x4_t numers, const struct libdivide_u32_branchfree_t *denom); +static inline int32x4_t libdivide_s32_branchfree_do_vec128( + int32x4_t numers, const struct libdivide_s32_branchfree_t *denom); +static inline uint64x2_t libdivide_u64_branchfree_do_vec128( + uint64x2_t numers, const struct libdivide_u64_branchfree_t *denom); +static inline int64x2_t libdivide_s64_branchfree_do_vec128( + int64x2_t numers, const struct libdivide_s64_branchfree_t *denom); //////// Internal Utility Functions -static inline __m512i libdivide_s64_signbits(__m512i v) {; +// Logical right shift by runtime value. +// NEON implements right shift as left shits by negative values. +static inline uint32x4_t libdivide_u32_neon_srl(uint32x4_t v, uint8_t amt) { + int32_t wamt = static_cast(amt); + return vshlq_u32(v, vdupq_n_s32(-wamt)); +} + +static inline uint64x2_t libdivide_u64_neon_srl(uint64x2_t v, uint8_t amt) { + int64_t wamt = static_cast(amt); + return vshlq_u64(v, vdupq_n_s64(-wamt)); +} + +// Arithmetic right shift by runtime value. +static inline int32x4_t libdivide_s32_neon_sra(int32x4_t v, uint8_t amt) { + int32_t wamt = static_cast(amt); + return vshlq_s32(v, vdupq_n_s32(-wamt)); +} + +static inline int64x2_t libdivide_s64_neon_sra(int64x2_t v, uint8_t amt) { + int64_t wamt = static_cast(amt); + return vshlq_s64(v, vdupq_n_s64(-wamt)); +} + +static inline int64x2_t libdivide_s64_signbits(int64x2_t v) { return vshrq_n_s64(v, 63); } + +static inline uint32x4_t libdivide_mullhi_u32_vec128(uint32x4_t a, uint32_t b) { + // Desire is [x0, x1, x2, x3] + uint32x4_t w1 = vreinterpretq_u32_u64(vmull_n_u32(vget_low_u32(a), b)); // [_, x0, _, x1] + uint32x4_t w2 = vreinterpretq_u32_u64(vmull_high_n_u32(a, b)); //[_, x2, _, x3] + return vuzp2q_u32(w1, w2); // [x0, x1, x2, x3] +} + +static inline int32x4_t libdivide_mullhi_s32_vec128(int32x4_t a, int32_t b) { + int32x4_t w1 = vreinterpretq_s32_s64(vmull_n_s32(vget_low_s32(a), b)); // [_, x0, _, x1] + int32x4_t w2 = vreinterpretq_s32_s64(vmull_high_n_s32(a, b)); //[_, x2, _, x3] + return vuzp2q_s32(w1, w2); // [x0, x1, x2, x3] +} + +static inline uint64x2_t libdivide_mullhi_u64_vec128(uint64x2_t x, uint64_t sy) { + // full 128 bits product is: + // x0*y0 + (x0*y1 << 32) + (x1*y0 << 32) + (x1*y1 << 64) + // Note x0,y0,x1,y1 are all conceptually uint32, products are 32x32->64. + + // Get low and high words. x0 contains low 32 bits, x1 is high 32 bits. + uint64x2_t y = vdupq_n_u64(sy); + uint32x2_t x0 = vmovn_u64(x); + uint32x2_t y0 = vmovn_u64(y); + uint32x2_t x1 = vshrn_n_u64(x, 32); + uint32x2_t y1 = vshrn_n_u64(y, 32); + + // Compute x0*y0. + uint64x2_t x0y0 = vmull_u32(x0, y0); + uint64x2_t x0y0_hi = vshrq_n_u64(x0y0, 32); + + // Compute other intermediate products. + uint64x2_t temp = vmlal_u32(x0y0_hi, x1, y0); // temp = x0y0_hi + x1*y0; + // We want to split temp into its low 32 bits and high 32 bits, both + // in the low half of 64 bit registers. + // Use shifts to avoid needing a reg for the mask. + uint64x2_t temp_lo = vshrq_n_u64(vshlq_n_u64(temp, 32), 32); // temp_lo = temp & 0xFFFFFFFF; + uint64x2_t temp_hi = vshrq_n_u64(temp, 32); // temp_hi = temp >> 32; + + temp_lo = vmlal_u32(temp_lo, x0, y1); // temp_lo += x0*y0 + temp_lo = vshrq_n_u64(temp_lo, 32); // temp_lo >>= 32 + temp_hi = vmlal_u32(temp_hi, x1, y1); // temp_hi += x1*y1 + uint64x2_t result = vaddq_u64(temp_hi, temp_lo); + return result; +} + +static inline int64x2_t libdivide_mullhi_s64_vec128(int64x2_t x, int64_t sy) { + int64x2_t p = vreinterpretq_s64_u64( + libdivide_mullhi_u64_vec128(vreinterpretq_u64_s64(x), static_cast(sy))); + int64x2_t y = vdupq_n_s64(sy); + int64x2_t t1 = vandq_s64(libdivide_s64_signbits(x), y); + int64x2_t t2 = vandq_s64(libdivide_s64_signbits(y), x); + p = vsubq_s64(p, t1); + p = vsubq_s64(p, t2); + return p; +} + +////////// UINT32 + +uint32x4_t libdivide_u32_do_vec128(uint32x4_t numers, const struct libdivide_u32_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + return libdivide_u32_neon_srl(numers, more); + } else { + uint32x4_t q = libdivide_mullhi_u32_vec128(numers, denom->magic); + if (more & LIBDIVIDE_ADD_MARKER) { + // uint32_t t = ((numer - q) >> 1) + q; + // return t >> denom->shift; + // Note we can use halving-subtract to avoid the shift. + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + uint32x4_t t = vaddq_u32(vhsubq_u32(numers, q), q); + return libdivide_u32_neon_srl(t, shift); + } else { + return libdivide_u32_neon_srl(q, more); + } + } +} + +uint32x4_t libdivide_u32_branchfree_do_vec128( + uint32x4_t numers, const struct libdivide_u32_branchfree_t *denom) { + uint32x4_t q = libdivide_mullhi_u32_vec128(numers, denom->magic); + uint32x4_t t = vaddq_u32(vhsubq_u32(numers, q), q); + return libdivide_u32_neon_srl(t, denom->more); +} + +////////// UINT64 + +uint64x2_t libdivide_u64_do_vec128(uint64x2_t numers, const struct libdivide_u64_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + return libdivide_u64_neon_srl(numers, more); + } else { + uint64x2_t q = libdivide_mullhi_u64_vec128(numers, denom->magic); + if (more & LIBDIVIDE_ADD_MARKER) { + // uint32_t t = ((numer - q) >> 1) + q; + // return t >> denom->shift; + // No 64-bit halving subtracts in NEON :( + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + uint64x2_t t = vaddq_u64(vshrq_n_u64(vsubq_u64(numers, q), 1), q); + return libdivide_u64_neon_srl(t, shift); + } else { + return libdivide_u64_neon_srl(q, more); + } + } +} + +uint64x2_t libdivide_u64_branchfree_do_vec128( + uint64x2_t numers, const struct libdivide_u64_branchfree_t *denom) { + uint64x2_t q = libdivide_mullhi_u64_vec128(numers, denom->magic); + uint64x2_t t = vaddq_u64(vshrq_n_u64(vsubq_u64(numers, q), 1), q); + return libdivide_u64_neon_srl(t, denom->more); +} + +////////// SINT32 + +int32x4_t libdivide_s32_do_vec128(int32x4_t numers, const struct libdivide_s32_t *denom) { + uint8_t more = denom->more; + if (!denom->magic) { + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + uint32_t mask = (1U << shift) - 1; + int32x4_t roundToZeroTweak = vdupq_n_s32((int)mask); + // q = numer + ((numer >> 31) & roundToZeroTweak); + int32x4_t q = vaddq_s32(numers, vandq_s32(vshrq_n_s32(numers, 31), roundToZeroTweak)); + q = libdivide_s32_neon_sra(q, shift); + int32x4_t sign = vdupq_n_s32((int8_t)more >> 7); + // q = (q ^ sign) - sign; + q = vsubq_s32(veorq_s32(q, sign), sign); + return q; + } else { + int32x4_t q = libdivide_mullhi_s32_vec128(numers, denom->magic); + if (more & LIBDIVIDE_ADD_MARKER) { + // must be arithmetic shift + int32x4_t sign = vdupq_n_s32((int8_t)more >> 7); + // q += ((numer ^ sign) - sign); + q = vaddq_s32(q, vsubq_s32(veorq_s32(numers, sign), sign)); + } + // q >>= shift + q = libdivide_s32_neon_sra(q, more & LIBDIVIDE_32_SHIFT_MASK); + q = vaddq_s32( + q, vreinterpretq_s32_u32(vshrq_n_u32(vreinterpretq_u32_s32(q), 31))); // q += (q < 0) + return q; + } +} + +int32x4_t libdivide_s32_branchfree_do_vec128( + int32x4_t numers, const struct libdivide_s32_branchfree_t *denom) { + int32_t magic = denom->magic; + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; + // must be arithmetic shift + int32x4_t sign = vdupq_n_s32((int8_t)more >> 7); + int32x4_t q = libdivide_mullhi_s32_vec128(numers, magic); + q = vaddq_s32(q, numers); // q += numers + + // If q is non-negative, we have nothing to do + // If q is negative, we want to add either (2**shift)-1 if d is + // a power of 2, or (2**shift) if it is not a power of 2 + uint32_t is_power_of_2 = (magic == 0); + int32x4_t q_sign = vshrq_n_s32(q, 31); // q_sign = q >> 31 + int32x4_t mask = vdupq_n_s32((1U << shift) - is_power_of_2); + q = vaddq_s32(q, vandq_s32(q_sign, mask)); // q = q + (q_sign & mask) + q = libdivide_s32_neon_sra(q, shift); // q >>= shift + q = vsubq_s32(veorq_s32(q, sign), sign); // q = (q ^ sign) - sign + return q; +} + +////////// SINT64 + +int64x2_t libdivide_s64_do_vec128(int64x2_t numers, const struct libdivide_s64_t *denom) { + uint8_t more = denom->more; + int64_t magic = denom->magic; + if (magic == 0) { // shift path + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + uint64_t mask = (1ULL << shift) - 1; + int64x2_t roundToZeroTweak = vdupq_n_s64(mask); // TODO: no need to sign extend + // q = numer + ((numer >> 63) & roundToZeroTweak); + int64x2_t q = + vaddq_s64(numers, vandq_s64(libdivide_s64_signbits(numers), roundToZeroTweak)); + q = libdivide_s64_neon_sra(q, shift); + // q = (q ^ sign) - sign; + int64x2_t sign = vreinterpretq_s64_s8(vdupq_n_s8((int8_t)more >> 7)); + q = vsubq_s64(veorq_s64(q, sign), sign); + return q; + } else { + int64x2_t q = libdivide_mullhi_s64_vec128(numers, magic); + if (more & LIBDIVIDE_ADD_MARKER) { + // must be arithmetic shift + int64x2_t sign = vdupq_n_s64((int8_t)more >> 7); // TODO: no need to widen + // q += ((numer ^ sign) - sign); + q = vaddq_s64(q, vsubq_s64(veorq_s64(numers, sign), sign)); + } + // q >>= denom->mult_path.shift + q = libdivide_s64_neon_sra(q, more & LIBDIVIDE_64_SHIFT_MASK); + q = vaddq_s64( + q, vreinterpretq_s64_u64(vshrq_n_u64(vreinterpretq_u64_s64(q), 63))); // q += (q < 0) + return q; + } +} + +int64x2_t libdivide_s64_branchfree_do_vec128( + int64x2_t numers, const struct libdivide_s64_branchfree_t *denom) { + int64_t magic = denom->magic; + uint8_t more = denom->more; + uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; + // must be arithmetic shift + int64x2_t sign = vdupq_n_s64((int8_t)more >> 7); // TODO: avoid sign extend + + // libdivide_mullhi_s64(numers, magic); + int64x2_t q = libdivide_mullhi_s64_vec128(numers, magic); + q = vaddq_s64(q, numers); // q += numers + + // If q is non-negative, we have nothing to do. + // If q is negative, we want to add either (2**shift)-1 if d is + // a power of 2, or (2**shift) if it is not a power of 2. + uint32_t is_power_of_2 = (magic == 0); + int64x2_t q_sign = libdivide_s64_signbits(q); // q_sign = q >> 63 + int64x2_t mask = vdupq_n_s64((1ULL << shift) - is_power_of_2); + q = vaddq_s64(q, vandq_s64(q_sign, mask)); // q = q + (q_sign & mask) + q = libdivide_s64_neon_sra(q, shift); // q >>= shift + q = vsubq_s64(veorq_s64(q, sign), sign); // q = (q ^ sign) - sign + return q; +} + +#endif + +#if defined(LIBDIVIDE_AVX512) + +static inline __m512i libdivide_u32_do_vec512(__m512i numers, const struct libdivide_u32_t *denom); +static inline __m512i libdivide_s32_do_vec512(__m512i numers, const struct libdivide_s32_t *denom); +static inline __m512i libdivide_u64_do_vec512(__m512i numers, const struct libdivide_u64_t *denom); +static inline __m512i libdivide_s64_do_vec512(__m512i numers, const struct libdivide_s64_t *denom); + +static inline __m512i libdivide_u32_branchfree_do_vec512( + __m512i numers, const struct libdivide_u32_branchfree_t *denom); +static inline __m512i libdivide_s32_branchfree_do_vec512( + __m512i numers, const struct libdivide_s32_branchfree_t *denom); +static inline __m512i libdivide_u64_branchfree_do_vec512( + __m512i numers, const struct libdivide_u64_branchfree_t *denom); +static inline __m512i libdivide_s64_branchfree_do_vec512( + __m512i numers, const struct libdivide_s64_branchfree_t *denom); + +//////// Internal Utility Functions + +static inline __m512i libdivide_s64_signbits(__m512i v) { + ; return _mm512_srai_epi64(v, 63); } -static inline __m512i libdivide_s64_shift_right_vector(__m512i v, int amt) { +static inline __m512i libdivide_s64_shift_right_vec512(__m512i v, int amt) { return _mm512_srai_epi64(v, amt); } // Here, b is assumed to contain one 32-bit value repeated. -static inline __m512i libdivide_mullhi_u32_vector(__m512i a, __m512i b) { +static inline __m512i libdivide_mullhi_u32_vec512(__m512i a, __m512i b) { __m512i hi_product_0Z2Z = _mm512_srli_epi64(_mm512_mul_epu32(a, b), 32); __m512i a1X3X = _mm512_srli_epi64(a, 32); __m512i mask = _mm512_set_epi32(-1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0); @@ -1238,7 +1507,7 @@ static inline __m512i libdivide_mullhi_u32_vector(__m512i a, __m512i b) { } // b is one 32-bit value repeated. -static inline __m512i libdivide_mullhi_s32_vector(__m512i a, __m512i b) { +static inline __m512i libdivide_mullhi_s32_vec512(__m512i a, __m512i b) { __m512i hi_product_0Z2Z = _mm512_srli_epi64(_mm512_mul_epi32(a, b), 32); __m512i a1X3X = _mm512_srli_epi64(a, 32); __m512i mask = _mm512_set_epi32(-1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0); @@ -1247,30 +1516,31 @@ static inline __m512i libdivide_mullhi_s32_vector(__m512i a, __m512i b) { } // Here, y is assumed to contain one 64-bit value repeated. -// https://stackoverflow.com/a/28827013 -static inline __m512i libdivide_mullhi_u64_vector(__m512i x, __m512i y) { - __m512i lomask = _mm512_set1_epi64(0xffffffff); - __m512i xh = _mm512_shuffle_epi32(x, (_MM_PERM_ENUM) 0xB1); - __m512i yh = _mm512_shuffle_epi32(y, (_MM_PERM_ENUM) 0xB1); - __m512i w0 = _mm512_mul_epu32(x, y); - __m512i w1 = _mm512_mul_epu32(x, yh); - __m512i w2 = _mm512_mul_epu32(xh, y); - __m512i w3 = _mm512_mul_epu32(xh, yh); - __m512i w0h = _mm512_srli_epi64(w0, 32); - __m512i s1 = _mm512_add_epi64(w1, w0h); - __m512i s1l = _mm512_and_si512(s1, lomask); - __m512i s1h = _mm512_srli_epi64(s1, 32); - __m512i s2 = _mm512_add_epi64(w2, s1l); - __m512i s2h = _mm512_srli_epi64(s2, 32); - __m512i hi = _mm512_add_epi64(w3, s1h); - hi = _mm512_add_epi64(hi, s2h); +static inline __m512i libdivide_mullhi_u64_vec512(__m512i x, __m512i y) { + // see m128i variant for comments. + __m512i x0y0 = _mm512_mul_epu32(x, y); + __m512i x0y0_hi = _mm512_srli_epi64(x0y0, 32); - return hi; + __m512i x1 = _mm512_shuffle_epi32(x, (_MM_PERM_ENUM)_MM_SHUFFLE(3, 3, 1, 1)); + __m512i y1 = _mm512_shuffle_epi32(y, (_MM_PERM_ENUM)_MM_SHUFFLE(3, 3, 1, 1)); + + __m512i x0y1 = _mm512_mul_epu32(x, y1); + __m512i x1y0 = _mm512_mul_epu32(x1, y); + __m512i x1y1 = _mm512_mul_epu32(x1, y1); + + __m512i mask = _mm512_set1_epi64(0xFFFFFFFF); + __m512i temp = _mm512_add_epi64(x1y0, x0y0_hi); + __m512i temp_lo = _mm512_and_si512(temp, mask); + __m512i temp_hi = _mm512_srli_epi64(temp, 32); + + temp_lo = _mm512_srli_epi64(_mm512_add_epi64(temp_lo, x0y1), 32); + temp_hi = _mm512_add_epi64(x1y1, temp_hi); + return _mm512_add_epi64(temp_lo, temp_hi); } // y is one 64-bit value repeated. -static inline __m512i libdivide_mullhi_s64_vector(__m512i x, __m512i y) { - __m512i p = libdivide_mullhi_u64_vector(x, y); +static inline __m512i libdivide_mullhi_s64_vec512(__m512i x, __m512i y) { + __m512i p = libdivide_mullhi_u64_vec512(x, y); __m512i t1 = _mm512_and_si512(libdivide_s64_signbits(x), y); __m512i t2 = _mm512_and_si512(libdivide_s64_signbits(y), x); p = _mm512_sub_epi64(p, t1); @@ -1280,131 +1550,130 @@ static inline __m512i libdivide_mullhi_s64_vector(__m512i x, __m512i y) { ////////// UINT32 -__m512i libdivide_u32_do_vector(__m512i numers, const struct libdivide_u32_t *denom) { +__m512i libdivide_u32_do_vec512(__m512i numers, const struct libdivide_u32_t *denom) { uint8_t more = denom->more; if (!denom->magic) { return _mm512_srli_epi32(numers, more); - } - else { - __m512i q = libdivide_mullhi_u32_vector(numers, _mm512_set1_epi32(denom->magic)); + } else { + __m512i q = libdivide_mullhi_u32_vec512(numers, _mm512_set1_epi32(denom->magic)); if (more & LIBDIVIDE_ADD_MARKER) { // uint32_t t = ((numer - q) >> 1) + q; // return t >> denom->shift; uint32_t shift = more & LIBDIVIDE_32_SHIFT_MASK; __m512i t = _mm512_add_epi32(_mm512_srli_epi32(_mm512_sub_epi32(numers, q), 1), q); return _mm512_srli_epi32(t, shift); - } - else { + } else { return _mm512_srli_epi32(q, more); } } } -__m512i libdivide_u32_branchfree_do_vector(__m512i numers, const struct libdivide_u32_branchfree_t *denom) { - __m512i q = libdivide_mullhi_u32_vector(numers, _mm512_set1_epi32(denom->magic)); +__m512i libdivide_u32_branchfree_do_vec512( + __m512i numers, const struct libdivide_u32_branchfree_t *denom) { + __m512i q = libdivide_mullhi_u32_vec512(numers, _mm512_set1_epi32(denom->magic)); __m512i t = _mm512_add_epi32(_mm512_srli_epi32(_mm512_sub_epi32(numers, q), 1), q); return _mm512_srli_epi32(t, denom->more); } ////////// UINT64 -__m512i libdivide_u64_do_vector(__m512i numers, const struct libdivide_u64_t *denom) { +__m512i libdivide_u64_do_vec512(__m512i numers, const struct libdivide_u64_t *denom) { uint8_t more = denom->more; if (!denom->magic) { return _mm512_srli_epi64(numers, more); - } - else { - __m512i q = libdivide_mullhi_u64_vector(numers, _mm512_set1_epi64(denom->magic)); + } else { + __m512i q = libdivide_mullhi_u64_vec512(numers, _mm512_set1_epi64(denom->magic)); if (more & LIBDIVIDE_ADD_MARKER) { // uint32_t t = ((numer - q) >> 1) + q; // return t >> denom->shift; uint32_t shift = more & LIBDIVIDE_64_SHIFT_MASK; __m512i t = _mm512_add_epi64(_mm512_srli_epi64(_mm512_sub_epi64(numers, q), 1), q); return _mm512_srli_epi64(t, shift); - } - else { + } else { return _mm512_srli_epi64(q, more); } } } -__m512i libdivide_u64_branchfree_do_vector(__m512i numers, const struct libdivide_u64_branchfree_t *denom) { - __m512i q = libdivide_mullhi_u64_vector(numers, _mm512_set1_epi64(denom->magic)); +__m512i libdivide_u64_branchfree_do_vec512( + __m512i numers, const struct libdivide_u64_branchfree_t *denom) { + __m512i q = libdivide_mullhi_u64_vec512(numers, _mm512_set1_epi64(denom->magic)); __m512i t = _mm512_add_epi64(_mm512_srli_epi64(_mm512_sub_epi64(numers, q), 1), q); return _mm512_srli_epi64(t, denom->more); } ////////// SINT32 -__m512i libdivide_s32_do_vector(__m512i numers, const struct libdivide_s32_t *denom) { +__m512i libdivide_s32_do_vec512(__m512i numers, const struct libdivide_s32_t *denom) { uint8_t more = denom->more; if (!denom->magic) { uint32_t shift = more & LIBDIVIDE_32_SHIFT_MASK; uint32_t mask = (1U << shift) - 1; __m512i roundToZeroTweak = _mm512_set1_epi32(mask); // q = numer + ((numer >> 31) & roundToZeroTweak); - __m512i q = _mm512_add_epi32(numers, _mm512_and_si512(_mm512_srai_epi32(numers, 31), roundToZeroTweak)); + __m512i q = _mm512_add_epi32( + numers, _mm512_and_si512(_mm512_srai_epi32(numers, 31), roundToZeroTweak)); q = _mm512_srai_epi32(q, shift); __m512i sign = _mm512_set1_epi32((int8_t)more >> 7); // q = (q ^ sign) - sign; q = _mm512_sub_epi32(_mm512_xor_si512(q, sign), sign); return q; - } - else { - __m512i q = libdivide_mullhi_s32_vector(numers, _mm512_set1_epi32(denom->magic)); + } else { + __m512i q = libdivide_mullhi_s32_vec512(numers, _mm512_set1_epi32(denom->magic)); if (more & LIBDIVIDE_ADD_MARKER) { - // must be arithmetic shift + // must be arithmetic shift __m512i sign = _mm512_set1_epi32((int8_t)more >> 7); - // q += ((numer ^ sign) - sign); + // q += ((numer ^ sign) - sign); q = _mm512_add_epi32(q, _mm512_sub_epi32(_mm512_xor_si512(numers, sign), sign)); } // q >>= shift q = _mm512_srai_epi32(q, more & LIBDIVIDE_32_SHIFT_MASK); - q = _mm512_add_epi32(q, _mm512_srli_epi32(q, 31)); // q += (q < 0) + q = _mm512_add_epi32(q, _mm512_srli_epi32(q, 31)); // q += (q < 0) return q; } } -__m512i libdivide_s32_branchfree_do_vector(__m512i numers, const struct libdivide_s32_branchfree_t *denom) { +__m512i libdivide_s32_branchfree_do_vec512( + __m512i numers, const struct libdivide_s32_branchfree_t *denom) { int32_t magic = denom->magic; uint8_t more = denom->more; uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; - // must be arithmetic shift + // must be arithmetic shift __m512i sign = _mm512_set1_epi32((int8_t)more >> 7); - __m512i q = libdivide_mullhi_s32_vector(numers, _mm512_set1_epi32(magic)); - q = _mm512_add_epi32(q, numers); // q += numers + __m512i q = libdivide_mullhi_s32_vec512(numers, _mm512_set1_epi32(magic)); + q = _mm512_add_epi32(q, numers); // q += numers // If q is non-negative, we have nothing to do // If q is negative, we want to add either (2**shift)-1 if d is // a power of 2, or (2**shift) if it is not a power of 2 uint32_t is_power_of_2 = (magic == 0); - __m512i q_sign = _mm512_srai_epi32(q, 31); // q_sign = q >> 31 + __m512i q_sign = _mm512_srai_epi32(q, 31); // q_sign = q >> 31 __m512i mask = _mm512_set1_epi32((1U << shift) - is_power_of_2); - q = _mm512_add_epi32(q, _mm512_and_si512(q_sign, mask)); // q = q + (q_sign & mask) - q = _mm512_srai_epi32(q, shift); // q >>= shift - q = _mm512_sub_epi32(_mm512_xor_si512(q, sign), sign); // q = (q ^ sign) - sign + q = _mm512_add_epi32(q, _mm512_and_si512(q_sign, mask)); // q = q + (q_sign & mask) + q = _mm512_srai_epi32(q, shift); // q >>= shift + q = _mm512_sub_epi32(_mm512_xor_si512(q, sign), sign); // q = (q ^ sign) - sign return q; } ////////// SINT64 -__m512i libdivide_s64_do_vector(__m512i numers, const struct libdivide_s64_t *denom) { +__m512i libdivide_s64_do_vec512(__m512i numers, const struct libdivide_s64_t *denom) { uint8_t more = denom->more; int64_t magic = denom->magic; - if (magic == 0) { // shift path + if (magic == 0) { // shift path uint32_t shift = more & LIBDIVIDE_64_SHIFT_MASK; uint64_t mask = (1ULL << shift) - 1; __m512i roundToZeroTweak = _mm512_set1_epi64(mask); // q = numer + ((numer >> 63) & roundToZeroTweak); - __m512i q = _mm512_add_epi64(numers, _mm512_and_si512(libdivide_s64_signbits(numers), roundToZeroTweak)); - q = libdivide_s64_shift_right_vector(q, shift); + __m512i q = _mm512_add_epi64( + numers, _mm512_and_si512(libdivide_s64_signbits(numers), roundToZeroTweak)); + q = libdivide_s64_shift_right_vec512(q, shift); __m512i sign = _mm512_set1_epi32((int8_t)more >> 7); - // q = (q ^ sign) - sign; + // q = (q ^ sign) - sign; q = _mm512_sub_epi64(_mm512_xor_si512(q, sign), sign); return q; - } - else { - __m512i q = libdivide_mullhi_s64_vector(numers, _mm512_set1_epi64(magic)); + } else { + __m512i q = libdivide_mullhi_s64_vec512(numers, _mm512_set1_epi64(magic)); if (more & LIBDIVIDE_ADD_MARKER) { // must be arithmetic shift __m512i sign = _mm512_set1_epi32((int8_t)more >> 7); @@ -1412,46 +1681,53 @@ __m512i libdivide_s64_do_vector(__m512i numers, const struct libdivide_s64_t *de q = _mm512_add_epi64(q, _mm512_sub_epi64(_mm512_xor_si512(numers, sign), sign)); } // q >>= denom->mult_path.shift - q = libdivide_s64_shift_right_vector(q, more & LIBDIVIDE_64_SHIFT_MASK); - q = _mm512_add_epi64(q, _mm512_srli_epi64(q, 63)); // q += (q < 0) + q = libdivide_s64_shift_right_vec512(q, more & LIBDIVIDE_64_SHIFT_MASK); + q = _mm512_add_epi64(q, _mm512_srli_epi64(q, 63)); // q += (q < 0) return q; } } -__m512i libdivide_s64_branchfree_do_vector(__m512i numers, const struct libdivide_s64_branchfree_t *denom) { +__m512i libdivide_s64_branchfree_do_vec512( + __m512i numers, const struct libdivide_s64_branchfree_t *denom) { int64_t magic = denom->magic; uint8_t more = denom->more; uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; // must be arithmetic shift __m512i sign = _mm512_set1_epi32((int8_t)more >> 7); - // libdivide_mullhi_s64(numers, magic); - __m512i q = libdivide_mullhi_s64_vector(numers, _mm512_set1_epi64(magic)); - q = _mm512_add_epi64(q, numers); // q += numers + // libdivide_mullhi_s64(numers, magic); + __m512i q = libdivide_mullhi_s64_vec512(numers, _mm512_set1_epi64(magic)); + q = _mm512_add_epi64(q, numers); // q += numers // If q is non-negative, we have nothing to do. // If q is negative, we want to add either (2**shift)-1 if d is // a power of 2, or (2**shift) if it is not a power of 2. uint32_t is_power_of_2 = (magic == 0); - __m512i q_sign = libdivide_s64_signbits(q); // q_sign = q >> 63 + __m512i q_sign = libdivide_s64_signbits(q); // q_sign = q >> 63 __m512i mask = _mm512_set1_epi64((1ULL << shift) - is_power_of_2); - q = _mm512_add_epi64(q, _mm512_and_si512(q_sign, mask)); // q = q + (q_sign & mask) - q = libdivide_s64_shift_right_vector(q, shift); // q >>= shift - q = _mm512_sub_epi64(_mm512_xor_si512(q, sign), sign); // q = (q ^ sign) - sign + q = _mm512_add_epi64(q, _mm512_and_si512(q_sign, mask)); // q = q + (q_sign & mask) + q = libdivide_s64_shift_right_vec512(q, shift); // q >>= shift + q = _mm512_sub_epi64(_mm512_xor_si512(q, sign), sign); // q = (q ^ sign) - sign return q; } -#elif defined(LIBDIVIDE_AVX2) +#endif -static inline __m256i libdivide_u32_do_vector(__m256i numers, const struct libdivide_u32_t *denom); -static inline __m256i libdivide_s32_do_vector(__m256i numers, const struct libdivide_s32_t *denom); -static inline __m256i libdivide_u64_do_vector(__m256i numers, const struct libdivide_u64_t *denom); -static inline __m256i libdivide_s64_do_vector(__m256i numers, const struct libdivide_s64_t *denom); +#if defined(LIBDIVIDE_AVX2) -static inline __m256i libdivide_u32_branchfree_do_vector(__m256i numers, const struct libdivide_u32_branchfree_t *denom); -static inline __m256i libdivide_s32_branchfree_do_vector(__m256i numers, const struct libdivide_s32_branchfree_t *denom); -static inline __m256i libdivide_u64_branchfree_do_vector(__m256i numers, const struct libdivide_u64_branchfree_t *denom); -static inline __m256i libdivide_s64_branchfree_do_vector(__m256i numers, const struct libdivide_s64_branchfree_t *denom); +static inline __m256i libdivide_u32_do_vec256(__m256i numers, const struct libdivide_u32_t *denom); +static inline __m256i libdivide_s32_do_vec256(__m256i numers, const struct libdivide_s32_t *denom); +static inline __m256i libdivide_u64_do_vec256(__m256i numers, const struct libdivide_u64_t *denom); +static inline __m256i libdivide_s64_do_vec256(__m256i numers, const struct libdivide_s64_t *denom); + +static inline __m256i libdivide_u32_branchfree_do_vec256( + __m256i numers, const struct libdivide_u32_branchfree_t *denom); +static inline __m256i libdivide_s32_branchfree_do_vec256( + __m256i numers, const struct libdivide_s32_branchfree_t *denom); +static inline __m256i libdivide_u64_branchfree_do_vec256( + __m256i numers, const struct libdivide_u64_branchfree_t *denom); +static inline __m256i libdivide_s64_branchfree_do_vec256( + __m256i numers, const struct libdivide_s64_branchfree_t *denom); //////// Internal Utility Functions @@ -1463,7 +1739,7 @@ static inline __m256i libdivide_s64_signbits(__m256i v) { } // Implementation of _mm256_srai_epi64 (from AVX512). -static inline __m256i libdivide_s64_shift_right_vector(__m256i v, int amt) { +static inline __m256i libdivide_s64_shift_right_vec256(__m256i v, int amt) { const int b = 64 - amt; __m256i m = _mm256_set1_epi64x(1ULL << (b - 1)); __m256i x = _mm256_srli_epi64(v, amt); @@ -1472,7 +1748,7 @@ static inline __m256i libdivide_s64_shift_right_vector(__m256i v, int amt) { } // Here, b is assumed to contain one 32-bit value repeated. -static inline __m256i libdivide_mullhi_u32_vector(__m256i a, __m256i b) { +static inline __m256i libdivide_mullhi_u32_vec256(__m256i a, __m256i b) { __m256i hi_product_0Z2Z = _mm256_srli_epi64(_mm256_mul_epu32(a, b), 32); __m256i a1X3X = _mm256_srli_epi64(a, 32); __m256i mask = _mm256_set_epi32(-1, 0, -1, 0, -1, 0, -1, 0); @@ -1481,7 +1757,7 @@ static inline __m256i libdivide_mullhi_u32_vector(__m256i a, __m256i b) { } // b is one 32-bit value repeated. -static inline __m256i libdivide_mullhi_s32_vector(__m256i a, __m256i b) { +static inline __m256i libdivide_mullhi_s32_vec256(__m256i a, __m256i b) { __m256i hi_product_0Z2Z = _mm256_srli_epi64(_mm256_mul_epi32(a, b), 32); __m256i a1X3X = _mm256_srli_epi64(a, 32); __m256i mask = _mm256_set_epi32(-1, 0, -1, 0, -1, 0, -1, 0); @@ -1490,30 +1766,31 @@ static inline __m256i libdivide_mullhi_s32_vector(__m256i a, __m256i b) { } // Here, y is assumed to contain one 64-bit value repeated. -// https://stackoverflow.com/a/28827013 -static inline __m256i libdivide_mullhi_u64_vector(__m256i x, __m256i y) { - __m256i lomask = _mm256_set1_epi64x(0xffffffff); - __m256i xh = _mm256_shuffle_epi32(x, 0xB1); // x0l, x0h, x1l, x1h - __m256i yh = _mm256_shuffle_epi32(y, 0xB1); // y0l, y0h, y1l, y1h - __m256i w0 = _mm256_mul_epu32(x, y); // x0l*y0l, x1l*y1l - __m256i w1 = _mm256_mul_epu32(x, yh); // x0l*y0h, x1l*y1h - __m256i w2 = _mm256_mul_epu32(xh, y); // x0h*y0l, x1h*y0l - __m256i w3 = _mm256_mul_epu32(xh, yh); // x0h*y0h, x1h*y1h - __m256i w0h = _mm256_srli_epi64(w0, 32); - __m256i s1 = _mm256_add_epi64(w1, w0h); - __m256i s1l = _mm256_and_si256(s1, lomask); - __m256i s1h = _mm256_srli_epi64(s1, 32); - __m256i s2 = _mm256_add_epi64(w2, s1l); - __m256i s2h = _mm256_srli_epi64(s2, 32); - __m256i hi = _mm256_add_epi64(w3, s1h); - hi = _mm256_add_epi64(hi, s2h); +static inline __m256i libdivide_mullhi_u64_vec256(__m256i x, __m256i y) { + // see m128i variant for comments. + __m256i x0y0 = _mm256_mul_epu32(x, y); + __m256i x0y0_hi = _mm256_srli_epi64(x0y0, 32); - return hi; + __m256i x1 = _mm256_shuffle_epi32(x, _MM_SHUFFLE(3, 3, 1, 1)); + __m256i y1 = _mm256_shuffle_epi32(y, _MM_SHUFFLE(3, 3, 1, 1)); + + __m256i x0y1 = _mm256_mul_epu32(x, y1); + __m256i x1y0 = _mm256_mul_epu32(x1, y); + __m256i x1y1 = _mm256_mul_epu32(x1, y1); + + __m256i mask = _mm256_set1_epi64x(0xFFFFFFFF); + __m256i temp = _mm256_add_epi64(x1y0, x0y0_hi); + __m256i temp_lo = _mm256_and_si256(temp, mask); + __m256i temp_hi = _mm256_srli_epi64(temp, 32); + + temp_lo = _mm256_srli_epi64(_mm256_add_epi64(temp_lo, x0y1), 32); + temp_hi = _mm256_add_epi64(x1y1, temp_hi); + return _mm256_add_epi64(temp_lo, temp_hi); } // y is one 64-bit value repeated. -static inline __m256i libdivide_mullhi_s64_vector(__m256i x, __m256i y) { - __m256i p = libdivide_mullhi_u64_vector(x, y); +static inline __m256i libdivide_mullhi_s64_vec256(__m256i x, __m256i y) { + __m256i p = libdivide_mullhi_u64_vec256(x, y); __m256i t1 = _mm256_and_si256(libdivide_s64_signbits(x), y); __m256i t2 = _mm256_and_si256(libdivide_s64_signbits(y), x); p = _mm256_sub_epi64(p, t1); @@ -1523,131 +1800,130 @@ static inline __m256i libdivide_mullhi_s64_vector(__m256i x, __m256i y) { ////////// UINT32 -__m256i libdivide_u32_do_vector(__m256i numers, const struct libdivide_u32_t *denom) { +__m256i libdivide_u32_do_vec256(__m256i numers, const struct libdivide_u32_t *denom) { uint8_t more = denom->more; if (!denom->magic) { return _mm256_srli_epi32(numers, more); - } - else { - __m256i q = libdivide_mullhi_u32_vector(numers, _mm256_set1_epi32(denom->magic)); + } else { + __m256i q = libdivide_mullhi_u32_vec256(numers, _mm256_set1_epi32(denom->magic)); if (more & LIBDIVIDE_ADD_MARKER) { // uint32_t t = ((numer - q) >> 1) + q; // return t >> denom->shift; uint32_t shift = more & LIBDIVIDE_32_SHIFT_MASK; __m256i t = _mm256_add_epi32(_mm256_srli_epi32(_mm256_sub_epi32(numers, q), 1), q); return _mm256_srli_epi32(t, shift); - } - else { + } else { return _mm256_srli_epi32(q, more); } } } -__m256i libdivide_u32_branchfree_do_vector(__m256i numers, const struct libdivide_u32_branchfree_t *denom) { - __m256i q = libdivide_mullhi_u32_vector(numers, _mm256_set1_epi32(denom->magic)); +__m256i libdivide_u32_branchfree_do_vec256( + __m256i numers, const struct libdivide_u32_branchfree_t *denom) { + __m256i q = libdivide_mullhi_u32_vec256(numers, _mm256_set1_epi32(denom->magic)); __m256i t = _mm256_add_epi32(_mm256_srli_epi32(_mm256_sub_epi32(numers, q), 1), q); return _mm256_srli_epi32(t, denom->more); } ////////// UINT64 -__m256i libdivide_u64_do_vector(__m256i numers, const struct libdivide_u64_t *denom) { +__m256i libdivide_u64_do_vec256(__m256i numers, const struct libdivide_u64_t *denom) { uint8_t more = denom->more; if (!denom->magic) { return _mm256_srli_epi64(numers, more); - } - else { - __m256i q = libdivide_mullhi_u64_vector(numers, _mm256_set1_epi64x(denom->magic)); + } else { + __m256i q = libdivide_mullhi_u64_vec256(numers, _mm256_set1_epi64x(denom->magic)); if (more & LIBDIVIDE_ADD_MARKER) { // uint32_t t = ((numer - q) >> 1) + q; // return t >> denom->shift; uint32_t shift = more & LIBDIVIDE_64_SHIFT_MASK; __m256i t = _mm256_add_epi64(_mm256_srli_epi64(_mm256_sub_epi64(numers, q), 1), q); return _mm256_srli_epi64(t, shift); - } - else { + } else { return _mm256_srli_epi64(q, more); } } } -__m256i libdivide_u64_branchfree_do_vector(__m256i numers, const struct libdivide_u64_branchfree_t *denom) { - __m256i q = libdivide_mullhi_u64_vector(numers, _mm256_set1_epi64x(denom->magic)); +__m256i libdivide_u64_branchfree_do_vec256( + __m256i numers, const struct libdivide_u64_branchfree_t *denom) { + __m256i q = libdivide_mullhi_u64_vec256(numers, _mm256_set1_epi64x(denom->magic)); __m256i t = _mm256_add_epi64(_mm256_srli_epi64(_mm256_sub_epi64(numers, q), 1), q); return _mm256_srli_epi64(t, denom->more); } ////////// SINT32 -__m256i libdivide_s32_do_vector(__m256i numers, const struct libdivide_s32_t *denom) { +__m256i libdivide_s32_do_vec256(__m256i numers, const struct libdivide_s32_t *denom) { uint8_t more = denom->more; if (!denom->magic) { uint32_t shift = more & LIBDIVIDE_32_SHIFT_MASK; uint32_t mask = (1U << shift) - 1; __m256i roundToZeroTweak = _mm256_set1_epi32(mask); // q = numer + ((numer >> 31) & roundToZeroTweak); - __m256i q = _mm256_add_epi32(numers, _mm256_and_si256(_mm256_srai_epi32(numers, 31), roundToZeroTweak)); + __m256i q = _mm256_add_epi32( + numers, _mm256_and_si256(_mm256_srai_epi32(numers, 31), roundToZeroTweak)); q = _mm256_srai_epi32(q, shift); __m256i sign = _mm256_set1_epi32((int8_t)more >> 7); // q = (q ^ sign) - sign; q = _mm256_sub_epi32(_mm256_xor_si256(q, sign), sign); return q; - } - else { - __m256i q = libdivide_mullhi_s32_vector(numers, _mm256_set1_epi32(denom->magic)); + } else { + __m256i q = libdivide_mullhi_s32_vec256(numers, _mm256_set1_epi32(denom->magic)); if (more & LIBDIVIDE_ADD_MARKER) { - // must be arithmetic shift + // must be arithmetic shift __m256i sign = _mm256_set1_epi32((int8_t)more >> 7); - // q += ((numer ^ sign) - sign); + // q += ((numer ^ sign) - sign); q = _mm256_add_epi32(q, _mm256_sub_epi32(_mm256_xor_si256(numers, sign), sign)); } // q >>= shift q = _mm256_srai_epi32(q, more & LIBDIVIDE_32_SHIFT_MASK); - q = _mm256_add_epi32(q, _mm256_srli_epi32(q, 31)); // q += (q < 0) + q = _mm256_add_epi32(q, _mm256_srli_epi32(q, 31)); // q += (q < 0) return q; } } -__m256i libdivide_s32_branchfree_do_vector(__m256i numers, const struct libdivide_s32_branchfree_t *denom) { +__m256i libdivide_s32_branchfree_do_vec256( + __m256i numers, const struct libdivide_s32_branchfree_t *denom) { int32_t magic = denom->magic; uint8_t more = denom->more; uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; - // must be arithmetic shift + // must be arithmetic shift __m256i sign = _mm256_set1_epi32((int8_t)more >> 7); - __m256i q = libdivide_mullhi_s32_vector(numers, _mm256_set1_epi32(magic)); - q = _mm256_add_epi32(q, numers); // q += numers + __m256i q = libdivide_mullhi_s32_vec256(numers, _mm256_set1_epi32(magic)); + q = _mm256_add_epi32(q, numers); // q += numers // If q is non-negative, we have nothing to do // If q is negative, we want to add either (2**shift)-1 if d is // a power of 2, or (2**shift) if it is not a power of 2 uint32_t is_power_of_2 = (magic == 0); - __m256i q_sign = _mm256_srai_epi32(q, 31); // q_sign = q >> 31 + __m256i q_sign = _mm256_srai_epi32(q, 31); // q_sign = q >> 31 __m256i mask = _mm256_set1_epi32((1U << shift) - is_power_of_2); - q = _mm256_add_epi32(q, _mm256_and_si256(q_sign, mask)); // q = q + (q_sign & mask) - q = _mm256_srai_epi32(q, shift); // q >>= shift - q = _mm256_sub_epi32(_mm256_xor_si256(q, sign), sign); // q = (q ^ sign) - sign + q = _mm256_add_epi32(q, _mm256_and_si256(q_sign, mask)); // q = q + (q_sign & mask) + q = _mm256_srai_epi32(q, shift); // q >>= shift + q = _mm256_sub_epi32(_mm256_xor_si256(q, sign), sign); // q = (q ^ sign) - sign return q; } ////////// SINT64 -__m256i libdivide_s64_do_vector(__m256i numers, const struct libdivide_s64_t *denom) { +__m256i libdivide_s64_do_vec256(__m256i numers, const struct libdivide_s64_t *denom) { uint8_t more = denom->more; int64_t magic = denom->magic; - if (magic == 0) { // shift path + if (magic == 0) { // shift path uint32_t shift = more & LIBDIVIDE_64_SHIFT_MASK; uint64_t mask = (1ULL << shift) - 1; __m256i roundToZeroTweak = _mm256_set1_epi64x(mask); // q = numer + ((numer >> 63) & roundToZeroTweak); - __m256i q = _mm256_add_epi64(numers, _mm256_and_si256(libdivide_s64_signbits(numers), roundToZeroTweak)); - q = libdivide_s64_shift_right_vector(q, shift); + __m256i q = _mm256_add_epi64( + numers, _mm256_and_si256(libdivide_s64_signbits(numers), roundToZeroTweak)); + q = libdivide_s64_shift_right_vec256(q, shift); __m256i sign = _mm256_set1_epi32((int8_t)more >> 7); - // q = (q ^ sign) - sign; + // q = (q ^ sign) - sign; q = _mm256_sub_epi64(_mm256_xor_si256(q, sign), sign); return q; - } - else { - __m256i q = libdivide_mullhi_s64_vector(numers, _mm256_set1_epi64x(magic)); + } else { + __m256i q = libdivide_mullhi_s64_vec256(numers, _mm256_set1_epi64x(magic)); if (more & LIBDIVIDE_ADD_MARKER) { // must be arithmetic shift __m256i sign = _mm256_set1_epi32((int8_t)more >> 7); @@ -1655,46 +1931,53 @@ __m256i libdivide_s64_do_vector(__m256i numers, const struct libdivide_s64_t *de q = _mm256_add_epi64(q, _mm256_sub_epi64(_mm256_xor_si256(numers, sign), sign)); } // q >>= denom->mult_path.shift - q = libdivide_s64_shift_right_vector(q, more & LIBDIVIDE_64_SHIFT_MASK); - q = _mm256_add_epi64(q, _mm256_srli_epi64(q, 63)); // q += (q < 0) + q = libdivide_s64_shift_right_vec256(q, more & LIBDIVIDE_64_SHIFT_MASK); + q = _mm256_add_epi64(q, _mm256_srli_epi64(q, 63)); // q += (q < 0) return q; } } -__m256i libdivide_s64_branchfree_do_vector(__m256i numers, const struct libdivide_s64_branchfree_t *denom) { +__m256i libdivide_s64_branchfree_do_vec256( + __m256i numers, const struct libdivide_s64_branchfree_t *denom) { int64_t magic = denom->magic; uint8_t more = denom->more; uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; // must be arithmetic shift __m256i sign = _mm256_set1_epi32((int8_t)more >> 7); - // libdivide_mullhi_s64(numers, magic); - __m256i q = libdivide_mullhi_s64_vector(numers, _mm256_set1_epi64x(magic)); - q = _mm256_add_epi64(q, numers); // q += numers + // libdivide_mullhi_s64(numers, magic); + __m256i q = libdivide_mullhi_s64_vec256(numers, _mm256_set1_epi64x(magic)); + q = _mm256_add_epi64(q, numers); // q += numers // If q is non-negative, we have nothing to do. // If q is negative, we want to add either (2**shift)-1 if d is // a power of 2, or (2**shift) if it is not a power of 2. uint32_t is_power_of_2 = (magic == 0); - __m256i q_sign = libdivide_s64_signbits(q); // q_sign = q >> 63 + __m256i q_sign = libdivide_s64_signbits(q); // q_sign = q >> 63 __m256i mask = _mm256_set1_epi64x((1ULL << shift) - is_power_of_2); - q = _mm256_add_epi64(q, _mm256_and_si256(q_sign, mask)); // q = q + (q_sign & mask) - q = libdivide_s64_shift_right_vector(q, shift); // q >>= shift - q = _mm256_sub_epi64(_mm256_xor_si256(q, sign), sign); // q = (q ^ sign) - sign + q = _mm256_add_epi64(q, _mm256_and_si256(q_sign, mask)); // q = q + (q_sign & mask) + q = libdivide_s64_shift_right_vec256(q, shift); // q >>= shift + q = _mm256_sub_epi64(_mm256_xor_si256(q, sign), sign); // q = (q ^ sign) - sign return q; } -#elif defined(LIBDIVIDE_SSE2) +#endif -static inline __m128i libdivide_u32_do_vector(__m128i numers, const struct libdivide_u32_t *denom); -static inline __m128i libdivide_s32_do_vector(__m128i numers, const struct libdivide_s32_t *denom); -static inline __m128i libdivide_u64_do_vector(__m128i numers, const struct libdivide_u64_t *denom); -static inline __m128i libdivide_s64_do_vector(__m128i numers, const struct libdivide_s64_t *denom); +#if defined(LIBDIVIDE_SSE2) -static inline __m128i libdivide_u32_branchfree_do_vector(__m128i numers, const struct libdivide_u32_branchfree_t *denom); -static inline __m128i libdivide_s32_branchfree_do_vector(__m128i numers, const struct libdivide_s32_branchfree_t *denom); -static inline __m128i libdivide_u64_branchfree_do_vector(__m128i numers, const struct libdivide_u64_branchfree_t *denom); -static inline __m128i libdivide_s64_branchfree_do_vector(__m128i numers, const struct libdivide_s64_branchfree_t *denom); +static inline __m128i libdivide_u32_do_vec128(__m128i numers, const struct libdivide_u32_t *denom); +static inline __m128i libdivide_s32_do_vec128(__m128i numers, const struct libdivide_s32_t *denom); +static inline __m128i libdivide_u64_do_vec128(__m128i numers, const struct libdivide_u64_t *denom); +static inline __m128i libdivide_s64_do_vec128(__m128i numers, const struct libdivide_s64_t *denom); + +static inline __m128i libdivide_u32_branchfree_do_vec128( + __m128i numers, const struct libdivide_u32_branchfree_t *denom); +static inline __m128i libdivide_s32_branchfree_do_vec128( + __m128i numers, const struct libdivide_s32_branchfree_t *denom); +static inline __m128i libdivide_u64_branchfree_do_vec128( + __m128i numers, const struct libdivide_u64_branchfree_t *denom); +static inline __m128i libdivide_s64_branchfree_do_vec128( + __m128i numers, const struct libdivide_s64_branchfree_t *denom); //////// Internal Utility Functions @@ -1706,7 +1989,7 @@ static inline __m128i libdivide_s64_signbits(__m128i v) { } // Implementation of _mm_srai_epi64 (from AVX512). -static inline __m128i libdivide_s64_shift_right_vector(__m128i v, int amt) { +static inline __m128i libdivide_s64_shift_right_vec128(__m128i v, int amt) { const int b = 64 - amt; __m128i m = _mm_set1_epi64x(1ULL << (b - 1)); __m128i x = _mm_srli_epi64(v, amt); @@ -1715,7 +1998,7 @@ static inline __m128i libdivide_s64_shift_right_vector(__m128i v, int amt) { } // Here, b is assumed to contain one 32-bit value repeated. -static inline __m128i libdivide_mullhi_u32_vector(__m128i a, __m128i b) { +static inline __m128i libdivide_mullhi_u32_vec128(__m128i a, __m128i b) { __m128i hi_product_0Z2Z = _mm_srli_epi64(_mm_mul_epu32(a, b), 32); __m128i a1X3X = _mm_srli_epi64(a, 32); __m128i mask = _mm_set_epi32(-1, 0, -1, 0); @@ -1726,8 +2009,8 @@ static inline __m128i libdivide_mullhi_u32_vector(__m128i a, __m128i b) { // SSE2 does not have a signed multiplication instruction, but we can convert // unsigned to signed pretty efficiently. Again, b is just a 32 bit value // repeated four times. -static inline __m128i libdivide_mullhi_s32_vector(__m128i a, __m128i b) { - __m128i p = libdivide_mullhi_u32_vector(a, b); +static inline __m128i libdivide_mullhi_s32_vec128(__m128i a, __m128i b) { + __m128i p = libdivide_mullhi_u32_vec128(a, b); // t1 = (a >> 31) & y, arithmetic shift __m128i t1 = _mm_and_si128(_mm_srai_epi32(a, 31), b); __m128i t2 = _mm_and_si128(_mm_srai_epi32(b, 31), a); @@ -1737,30 +2020,41 @@ static inline __m128i libdivide_mullhi_s32_vector(__m128i a, __m128i b) { } // Here, y is assumed to contain one 64-bit value repeated. -// https://stackoverflow.com/a/28827013 -static inline __m128i libdivide_mullhi_u64_vector(__m128i x, __m128i y) { - __m128i lomask = _mm_set1_epi64x(0xffffffff); - __m128i xh = _mm_shuffle_epi32(x, 0xB1); // x0l, x0h, x1l, x1h - __m128i yh = _mm_shuffle_epi32(y, 0xB1); // y0l, y0h, y1l, y1h - __m128i w0 = _mm_mul_epu32(x, y); // x0l*y0l, x1l*y1l - __m128i w1 = _mm_mul_epu32(x, yh); // x0l*y0h, x1l*y1h - __m128i w2 = _mm_mul_epu32(xh, y); // x0h*y0l, x1h*y0l - __m128i w3 = _mm_mul_epu32(xh, yh); // x0h*y0h, x1h*y1h - __m128i w0h = _mm_srli_epi64(w0, 32); - __m128i s1 = _mm_add_epi64(w1, w0h); - __m128i s1l = _mm_and_si128(s1, lomask); - __m128i s1h = _mm_srli_epi64(s1, 32); - __m128i s2 = _mm_add_epi64(w2, s1l); - __m128i s2h = _mm_srli_epi64(s2, 32); - __m128i hi = _mm_add_epi64(w3, s1h); - hi = _mm_add_epi64(hi, s2h); +static inline __m128i libdivide_mullhi_u64_vec128(__m128i x, __m128i y) { + // full 128 bits product is: + // x0*y0 + (x0*y1 << 32) + (x1*y0 << 32) + (x1*y1 << 64) + // Note x0,y0,x1,y1 are all conceptually uint32, products are 32x32->64. - return hi; + // Compute x0*y0. + // Note x1, y1 are ignored by mul_epu32. + __m128i x0y0 = _mm_mul_epu32(x, y); + __m128i x0y0_hi = _mm_srli_epi64(x0y0, 32); + + // Get x1, y1 in the low bits. + // We could shuffle or right shift. Shuffles are preferred as they preserve + // the source register for the next computation. + __m128i x1 = _mm_shuffle_epi32(x, _MM_SHUFFLE(3, 3, 1, 1)); + __m128i y1 = _mm_shuffle_epi32(y, _MM_SHUFFLE(3, 3, 1, 1)); + + // No need to mask off top 32 bits for mul_epu32. + __m128i x0y1 = _mm_mul_epu32(x, y1); + __m128i x1y0 = _mm_mul_epu32(x1, y); + __m128i x1y1 = _mm_mul_epu32(x1, y1); + + // Mask here selects low bits only. + __m128i mask = _mm_set1_epi64x(0xFFFFFFFF); + __m128i temp = _mm_add_epi64(x1y0, x0y0_hi); + __m128i temp_lo = _mm_and_si128(temp, mask); + __m128i temp_hi = _mm_srli_epi64(temp, 32); + + temp_lo = _mm_srli_epi64(_mm_add_epi64(temp_lo, x0y1), 32); + temp_hi = _mm_add_epi64(x1y1, temp_hi); + return _mm_add_epi64(temp_lo, temp_hi); } // y is one 64-bit value repeated. -static inline __m128i libdivide_mullhi_s64_vector(__m128i x, __m128i y) { - __m128i p = libdivide_mullhi_u64_vector(x, y); +static inline __m128i libdivide_mullhi_s64_vec128(__m128i x, __m128i y) { + __m128i p = libdivide_mullhi_u64_vec128(x, y); __m128i t1 = _mm_and_si128(libdivide_s64_signbits(x), y); __m128i t2 = _mm_and_si128(libdivide_s64_signbits(y), x); p = _mm_sub_epi64(p, t1); @@ -1770,131 +2064,130 @@ static inline __m128i libdivide_mullhi_s64_vector(__m128i x, __m128i y) { ////////// UINT32 -__m128i libdivide_u32_do_vector(__m128i numers, const struct libdivide_u32_t *denom) { +__m128i libdivide_u32_do_vec128(__m128i numers, const struct libdivide_u32_t *denom) { uint8_t more = denom->more; if (!denom->magic) { return _mm_srli_epi32(numers, more); - } - else { - __m128i q = libdivide_mullhi_u32_vector(numers, _mm_set1_epi32(denom->magic)); + } else { + __m128i q = libdivide_mullhi_u32_vec128(numers, _mm_set1_epi32(denom->magic)); if (more & LIBDIVIDE_ADD_MARKER) { // uint32_t t = ((numer - q) >> 1) + q; // return t >> denom->shift; uint32_t shift = more & LIBDIVIDE_32_SHIFT_MASK; __m128i t = _mm_add_epi32(_mm_srli_epi32(_mm_sub_epi32(numers, q), 1), q); return _mm_srli_epi32(t, shift); - } - else { + } else { return _mm_srli_epi32(q, more); } } } -__m128i libdivide_u32_branchfree_do_vector(__m128i numers, const struct libdivide_u32_branchfree_t *denom) { - __m128i q = libdivide_mullhi_u32_vector(numers, _mm_set1_epi32(denom->magic)); +__m128i libdivide_u32_branchfree_do_vec128( + __m128i numers, const struct libdivide_u32_branchfree_t *denom) { + __m128i q = libdivide_mullhi_u32_vec128(numers, _mm_set1_epi32(denom->magic)); __m128i t = _mm_add_epi32(_mm_srli_epi32(_mm_sub_epi32(numers, q), 1), q); return _mm_srli_epi32(t, denom->more); } ////////// UINT64 -__m128i libdivide_u64_do_vector(__m128i numers, const struct libdivide_u64_t *denom) { +__m128i libdivide_u64_do_vec128(__m128i numers, const struct libdivide_u64_t *denom) { uint8_t more = denom->more; if (!denom->magic) { return _mm_srli_epi64(numers, more); - } - else { - __m128i q = libdivide_mullhi_u64_vector(numers, _mm_set1_epi64x(denom->magic)); + } else { + __m128i q = libdivide_mullhi_u64_vec128(numers, _mm_set1_epi64x(denom->magic)); if (more & LIBDIVIDE_ADD_MARKER) { // uint32_t t = ((numer - q) >> 1) + q; // return t >> denom->shift; uint32_t shift = more & LIBDIVIDE_64_SHIFT_MASK; __m128i t = _mm_add_epi64(_mm_srli_epi64(_mm_sub_epi64(numers, q), 1), q); return _mm_srli_epi64(t, shift); - } - else { + } else { return _mm_srli_epi64(q, more); } } } -__m128i libdivide_u64_branchfree_do_vector(__m128i numers, const struct libdivide_u64_branchfree_t *denom) { - __m128i q = libdivide_mullhi_u64_vector(numers, _mm_set1_epi64x(denom->magic)); +__m128i libdivide_u64_branchfree_do_vec128( + __m128i numers, const struct libdivide_u64_branchfree_t *denom) { + __m128i q = libdivide_mullhi_u64_vec128(numers, _mm_set1_epi64x(denom->magic)); __m128i t = _mm_add_epi64(_mm_srli_epi64(_mm_sub_epi64(numers, q), 1), q); return _mm_srli_epi64(t, denom->more); } ////////// SINT32 -__m128i libdivide_s32_do_vector(__m128i numers, const struct libdivide_s32_t *denom) { +__m128i libdivide_s32_do_vec128(__m128i numers, const struct libdivide_s32_t *denom) { uint8_t more = denom->more; if (!denom->magic) { uint32_t shift = more & LIBDIVIDE_32_SHIFT_MASK; uint32_t mask = (1U << shift) - 1; __m128i roundToZeroTweak = _mm_set1_epi32(mask); // q = numer + ((numer >> 31) & roundToZeroTweak); - __m128i q = _mm_add_epi32(numers, _mm_and_si128(_mm_srai_epi32(numers, 31), roundToZeroTweak)); + __m128i q = + _mm_add_epi32(numers, _mm_and_si128(_mm_srai_epi32(numers, 31), roundToZeroTweak)); q = _mm_srai_epi32(q, shift); __m128i sign = _mm_set1_epi32((int8_t)more >> 7); // q = (q ^ sign) - sign; q = _mm_sub_epi32(_mm_xor_si128(q, sign), sign); return q; - } - else { - __m128i q = libdivide_mullhi_s32_vector(numers, _mm_set1_epi32(denom->magic)); + } else { + __m128i q = libdivide_mullhi_s32_vec128(numers, _mm_set1_epi32(denom->magic)); if (more & LIBDIVIDE_ADD_MARKER) { - // must be arithmetic shift + // must be arithmetic shift __m128i sign = _mm_set1_epi32((int8_t)more >> 7); - // q += ((numer ^ sign) - sign); + // q += ((numer ^ sign) - sign); q = _mm_add_epi32(q, _mm_sub_epi32(_mm_xor_si128(numers, sign), sign)); } // q >>= shift q = _mm_srai_epi32(q, more & LIBDIVIDE_32_SHIFT_MASK); - q = _mm_add_epi32(q, _mm_srli_epi32(q, 31)); // q += (q < 0) + q = _mm_add_epi32(q, _mm_srli_epi32(q, 31)); // q += (q < 0) return q; } } -__m128i libdivide_s32_branchfree_do_vector(__m128i numers, const struct libdivide_s32_branchfree_t *denom) { +__m128i libdivide_s32_branchfree_do_vec128( + __m128i numers, const struct libdivide_s32_branchfree_t *denom) { int32_t magic = denom->magic; uint8_t more = denom->more; uint8_t shift = more & LIBDIVIDE_32_SHIFT_MASK; - // must be arithmetic shift + // must be arithmetic shift __m128i sign = _mm_set1_epi32((int8_t)more >> 7); - __m128i q = libdivide_mullhi_s32_vector(numers, _mm_set1_epi32(magic)); - q = _mm_add_epi32(q, numers); // q += numers + __m128i q = libdivide_mullhi_s32_vec128(numers, _mm_set1_epi32(magic)); + q = _mm_add_epi32(q, numers); // q += numers // If q is non-negative, we have nothing to do // If q is negative, we want to add either (2**shift)-1 if d is // a power of 2, or (2**shift) if it is not a power of 2 uint32_t is_power_of_2 = (magic == 0); - __m128i q_sign = _mm_srai_epi32(q, 31); // q_sign = q >> 31 + __m128i q_sign = _mm_srai_epi32(q, 31); // q_sign = q >> 31 __m128i mask = _mm_set1_epi32((1U << shift) - is_power_of_2); - q = _mm_add_epi32(q, _mm_and_si128(q_sign, mask)); // q = q + (q_sign & mask) - q = _mm_srai_epi32(q, shift); // q >>= shift - q = _mm_sub_epi32(_mm_xor_si128(q, sign), sign); // q = (q ^ sign) - sign + q = _mm_add_epi32(q, _mm_and_si128(q_sign, mask)); // q = q + (q_sign & mask) + q = _mm_srai_epi32(q, shift); // q >>= shift + q = _mm_sub_epi32(_mm_xor_si128(q, sign), sign); // q = (q ^ sign) - sign return q; } ////////// SINT64 -__m128i libdivide_s64_do_vector(__m128i numers, const struct libdivide_s64_t *denom) { +__m128i libdivide_s64_do_vec128(__m128i numers, const struct libdivide_s64_t *denom) { uint8_t more = denom->more; int64_t magic = denom->magic; - if (magic == 0) { // shift path + if (magic == 0) { // shift path uint32_t shift = more & LIBDIVIDE_64_SHIFT_MASK; uint64_t mask = (1ULL << shift) - 1; __m128i roundToZeroTweak = _mm_set1_epi64x(mask); // q = numer + ((numer >> 63) & roundToZeroTweak); - __m128i q = _mm_add_epi64(numers, _mm_and_si128(libdivide_s64_signbits(numers), roundToZeroTweak)); - q = libdivide_s64_shift_right_vector(q, shift); + __m128i q = + _mm_add_epi64(numers, _mm_and_si128(libdivide_s64_signbits(numers), roundToZeroTweak)); + q = libdivide_s64_shift_right_vec128(q, shift); __m128i sign = _mm_set1_epi32((int8_t)more >> 7); - // q = (q ^ sign) - sign; + // q = (q ^ sign) - sign; q = _mm_sub_epi64(_mm_xor_si128(q, sign), sign); return q; - } - else { - __m128i q = libdivide_mullhi_s64_vector(numers, _mm_set1_epi64x(magic)); + } else { + __m128i q = libdivide_mullhi_s64_vec128(numers, _mm_set1_epi64x(magic)); if (more & LIBDIVIDE_ADD_MARKER) { // must be arithmetic shift __m128i sign = _mm_set1_epi32((int8_t)more >> 7); @@ -1902,32 +2195,33 @@ __m128i libdivide_s64_do_vector(__m128i numers, const struct libdivide_s64_t *de q = _mm_add_epi64(q, _mm_sub_epi64(_mm_xor_si128(numers, sign), sign)); } // q >>= denom->mult_path.shift - q = libdivide_s64_shift_right_vector(q, more & LIBDIVIDE_64_SHIFT_MASK); - q = _mm_add_epi64(q, _mm_srli_epi64(q, 63)); // q += (q < 0) + q = libdivide_s64_shift_right_vec128(q, more & LIBDIVIDE_64_SHIFT_MASK); + q = _mm_add_epi64(q, _mm_srli_epi64(q, 63)); // q += (q < 0) return q; } } -__m128i libdivide_s64_branchfree_do_vector(__m128i numers, const struct libdivide_s64_branchfree_t *denom) { +__m128i libdivide_s64_branchfree_do_vec128( + __m128i numers, const struct libdivide_s64_branchfree_t *denom) { int64_t magic = denom->magic; uint8_t more = denom->more; uint8_t shift = more & LIBDIVIDE_64_SHIFT_MASK; // must be arithmetic shift __m128i sign = _mm_set1_epi32((int8_t)more >> 7); - // libdivide_mullhi_s64(numers, magic); - __m128i q = libdivide_mullhi_s64_vector(numers, _mm_set1_epi64x(magic)); - q = _mm_add_epi64(q, numers); // q += numers + // libdivide_mullhi_s64(numers, magic); + __m128i q = libdivide_mullhi_s64_vec128(numers, _mm_set1_epi64x(magic)); + q = _mm_add_epi64(q, numers); // q += numers // If q is non-negative, we have nothing to do. // If q is negative, we want to add either (2**shift)-1 if d is // a power of 2, or (2**shift) if it is not a power of 2. uint32_t is_power_of_2 = (magic == 0); - __m128i q_sign = libdivide_s64_signbits(q); // q_sign = q >> 63 + __m128i q_sign = libdivide_s64_signbits(q); // q_sign = q >> 63 __m128i mask = _mm_set1_epi64x((1ULL << shift) - is_power_of_2); - q = _mm_add_epi64(q, _mm_and_si128(q_sign, mask)); // q = q + (q_sign & mask) - q = libdivide_s64_shift_right_vector(q, shift); // q >>= shift - q = _mm_sub_epi64(_mm_xor_si128(q, sign), sign); // q = (q ^ sign) - sign + q = _mm_add_epi64(q, _mm_and_si128(q_sign, mask)); // q = q + (q_sign & mask) + q = libdivide_s64_shift_right_vec128(q, shift); // q >>= shift + q = _mm_sub_epi64(_mm_xor_si128(q, sign), sign); // q = (q ^ sign) - sign return q; } @@ -1937,143 +2231,273 @@ __m128i libdivide_s64_branchfree_do_vector(__m128i numers, const struct libdivid #ifdef __cplusplus -// The C++ divider class is templated on both an integer type -// (like uint64_t) and an algorithm type. -// * BRANCHFULL is the default algorithm type. -// * BRANCHFREE is the branchfree algorithm type. -enum { - BRANCHFULL, - BRANCHFREE +enum Branching { + BRANCHFULL, // use branching algorithms + BRANCHFREE // use branchfree algorithms }; -#if defined(LIBDIVIDE_AVX512) - #define LIBDIVIDE_VECTOR_TYPE __m512i -#elif defined(LIBDIVIDE_AVX2) - #define LIBDIVIDE_VECTOR_TYPE __m256i -#elif defined(LIBDIVIDE_SSE2) - #define LIBDIVIDE_VECTOR_TYPE __m128i +#if defined(LIBDIVIDE_NEON) +// Helper to deduce NEON vector type for integral type. +template +struct NeonVecFor {}; + +template <> +struct NeonVecFor { + typedef uint32x4_t type; +}; + +template <> +struct NeonVecFor { + typedef int32x4_t type; +}; + +template <> +struct NeonVecFor { + typedef uint64x2_t type; +}; + +template <> +struct NeonVecFor { + typedef int64x2_t type; +}; #endif -#if !defined(LIBDIVIDE_VECTOR_TYPE) - #define LIBDIVIDE_DIVIDE_VECTOR(ALGO) +// Versions of our algorithms for SIMD. +#if defined(LIBDIVIDE_NEON) +#define LIBDIVIDE_DIVIDE_NEON(ALGO, INT_TYPE) \ + typename NeonVecFor::type divide(typename NeonVecFor::type n) const { \ + return libdivide_##ALGO##_do_vec128(n, &denom); \ + } #else - #define LIBDIVIDE_DIVIDE_VECTOR(ALGO) \ - LIBDIVIDE_VECTOR_TYPE divide(LIBDIVIDE_VECTOR_TYPE n) const { \ - return libdivide_##ALGO##_do_vector(n, &denom); \ - } +#define LIBDIVIDE_DIVIDE_NEON(ALGO, INT_TYPE) +#endif +#if defined(LIBDIVIDE_SSE2) +#define LIBDIVIDE_DIVIDE_SSE2(ALGO) \ + __m128i divide(__m128i n) const { return libdivide_##ALGO##_do_vec128(n, &denom); } +#else +#define LIBDIVIDE_DIVIDE_SSE2(ALGO) +#endif + +#if defined(LIBDIVIDE_AVX2) +#define LIBDIVIDE_DIVIDE_AVX2(ALGO) \ + __m256i divide(__m256i n) const { return libdivide_##ALGO##_do_vec256(n, &denom); } +#else +#define LIBDIVIDE_DIVIDE_AVX2(ALGO) +#endif + +#if defined(LIBDIVIDE_AVX512) +#define LIBDIVIDE_DIVIDE_AVX512(ALGO) \ + __m512i divide(__m512i n) const { return libdivide_##ALGO##_do_vec512(n, &denom); } +#else +#define LIBDIVIDE_DIVIDE_AVX512(ALGO) #endif // The DISPATCHER_GEN() macro generates C++ methods (for the given integer // and algorithm types) that redirect to libdivide's C API. -#define DISPATCHER_GEN(T, ALGO) \ - libdivide_##ALGO##_t denom; \ - dispatcher() { } \ - dispatcher(T d) \ - : denom(libdivide_##ALGO##_gen(d)) \ - { } \ - T divide(T n) const { \ - return libdivide_##ALGO##_do(n, &denom); \ - } \ - LIBDIVIDE_DIVIDE_VECTOR(ALGO) \ - T recover() const { \ - return libdivide_##ALGO##_recover(&denom); \ - } +#define DISPATCHER_GEN(T, ALGO) \ + libdivide_##ALGO##_t denom; \ + dispatcher() {} \ + dispatcher(T d) : denom(libdivide_##ALGO##_gen(d)) {} \ + T divide(T n) const { return libdivide_##ALGO##_do(n, &denom); } \ + T recover() const { return libdivide_##ALGO##_recover(&denom); } \ + LIBDIVIDE_DIVIDE_NEON(ALGO, T) \ + LIBDIVIDE_DIVIDE_SSE2(ALGO) \ + LIBDIVIDE_DIVIDE_AVX2(ALGO) \ + LIBDIVIDE_DIVIDE_AVX512(ALGO) // The dispatcher selects a specific division algorithm for a given // type and ALGO using partial template specialization. -template struct dispatcher { }; +template +struct dispatcher {}; -template<> struct dispatcher { DISPATCHER_GEN(int32_t, s32) }; -template<> struct dispatcher { DISPATCHER_GEN(int32_t, s32_branchfree) }; -template<> struct dispatcher { DISPATCHER_GEN(uint32_t, u32) }; -template<> struct dispatcher { DISPATCHER_GEN(uint32_t, u32_branchfree) }; -template<> struct dispatcher { DISPATCHER_GEN(int64_t, s64) }; -template<> struct dispatcher { DISPATCHER_GEN(int64_t, s64_branchfree) }; -template<> struct dispatcher { DISPATCHER_GEN(uint64_t, u64) }; -template<> struct dispatcher { DISPATCHER_GEN(uint64_t, u64_branchfree) }; +template <> +struct dispatcher { + DISPATCHER_GEN(int32_t, s32) +}; +template <> +struct dispatcher { + DISPATCHER_GEN(int32_t, s32_branchfree) +}; +template <> +struct dispatcher { + DISPATCHER_GEN(uint32_t, u32) +}; +template <> +struct dispatcher { + DISPATCHER_GEN(uint32_t, u32_branchfree) +}; +template <> +struct dispatcher { + DISPATCHER_GEN(int64_t, s64) +}; +template <> +struct dispatcher { + DISPATCHER_GEN(int64_t, s64_branchfree) +}; +template <> +struct dispatcher { + DISPATCHER_GEN(uint64_t, u64) +}; +template <> +struct dispatcher { + DISPATCHER_GEN(uint64_t, u64_branchfree) +}; // This is the main divider class for use by the user (C++ API). // The actual division algorithm is selected using the dispatcher struct // based on the integer and algorithm template parameters. -template +template class divider { -public: + public: // We leave the default constructor empty so that creating // an array of dividers and then initializing them // later doesn't slow us down. - divider() { } + divider() {} // Constructor that takes the divisor as a parameter - divider(T d) : div(d) { } + divider(T d) : div(d) {} // Divides n by the divisor - T divide(T n) const { - return div.divide(n); - } + T divide(T n) const { return div.divide(n); } // Recovers the divisor, returns the value that was // used to initialize this divider object. - T recover() const { - return div.recover(); + T recover() const { return div.recover(); } + + bool operator==(const divider &other) const { + return div.denom.magic == other.denom.magic && div.denom.more == other.denom.more; } - bool operator==(const divider& other) const { - return div.denom.magic == other.denom.magic && - div.denom.more == other.denom.more; - } + bool operator!=(const divider &other) const { return !(*this == other); } - bool operator!=(const divider& other) const { - return !(*this == other); - } - -#if defined(LIBDIVIDE_VECTOR_TYPE) - // Treats the vector as packed integer values with the same type as - // the divider (e.g. s32, u32, s64, u64) and divides each of - // them by the divider, returning the packed quotients. - LIBDIVIDE_VECTOR_TYPE divide(LIBDIVIDE_VECTOR_TYPE n) const { + // Vector variants treat the input as packed integer values with the same type as the divider + // (e.g. s32, u32, s64, u64) and divides each of them by the divider, returning the packed + // quotients. +#if defined(LIBDIVIDE_SSE2) + __m128i divide(__m128i n) const { return div.divide(n); } +#endif +#if defined(LIBDIVIDE_AVX2) + __m256i divide(__m256i n) const { return div.divide(n); } +#endif +#if defined(LIBDIVIDE_AVX512) + __m512i divide(__m512i n) const { return div.divide(n); } +#endif +#if defined(LIBDIVIDE_NEON) + typename NeonVecFor::type divide(typename NeonVecFor::type n) const { return div.divide(n); } #endif -private: + private: // Storage for the actual divisor - dispatcher::value, - std::is_signed::value, sizeof(T), ALGO> div; + dispatcher::value, std::is_signed::value, sizeof(T), ALGO> div; }; // Overload of operator / for scalar division -template -T operator/(T n, const divider& div) { +template +T operator/(T n, const divider &div) { return div.divide(n); } // Overload of operator /= for scalar division -template -T& operator/=(T& n, const divider& div) { +template +T &operator/=(T &n, const divider &div) { n = div.divide(n); return n; } -#if defined(LIBDIVIDE_VECTOR_TYPE) - // Overload of operator / for vector division - template - LIBDIVIDE_VECTOR_TYPE operator/(LIBDIVIDE_VECTOR_TYPE n, const divider& div) { - return div.divide(n); - } - // Overload of operator /= for vector division - template - LIBDIVIDE_VECTOR_TYPE& operator/=(LIBDIVIDE_VECTOR_TYPE& n, const divider& div) { - n = div.divide(n); - return n; - } +// Overloads for vector types. +#if defined(LIBDIVIDE_SSE2) +template +__m128i operator/(__m128i n, const divider &div) { + return div.divide(n); +} + +template +__m128i operator/=(__m128i &n, const divider &div) { + n = div.divide(n); + return n; +} +#endif +#if defined(LIBDIVIDE_AVX2) +template +__m256i operator/(__m256i n, const divider &div) { + return div.divide(n); +} + +template +__m256i operator/=(__m256i &n, const divider &div) { + n = div.divide(n); + return n; +} +#endif +#if defined(LIBDIVIDE_AVX512) +template +__m512i operator/(__m512i n, const divider &div) { + return div.divide(n); +} + +template +__m512i operator/=(__m512i &n, const divider &div) { + n = div.divide(n); + return n; +} #endif -// libdivdie::branchfree_divider +#if defined(LIBDIVIDE_NEON) +template +uint32x4_t operator/(uint32x4_t n, const divider &div) { + return div.divide(n); +} + +template +int32x4_t operator/(int32x4_t n, const divider &div) { + return div.divide(n); +} + +template +uint64x2_t operator/(uint64x2_t n, const divider &div) { + return div.divide(n); +} + +template +int64x2_t operator/(int64x2_t n, const divider &div) { + return div.divide(n); +} + +template +uint32x4_t operator/=(uint32x4_t &n, const divider &div) { + n = div.divide(n); + return n; +} + +template +int32x4_t operator/=(int32x4_t &n, const divider &div) { + n = div.divide(n); + return n; +} + +template +uint64x2_t operator/=(uint64x2_t &n, const divider &div) { + n = div.divide(n); + return n; +} + +template +int64x2_t operator/=(int64x2_t &n, const divider &div) { + n = div.divide(n); + return n; +} +#endif + +#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1900) +// libdivide::branchfree_divider template using branchfree_divider = divider; +#endif -} // namespace libdivide +} // namespace libdivide -#endif // __cplusplus +#endif // __cplusplus -#endif // LIBDIVIDE_H +#endif // LIBDIVIDE_H diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index 5e7f1e4a98d..af643e9f7e4 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -243,7 +243,7 @@ The function works according to the algorithm: **Syntax** ``` sql -windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN) +windowFunnel(window, [mode, [mode, ... ]])(timestamp, cond1, cond2, ..., condN) ``` **Arguments** @@ -254,8 +254,10 @@ windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN) **Parameters** - `window` — Length of the sliding window. The unit of `window` depends on the `timestamp` itself and varies. Determined using the expression `timestamp of cond2 <= timestamp of cond1 + window`. -- `mode` — It is an optional argument. - - `'strict'` — When the `'strict'` is set, the windowFunnel() applies conditions only for the unique values. +- `mode` — It is an optional argument. One or more modes can be set. + - `'strict'` — If same condition holds for sequence of events then such non-unique events would be skipped. + - `'strict_order'` — Don't allow interventions of other events. E.g. in the case of `A->B->D->C`, it stops finding `A->B->C` at the `D` and the max event level is 2. + - `'strict_increase'` — Apply conditions only to events with strictly increasing timestamps. **Returned value** diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 86ed15e368d..01b1dd2d004 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -12,7 +12,9 @@ The search is case-sensitive by default in all these functions. There are separa ## position(haystack, needle), locate(haystack, needle) {#position} -Returns the position (in bytes) of the found substring in the string, starting from 1. +Searches for the substring `needle` in the string `haystack`. + +Returns the position (in bytes) of the found substring in the string, starting from 1. For a case-insensitive search, use the function [positionCaseInsensitive](#positioncaseinsensitive). @@ -20,15 +22,22 @@ For a case-insensitive search, use the function [positionCaseInsensitive](#posit ``` sql position(haystack, needle[, start_pos]) -``` +``` + +``` sql +position(needle IN haystack) +``` Alias: `locate(haystack, needle[, start_pos])`. +!!! note "Note" + Syntax of `position(needle IN haystack)` provides SQL-compatibility, the function works the same way as to `position(haystack, needle)`. + **Arguments** - `haystack` — String, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). - `needle` — Substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). -- `start_pos` — Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md). +- `start_pos` – Position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md). Optional. **Returned values** @@ -83,6 +92,36 @@ Result: └───────────────────────────────┘ ``` +**Examples for POSITION(needle IN haystack) syntax** + +Query: + +```sql +SELECT 3 = position('c' IN 'abc'); +``` + +Result: + +```text +┌─equals(3, position('abc', 'c'))─┐ +│ 1 │ +└─────────────────────────────────┘ +``` + +Query: + +```sql +SELECT 6 = position('/' IN s) FROM (SELECT 'Hello/World' AS s); +``` + +Result: + +```text +┌─equals(6, position(s, '/'))─┐ +│ 1 │ +└─────────────────────────────┘ +``` + ## positionCaseInsensitive {#positioncaseinsensitive} The same as [position](#position) returns the position (in bytes) of the found substring in the string, starting from 1. Use the function for a case-insensitive search. @@ -772,4 +811,3 @@ Result: │ 2 │ └───────────────────────────────┘ ``` - diff --git a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md index c2859b76c32..e5162b63b88 100644 --- a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md @@ -243,7 +243,7 @@ SELECT sequenceCount('(?1).*(?2)')(time, number = 1, number = 2) FROM t **Синтаксис** ``` sql -windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN) +windowFunnel(window, [mode, [mode, ... ]])(timestamp, cond1, cond2, ..., condN) ``` **Аргументы** @@ -254,7 +254,10 @@ windowFunnel(window, [mode])(timestamp, cond1, cond2, ..., condN) **Параметры** - `window` — ширина скользящего окна по времени. Единица измерения зависит от `timestamp` и может варьироваться. Должно соблюдаться условие `timestamp события cond2 <= timestamp события cond1 + window`. -- `mode` — необязательный параметр. Если установлено значение `'strict'`, то функция `windowFunnel()` применяет условия только для уникальных значений. +- `mode` — необязательный параметр. Может быть установленно несколько значений одновременно. + - `'strict'` — не учитывать подряд идущие повторяющиеся события. + - `'strict_order'` — запрещает посторонние события в искомой последовательности. Например, при поиске цепочки `A->B->C` в `A->B->D->C` поиск будет остановлен на `D` и функция вернет 2. + - `'strict_increase'` — условия прменяются только для событий со строго возрастающими временными метками. **Возвращаемое значение** diff --git a/docs/ru/sql-reference/functions/string-search-functions.md b/docs/ru/sql-reference/functions/string-search-functions.md index 658b0624b83..2417a1c6ffd 100644 --- a/docs/ru/sql-reference/functions/string-search-functions.md +++ b/docs/ru/sql-reference/functions/string-search-functions.md @@ -7,7 +7,7 @@ toc_title: "Функции поиска в строках" Во всех функциях, поиск регистрозависимый по умолчанию. Существуют варианты функций для регистронезависимого поиска. -## position(haystack, needle) {#position} +## position(haystack, needle), locate(haystack, needle) {#position} Поиск подстроки `needle` в строке `haystack`. @@ -21,8 +21,15 @@ toc_title: "Функции поиска в строках" position(haystack, needle[, start_pos]) ``` +``` sql +position(needle IN haystack) +``` + Алиас: `locate(haystack, needle[, start_pos])`. +!!! note "Примечание" + Синтаксис `position(needle IN haystack)` обеспечивает совместимость с SQL, функция работает так же, как `position(haystack, needle)`. + **Аргументы** - `haystack` — строка, по которой выполняется поиск. [Строка](../syntax.md#syntax-string-literal). @@ -70,6 +77,36 @@ SELECT position('Привет, мир!', '!'); └───────────────────────────────┘ ``` +**Примеры работы функции с синтаксисом POSITION(needle IN haystack)** + +Запрос: + +```sql +SELECT 1 = position('абв' IN 'абв'); +``` + +Результат: + +```text +┌─equals(1, position('абв', 'абв'))─┐ +│ 1 │ +└───────────────────────────────────┘ +``` + +Запрос: + +```sql +SELECT 0 = position('абв' IN ''); +``` + +Результат: + +```text +┌─equals(0, position('', 'абв'))─┐ +│ 1 │ +└────────────────────────────────┘ +``` + ## positionCaseInsensitive {#positioncaseinsensitive} Такая же, как и [position](#position), но работает без учета регистра. Возвращает позицию в байтах найденной подстроки в строке, начиная с 1. @@ -758,4 +795,3 @@ SELECT countSubstringsCaseInsensitiveUTF8('аБв__АбВ__абв', 'Абв'); │ 3 │ └────────────────────────────────────────────────────────────┘ ``` - diff --git a/docs/tools/requirements.txt b/docs/tools/requirements.txt index 9605525edbf..85f9dc2a9dd 100644 --- a/docs/tools/requirements.txt +++ b/docs/tools/requirements.txt @@ -23,7 +23,6 @@ nltk==3.5 nose==1.3.7 protobuf==3.14.0 numpy==1.19.2 -Pygments==2.5.2 pymdown-extensions==8.0 python-slugify==4.0.1 PyYAML==5.4.1 @@ -36,3 +35,4 @@ termcolor==1.1.0 tornado==6.1 Unidecode==1.1.1 urllib3==1.25.10 +Pygments>=2.7.4 diff --git a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp index 1e9f2782d95..ed732a197a1 100644 --- a/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp +++ b/src/AggregateFunctions/AggregateFunctionWindowFunnel.cpp @@ -6,7 +6,6 @@ #include #include -#include "registerAggregateFunctions.h" namespace DB diff --git a/src/AggregateFunctions/AggregateFunctionWindowFunnel.h b/src/AggregateFunctions/AggregateFunctionWindowFunnel.h index c765024507e..e4a275555bf 100644 --- a/src/AggregateFunctions/AggregateFunctionWindowFunnel.h +++ b/src/AggregateFunctions/AggregateFunctionWindowFunnel.h @@ -29,6 +29,7 @@ struct ComparePair final }; static constexpr auto max_events = 32; + template struct AggregateFunctionWindowFunnelData { @@ -46,7 +47,7 @@ struct AggregateFunctionWindowFunnelData void add(T timestamp, UInt8 event) { - // Since most events should have already been sorted by timestamp. + /// Since most events should have already been sorted by timestamp. if (sorted && events_list.size() > 0) { if (events_list.back().first == timestamp) @@ -145,14 +146,20 @@ class AggregateFunctionWindowFunnel final private: UInt64 window; UInt8 events_size; - UInt8 strict; // When the 'strict' is set, it applies conditions only for the not repeating values. - UInt8 strict_order; // When the 'strict_order' is set, it doesn't allow interventions of other events. - // In the case of 'A->B->D->C', it stops finding 'A->B->C' at the 'D' and the max event level is 2. + /// When the 'strict' is set, it applies conditions only for the not repeating values. + bool strict; - // Loop through the entire events_list, update the event timestamp value - // The level path must be 1---2---3---...---check_events_size, find the max event level that satisfied the path in the sliding window. - // If found, returns the max event level, else return 0. - // The Algorithm complexity is O(n). + /// When the 'strict_order' is set, it doesn't allow interventions of other events. + /// In the case of 'A->B->D->C', it stops finding 'A->B->C' at the 'D' and the max event level is 2. + bool strict_order; + + /// Applies conditions only to events with strictly increasing timestamps + bool strict_increase; + + /// Loop through the entire events_list, update the event timestamp value + /// The level path must be 1---2---3---...---check_events_size, find the max event level that satisfied the path in the sliding window. + /// If found, returns the max event level, else return 0. + /// The Algorithm complexity is O(n). UInt8 getEventLevel(Data & data) const { if (data.size() == 0) @@ -162,16 +169,13 @@ private: data.sort(); - /// events_timestamp stores the timestamp that latest i-th level event happen within time window after previous level event. - /// timestamp defaults to -1, which unsigned timestamp value never meet - /// there may be some bugs when UInt64 type timstamp overflows Int64, but it works on most cases. - std::vector events_timestamp(events_size, -1); + /// events_timestamp stores the timestamp of the first and previous i-th level event happen within time window + std::vector>> events_timestamp(events_size); bool first_event = false; for (const auto & pair : data.events_list) { const T & timestamp = pair.first; const auto & event_idx = pair.second - 1; - if (strict_order && event_idx == -1) { if (first_event) @@ -181,31 +185,39 @@ private: } else if (event_idx == 0) { - events_timestamp[0] = timestamp; + events_timestamp[0] = std::make_pair(timestamp, timestamp); first_event = true; } - else if (strict && events_timestamp[event_idx] >= 0) + else if (strict && events_timestamp[event_idx].has_value()) { return event_idx + 1; } - else if (strict_order && first_event && events_timestamp[event_idx - 1] == -1) + else if (strict_order && first_event && !events_timestamp[event_idx - 1].has_value()) { for (size_t event = 0; event < events_timestamp.size(); ++event) { - if (events_timestamp[event] == -1) + if (!events_timestamp[event].has_value()) return event; } } - else if (events_timestamp[event_idx - 1] >= 0 && timestamp <= events_timestamp[event_idx - 1] + window) + else if (events_timestamp[event_idx - 1].has_value()) { - events_timestamp[event_idx] = events_timestamp[event_idx - 1]; - if (event_idx + 1 == events_size) - return events_size; + auto first_timestamp = events_timestamp[event_idx - 1]->first; + bool time_matched = timestamp <= first_timestamp + window; + if (strict_increase) + time_matched = time_matched && events_timestamp[event_idx - 1]->second < timestamp; + if (time_matched) + { + events_timestamp[event_idx] = std::make_pair(first_timestamp, timestamp); + if (event_idx + 1 == events_size) + return events_size; + } } } + for (size_t event = events_timestamp.size(); event > 0; --event) { - if (events_timestamp[event - 1] >= 0) + if (events_timestamp[event - 1].has_value()) return event; } return 0; @@ -223,15 +235,18 @@ public: events_size = arguments.size() - 1; window = params.at(0).safeGet(); - strict = 0; - strict_order = 0; + strict = false; + strict_order = false; + strict_increase = false; for (size_t i = 1; i < params.size(); ++i) { String option = params.at(i).safeGet(); - if (option.compare("strict") == 0) - strict = 1; - else if (option.compare("strict_order") == 0) - strict_order = 1; + if (option == "strict") + strict = true; + else if (option == "strict_order") + strict_order = true; + else if (option == "strict_increase") + strict_increase = true; else throw Exception{"Aggregate function " + getName() + " doesn't support a parameter: " + option, ErrorCodes::BAD_ARGUMENTS}; } @@ -253,7 +268,7 @@ public: { bool has_event = false; const auto timestamp = assert_cast *>(columns[0])->getData()[row_num]; - // reverse iteration and stable sorting are needed for events that are qualified by more than one condition. + /// reverse iteration and stable sorting are needed for events that are qualified by more than one condition. for (auto i = events_size; i > 0; --i) { auto event = assert_cast *>(columns[i])->getData()[row_num]; diff --git a/src/AggregateFunctions/ReservoirSamplerDeterministic.h b/src/AggregateFunctions/ReservoirSamplerDeterministic.h index 3013a17e1ca..5cf97ae0f85 100644 --- a/src/AggregateFunctions/ReservoirSamplerDeterministic.h +++ b/src/AggregateFunctions/ReservoirSamplerDeterministic.h @@ -13,6 +13,7 @@ #include #include + namespace DB { namespace ErrorCodes @@ -162,6 +163,11 @@ public: sorted = false; } +#if !__clang__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wclass-memaccess" +#endif + void write(DB::WriteBuffer & buf) const { size_t size = samples.size(); @@ -169,9 +175,26 @@ public: DB::writeIntBinary(total_values, buf); for (size_t i = 0; i < size; ++i) - DB::writePODBinary(samples[i], buf); + { + /// There was a mistake in this function. + /// Instead of correctly serializing the elements, + /// it was writing them with uninitialized padding. + /// Here we ensure that padding is zero without changing the protocol. + /// TODO: After implementation of "versioning aggregate function state", + /// change the serialization format. + + Element elem; + memset(&elem, 0, sizeof(elem)); + elem = samples[i]; + + DB::writePODBinary(elem, buf); + } } +#if !__clang__ +#pragma GCC diagnostic pop +#endif + private: /// We allocate some memory on the stack to avoid allocations when there are many objects with a small number of elements. using Element = std::pair; diff --git a/src/Client/Connection.h b/src/Client/Connection.h index 8e653dc4153..7e3b9b7ab61 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -88,9 +88,9 @@ public: const String & user_, const String & password_, const String & cluster_, const String & cluster_secret_, - const String & client_name_ = "client", - Protocol::Compression compression_ = Protocol::Compression::Enable, - Protocol::Secure secure_ = Protocol::Secure::Disable, + const String & client_name_, + Protocol::Compression compression_, + Protocol::Secure secure_, Poco::Timespan sync_request_timeout_ = Poco::Timespan(DBMS_DEFAULT_SYNC_REQUEST_TIMEOUT_SEC, 0)) : host(host_), port(port_), default_database(default_database_), diff --git a/src/Client/ConnectionPool.h b/src/Client/ConnectionPool.h index 2389cc6755d..9e1d5f78b03 100644 --- a/src/Client/ConnectionPool.h +++ b/src/Client/ConnectionPool.h @@ -56,9 +56,9 @@ public: const String & password_, const String & cluster_, const String & cluster_secret_, - const String & client_name_ = "client", - Protocol::Compression compression_ = Protocol::Compression::Enable, - Protocol::Secure secure_ = Protocol::Secure::Disable, + const String & client_name_, + Protocol::Compression compression_, + Protocol::Secure secure_, Int64 priority_ = 1) : Base(max_connections_, &Poco::Logger::get("ConnectionPool (" + host_ + ":" + toString(port_) + ")")), diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index 061e3cbdff4..eff589edaa8 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -13,6 +13,7 @@ #include + namespace DB { diff --git a/src/Core/Block.h b/src/Core/Block.h index 14f4f57caed..914d6b361b9 100644 --- a/src/Core/Block.h +++ b/src/Core/Block.h @@ -116,7 +116,7 @@ public: /** Get a list of column names separated by commas. */ std::string dumpNames() const; - /** List of names, types and lengths of columns. Designed for debugging. */ + /** List of names, types and lengths of columns. Designed for debugging. */ std::string dumpStructure() const; /** List of column names and positions from index */ diff --git a/src/Interpreters/Cluster.cpp b/src/Interpreters/Cluster.cpp index ad759b99c29..9c19094b061 100644 --- a/src/Interpreters/Cluster.cpp +++ b/src/Interpreters/Cluster.cpp @@ -593,6 +593,7 @@ Cluster::Cluster(Cluster::ReplicasAsShardsTag, const Cluster & from, const Setti if (from.addresses_with_failover.empty()) throw Exception("Cluster is empty", ErrorCodes::LOGICAL_ERROR); + UInt32 shard_num = 0; std::set> unique_hosts; for (size_t shard_index : ext::range(0, from.shards_info.size())) { @@ -603,6 +604,8 @@ Cluster::Cluster(Cluster::ReplicasAsShardsTag, const Cluster & from, const Setti continue; /// Duplicate host, skip. ShardInfo info; + info.shard_num = ++shard_num; + if (address.is_local) info.local_addresses.push_back(address); diff --git a/src/Interpreters/Cluster.h b/src/Interpreters/Cluster.h index 88f1cf2b083..0fb75211bd2 100644 --- a/src/Interpreters/Cluster.h +++ b/src/Interpreters/Cluster.h @@ -97,6 +97,7 @@ public: Int64 priority = 1; Address() = default; + Address( const Poco::Util::AbstractConfiguration & config, const String & config_prefix, @@ -104,6 +105,7 @@ public: const String & cluster_secret_, UInt32 shard_index_ = 0, UInt32 replica_index_ = 0); + Address( const String & host_port_, const String & user_, diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 8a1ea6eb649..7a1ef8aeed6 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -83,6 +83,9 @@ struct Settings; M(UInt64, replicated_max_parallel_fetches_for_host, DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT, "Limit parallel fetches from endpoint (actually pool size).", 0) \ M(UInt64, replicated_max_parallel_sends, 0, "Limit parallel sends.", 0) \ M(UInt64, replicated_max_parallel_sends_for_table, 0, "Limit parallel sends for one table.", 0) \ + M(Seconds, replicated_fetches_http_connection_timeout, 0, "HTTP connection timeout for part fetch requests. Inherited from default profile `http_connection_timeout` if not set explicitly.", 0) \ + M(Seconds, replicated_fetches_http_send_timeout, 0, "HTTP send timeout for part fetch requests. Inherited from default profile `http_send_timeout` if not set explicitly.", 0) \ + M(Seconds, replicated_fetches_http_receive_timeout, 0, "HTTP receive timeout for fetch part requests. Inherited from default profile `http_receive_timeout` if not set explicitly.", 0) \ M(Bool, replicated_can_become_leader, true, "If true, Replicated tables replicas on this node will try to acquire leadership.", 0) \ M(Seconds, zookeeper_session_expiration_check_period, 60, "ZooKeeper session expiration check period, in seconds.", 0) \ M(Bool, detach_old_local_parts_when_cloning_replica, 1, "Do not remove old local parts when repairing lost replica.", 0) \ diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index 692d2ac4b94..f0f178cb71c 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -39,7 +39,7 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer( , queried_columns{queried_columns_} , sorting_key_names{NameSet( metadata_snapshot->getSortingKey().column_names.begin(), metadata_snapshot->getSortingKey().column_names.end())} - , block_with_constants{KeyCondition::getBlockWithConstants(query_info.query, query_info.syntax_analyzer_result, context)} + , block_with_constants{KeyCondition::getBlockWithConstants(query_info.query->clone(), query_info.syntax_analyzer_result, context)} , log{log_} , column_sizes{std::move(column_sizes_)} { diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 0849f65477d..cfae9c346b5 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2313,7 +2313,8 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) { String source_replica_path = zookeeper_path + "/replicas/" + part_desc->replica; ReplicatedMergeTreeAddress address(getZooKeeper()->get(source_replica_path + "/host")); - auto timeouts = ConnectionTimeouts::getHTTPTimeouts(global_context); + auto timeouts = getFetchPartHTTPTimeouts(global_context); + auto [user, password] = global_context.getInterserverCredentials(); String interserver_scheme = global_context.getInterserverScheme(); @@ -3246,6 +3247,23 @@ void StorageReplicatedMergeTree::exitLeaderElection() leader_election = nullptr; } +ConnectionTimeouts StorageReplicatedMergeTree::getFetchPartHTTPTimeouts(const Context & context) +{ + auto timeouts = ConnectionTimeouts::getHTTPTimeouts(context); + auto settings = getSettings(); + + if (settings->replicated_fetches_http_connection_timeout.changed) + timeouts.connection_timeout = settings->replicated_fetches_http_connection_timeout; + + if (settings->replicated_fetches_http_send_timeout.changed) + timeouts.send_timeout = settings->replicated_fetches_http_send_timeout; + + if (settings->replicated_fetches_http_receive_timeout.changed) + timeouts.receive_timeout = settings->replicated_fetches_http_receive_timeout; + + return timeouts; +} + bool StorageReplicatedMergeTree::checkReplicaHavePart(const String & replica, const String & part_name) { auto zookeeper = getZooKeeper(); @@ -3661,7 +3679,8 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora else { address.fromString(zookeeper->get(source_replica_path + "/host")); - timeouts = ConnectionTimeouts::getHTTPTimeouts(global_context); + timeouts = getFetchPartHTTPTimeouts(global_context); + user_password = global_context.getInterserverCredentials(); interserver_scheme = global_context.getInterserverScheme(); diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 2d1b50ede4c..6393f4d58a5 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -507,6 +507,8 @@ private: /// Exchange parts. + ConnectionTimeouts getFetchPartHTTPTimeouts(const Context & context); + /** Returns an empty string if no one has a part. */ String findReplicaHavingPart(const String & part_name, bool active); diff --git a/tests/config/config.d/clusters.xml b/tests/config/config.d/clusters.xml index b783372f83b..86dba46e07a 100644 --- a/tests/config/config.d/clusters.xml +++ b/tests/config/config.d/clusters.xml @@ -16,5 +16,17 @@ + + + + 127.0.0.1 + 9000 + + + 127.0.0.2 + 9000 + + + diff --git a/tests/integration/test_replicated_fetches_timeouts/__init__.py b/tests/integration/test_replicated_fetches_timeouts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_replicated_fetches_timeouts/configs/server.xml b/tests/integration/test_replicated_fetches_timeouts/configs/server.xml new file mode 100644 index 00000000000..d4b441b91fb --- /dev/null +++ b/tests/integration/test_replicated_fetches_timeouts/configs/server.xml @@ -0,0 +1,3 @@ + + 0.1 + diff --git a/tests/integration/test_replicated_fetches_timeouts/test.py b/tests/integration/test_replicated_fetches_timeouts/test.py new file mode 100644 index 00000000000..963ec2487fd --- /dev/null +++ b/tests/integration/test_replicated_fetches_timeouts/test.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 + +import random +import string +import time + +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance( + 'node1', with_zookeeper=True, + main_configs=['configs/server.xml']) + +node2 = cluster.add_instance( + 'node2', with_zookeeper=True, + main_configs=['configs/server.xml']) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + yield cluster + + finally: + cluster.shutdown() + + +def get_random_string(length): + return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(length)) + + +def test_no_stall(started_cluster): + for instance in started_cluster.instances.values(): + instance.query(""" + CREATE TABLE t (key UInt64, data String) + ENGINE = ReplicatedMergeTree('/clickhouse/test/t', '{instance}') + ORDER BY tuple() + PARTITION BY key""") + + # Pause node3 until the test setup is prepared + node2.query("SYSTEM STOP FETCHES t") + + node1.query("INSERT INTO t SELECT 1, '{}' FROM numbers(500)".format(get_random_string(104857))) + node1.query("INSERT INTO t SELECT 2, '{}' FROM numbers(500)".format(get_random_string(104857))) + + with PartitionManager() as pm: + pm.add_network_delay(node1, 2000) + node2.query("SYSTEM START FETCHES t") + + # Wait for timeout exceptions to confirm that timeout is triggered. + while True: + conn_timeout_exceptions = int(node2.query( + """ + SELECT count() + FROM system.replication_queue + WHERE last_exception LIKE '%connect timed out%' + """)) + + if conn_timeout_exceptions >= 2: + break + + time.sleep(0.1) + + print("Connection timeouts tested!") + + # Increase connection timeout and wait for receive timeouts. + node2.query(""" + ALTER TABLE t + MODIFY SETTING replicated_fetches_http_connection_timeout = 30, + replicated_fetches_http_receive_timeout = 1""") + + while True: + timeout_exceptions = int(node2.query( + """ + SELECT count() + FROM system.replication_queue + WHERE last_exception LIKE '%e.displayText() = Timeout%' + AND last_exception NOT LIKE '%connect timed out%' + """).strip()) + + if timeout_exceptions >= 2: + break + + time.sleep(0.1) + + for instance in started_cluster.instances.values(): + # Workaround for DROP TABLE not finishing if it is started while table is readonly. + instance.query("SYSTEM RESTART REPLICA t") + + # Cleanup data directory from test results archive. + instance.query("DROP TABLE t SYNC") diff --git a/tests/jepsen.clickhouse-keeper/test/jepsen/nukeeper_test.clj b/tests/jepsen.clickhouse-keeper/test/jepsen/keeper_test.clj similarity index 94% rename from tests/jepsen.clickhouse-keeper/test/jepsen/nukeeper_test.clj rename to tests/jepsen.clickhouse-keeper/test/jepsen/keeper_test.clj index db84ff33ee3..35b1bab7b38 100644 --- a/tests/jepsen.clickhouse-keeper/test/jepsen/nukeeper_test.clj +++ b/tests/jepsen.clickhouse-keeper/test/jepsen/keeper_test.clj @@ -1,6 +1,6 @@ -(ns jepsen.nukeeper-test +(ns jepsen.keeper-test (:require [clojure.test :refer :all] - [jepsen.nukeeper.utils :refer :all] + [jepsen.clickhouse-keeper.utils :refer :all] [zookeeper :as zk] [zookeeper.data :as data]) (:import (ch.qos.logback.classic Level) diff --git a/tests/queries/0_stateless/00632_aggregation_window_funnel.reference b/tests/queries/0_stateless/00632_aggregation_window_funnel.reference index 492135567ea..2c68f277bfa 100644 --- a/tests/queries/0_stateless/00632_aggregation_window_funnel.reference +++ b/tests/queries/0_stateless/00632_aggregation_window_funnel.reference @@ -57,3 +57,7 @@ [2, 0] [3, 1] [4, 1] +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/00632_aggregation_window_funnel.sql b/tests/queries/0_stateless/00632_aggregation_window_funnel.sql index 5a1610256ac..d9991be5583 100644 --- a/tests/queries/0_stateless/00632_aggregation_window_funnel.sql +++ b/tests/queries/0_stateless/00632_aggregation_window_funnel.sql @@ -79,3 +79,11 @@ select u, windowFunnel(86400)(dt, a is null and b is null) as s from funnel_test select u, windowFunnel(86400)(dt, a is null, b = 'b3') as s from funnel_test_non_null group by u order by u format JSONCompactEachRow; select u, windowFunnel(86400, 'strict_order')(dt, a is null, b = 'b3') as s from funnel_test_non_null group by u order by u format JSONCompactEachRow; drop table funnel_test_non_null; + +create table funnel_test_strict_increase (timestamp UInt32, event UInt32) engine=Memory; +insert into funnel_test_strict_increase values (0,1000),(1,1001),(1,1002),(1,1003),(2,1004); + +select 5 = windowFunnel(10000)(timestamp, event = 1000, event = 1001, event = 1002, event = 1003, event = 1004) from funnel_test_strict_increase; +select 2 = windowFunnel(10000, 'strict_increase')(timestamp, event = 1000, event = 1001, event = 1002, event = 1003, event = 1004) from funnel_test_strict_increase; +select 3 = windowFunnel(10000)(timestamp, event = 1004, event = 1004, event = 1004) from funnel_test_strict_increase; +select 1 = windowFunnel(10000, 'strict_increase')(timestamp, event = 1004, event = 1004, event = 1004) from funnel_test_strict_increase; diff --git a/tests/queries/0_stateless/01754_cluster_all_replicas_shard_num.reference b/tests/queries/0_stateless/01754_cluster_all_replicas_shard_num.reference new file mode 100644 index 00000000000..d308efd8662 --- /dev/null +++ b/tests/queries/0_stateless/01754_cluster_all_replicas_shard_num.reference @@ -0,0 +1,9 @@ +1 +1 +1 +2 +1 +2 +1 +1 +2 diff --git a/tests/queries/0_stateless/01754_cluster_all_replicas_shard_num.sql b/tests/queries/0_stateless/01754_cluster_all_replicas_shard_num.sql new file mode 100644 index 00000000000..833f86c538d --- /dev/null +++ b/tests/queries/0_stateless/01754_cluster_all_replicas_shard_num.sql @@ -0,0 +1,8 @@ +SELECT _shard_num FROM cluster('test_shard_localhost', system.one); +SELECT _shard_num FROM clusterAllReplicas('test_shard_localhost', system.one); + +SELECT _shard_num FROM cluster('test_cluster_two_shards', system.one) ORDER BY _shard_num; +SELECT _shard_num FROM clusterAllReplicas('test_cluster_two_shards', system.one) ORDER BY _shard_num; + +SELECT _shard_num FROM cluster('test_cluster_one_shard_two_replicas', system.one) ORDER BY _shard_num; +SELECT _shard_num FROM clusterAllReplicas('test_cluster_one_shard_two_replicas', system.one) ORDER BY _shard_num; diff --git a/tests/queries/0_stateless/01778_where_with_column_name.reference b/tests/queries/0_stateless/01778_where_with_column_name.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01778_where_with_column_name.sql b/tests/queries/0_stateless/01778_where_with_column_name.sql new file mode 100644 index 00000000000..effde87f018 --- /dev/null +++ b/tests/queries/0_stateless/01778_where_with_column_name.sql @@ -0,0 +1,5 @@ +DROP TABLE IF EXISTS ttt01778; +CREATE TABLE ttt01778 (`1` String, `2` INT) ENGINE = MergeTree() ORDER BY tuple(); +INSERT INTO ttt01778 values('1',1),('2',2),('3',3); +select * from ttt01778 where 1=2; -- no server error +DROP TABLE ttt01778; diff --git a/tests/queries/0_stateless/01779_quantile_deterministic_msan.reference b/tests/queries/0_stateless/01779_quantile_deterministic_msan.reference new file mode 100644 index 00000000000..b432de6ddb3 --- /dev/null +++ b/tests/queries/0_stateless/01779_quantile_deterministic_msan.reference @@ -0,0 +1 @@ +11447494982455782708 diff --git a/tests/queries/0_stateless/01779_quantile_deterministic_msan.sql b/tests/queries/0_stateless/01779_quantile_deterministic_msan.sql new file mode 100644 index 00000000000..ef4234da306 --- /dev/null +++ b/tests/queries/0_stateless/01779_quantile_deterministic_msan.sql @@ -0,0 +1 @@ +SELECT cityHash64(toString(quantileDeterministicState(number, sipHash64(number)))) FROM numbers(8193); diff --git a/tests/queries/0_stateless/helpers/httpechoserver.py b/tests/queries/0_stateless/helpers/httpechoserver.py deleted file mode 100644 index a1176c5e72d..00000000000 --- a/tests/queries/0_stateless/helpers/httpechoserver.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -import time -import subprocess -import threading -from io import StringIO, SEEK_END -from http.server import BaseHTTPRequestHandler, HTTPServer - -CLICKHOUSE_HOST = os.environ.get('CLICKHOUSE_HOST', '127.0.0.1') -CLICKHOUSE_PORT_HTTP = os.environ.get('CLICKHOUSE_PORT_HTTP', '8123') - -# IP-address of this host accessible from outside world. -HTTP_SERVER_HOST = os.environ.get('HTTP_SERVER_HOST', subprocess.check_output(['hostname', '-i']).decode('utf-8').strip()) -HTTP_SERVER_PORT = int(os.environ.get('CLICKHOUSE_TEST_HOST_EXPOSED_PORT', 51234)) - -# IP address and port of the HTTP server started from this script. -HTTP_SERVER_ADDRESS = (HTTP_SERVER_HOST, HTTP_SERVER_PORT) -HTTP_SERVER_URL_STR = 'http://' + ':'.join(str(s) for s in HTTP_SERVER_ADDRESS) + "/" - -ostream = StringIO() -istream = sys.stdout - -class EchoCSVHTTPServer(BaseHTTPRequestHandler): - def _set_headers(self): - self.send_response(200) - self.send_header('Content-type', 'text/plain') - self.end_headers() - - def do_GET(self): - self._set_headers() - with open(CSV_DATA, 'r') as fl: - ostream.seek(0) - for row in ostream: - self.wfile.write(row + '\n') - return - - def read_chunk(self): - msg = '' - while True: - sym = self.rfile.read(1) - if sym == '': - break - msg += sym.decode('utf-8') - if msg.endswith('\r\n'): - break - length = int(msg[:-2], 16) - if length == 0: - return '' - content = self.rfile.read(length) - self.rfile.read(2) # read sep \r\n - return content.decode('utf-8') - - def do_POST(self): - while True: - chunk = self.read_chunk() - if not chunk: - break - istream.write(chunk) - istream.flush() - text = "" - self._set_headers() - self.wfile.write("ok") - - def log_message(self, format, *args): - return - -def start_server(requests_amount, test_data="Hello,2,-2,7.7\nWorld,2,-5,8.8"): - ostream = StringIO(test_data.decode("utf-8")) - - httpd = HTTPServer(HTTP_SERVER_ADDRESS, EchoCSVHTTPServer) - - def real_func(): - for i in range(requests_amount): - httpd.handle_request() - - t = threading.Thread(target=real_func) - return t - -def run(requests_amount=1): - t = start_server(requests_amount) - t.start() - t.join() - -if __name__ == "__main__": - exception_text = '' - for i in range(1, 5): - try: - run(int(sys.argv[1]) if len(sys.argv) > 1 else 1) - break - except Exception as ex: - exception_text = str(ex) - time.sleep(1) - - if exception_text: - print("Exception: {}".format(exception_text), file=sys.stderr) - os._exit(1) - diff --git a/tests/server-test.xml b/tests/server-test.xml index 0b5e8f760a8..dd21d55c78c 100644 --- a/tests/server-test.xml +++ b/tests/server-test.xml @@ -140,4 +140,4 @@ [hidden] - \ No newline at end of file + diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index d3e1c2acd69..33982fdb61b 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -21,7 +21,7 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS) add_subdirectory (corrector_utf8) add_subdirectory (zookeeper-cli) add_subdirectory (zookeeper-test) - add_subdirectory (nukeeper-data-dumper) + add_subdirectory (keeper-data-dumper) add_subdirectory (zookeeper-dump-tree) add_subdirectory (zookeeper-remove-by-list) add_subdirectory (zookeeper-create-entry-to-download-part) diff --git a/utils/keeper-data-dumper/CMakeLists.txt b/utils/keeper-data-dumper/CMakeLists.txt new file mode 100644 index 00000000000..af16924547f --- /dev/null +++ b/utils/keeper-data-dumper/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(keeper-data-dumper main.cpp) +target_link_libraries(keeper-data-dumper PRIVATE dbms) diff --git a/utils/nukeeper-data-dumper/main.cpp b/utils/keeper-data-dumper/main.cpp similarity index 100% rename from utils/nukeeper-data-dumper/main.cpp rename to utils/keeper-data-dumper/main.cpp diff --git a/utils/nukeeper-data-dumper/CMakeLists.txt b/utils/nukeeper-data-dumper/CMakeLists.txt deleted file mode 100644 index bab1137bf4d..00000000000 --- a/utils/nukeeper-data-dumper/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -add_executable(nukeeper-data-dumper main.cpp) -target_link_libraries(nukeeper-data-dumper PRIVATE dbms)