Merge branch 'master' of https://github.com/ClickHouse/ClickHouse into testflows_modules_in_parallel

This commit is contained in:
Vitaliy Zakaznikov 2021-05-11 17:17:53 -04:00
commit 9c2b30f9fd
306 changed files with 4501 additions and 3548 deletions

View File

@ -0,0 +1,216 @@
#pragma once
#include <cstdint>
#include <cstddef>
#include <cstring>
#include <common/extended_types.h>
/// Allows to check the internals of IEEE-754 floating point number.
template <typename T> struct FloatTraits;
template <>
struct FloatTraits<float>
{
using UInt = uint32_t;
static constexpr size_t bits = 32;
static constexpr size_t exponent_bits = 8;
static constexpr size_t mantissa_bits = bits - exponent_bits - 1;
};
template <>
struct FloatTraits<double>
{
using UInt = uint64_t;
static constexpr size_t bits = 64;
static constexpr size_t exponent_bits = 11;
static constexpr size_t mantissa_bits = bits - exponent_bits - 1;
};
/// x = sign * (2 ^ normalized_exponent) * (1 + mantissa * 2 ^ -mantissa_bits)
/// x = sign * (2 ^ normalized_exponent + mantissa * 2 ^ (normalized_exponent - mantissa_bits))
template <typename T>
struct DecomposedFloat
{
using Traits = FloatTraits<T>;
DecomposedFloat(T x)
{
memcpy(&x_uint, &x, sizeof(x));
}
typename Traits::UInt x_uint;
bool is_negative() const
{
return x_uint >> (Traits::bits - 1);
}
/// Returns 0 for both +0. and -0.
int sign() const
{
return (exponent() == 0 && mantissa() == 0)
? 0
: (is_negative()
? -1
: 1);
}
uint16_t exponent() const
{
return (x_uint >> (Traits::mantissa_bits)) & (((1ull << (Traits::exponent_bits + 1)) - 1) >> 1);
}
int16_t normalized_exponent() const
{
return int16_t(exponent()) - ((1ull << (Traits::exponent_bits - 1)) - 1);
}
uint64_t mantissa() const
{
return x_uint & ((1ull << Traits::mantissa_bits) - 1);
}
int64_t mantissa_with_sign() const
{
return is_negative() ? -mantissa() : mantissa();
}
/// NOTE Probably floating point instructions can be better.
bool is_integer_in_representable_range() const
{
return x_uint == 0
|| (normalized_exponent() >= 0 /// The number is not less than one
/// The number is inside the range where every integer has exact representation in float
&& normalized_exponent() <= static_cast<int16_t>(Traits::mantissa_bits)
/// After multiplying by 2^exp, the fractional part becomes zero, means the number is integer
&& ((mantissa() & ((1ULL << (Traits::mantissa_bits - normalized_exponent())) - 1)) == 0));
}
/// Compare float with integer of arbitrary width (both signed and unsigned are supported). Assuming two's complement arithmetic.
/// Infinities are compared correctly. NaNs are treat similarly to infinities, so they can be less than all numbers.
/// (note that we need total order)
template <typename Int>
int compare(Int rhs)
{
if (rhs == 0)
return sign();
/// Different signs
if (is_negative() && rhs > 0)
return -1;
if (!is_negative() && rhs < 0)
return 1;
/// Fractional number with magnitude less than one
if (normalized_exponent() < 0)
{
if (!is_negative())
return rhs > 0 ? -1 : 1;
else
return rhs >= 0 ? -1 : 1;
}
/// The case of the most negative integer
if constexpr (is_signed_v<Int>)
{
if (rhs == std::numeric_limits<Int>::lowest())
{
assert(is_negative());
if (normalized_exponent() < static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
return 1;
if (normalized_exponent() > static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
return -1;
if (mantissa() == 0)
return 0;
else
return -1;
}
}
/// Too large number: abs(float) > abs(rhs). Also the case with infinities and NaN.
if (normalized_exponent() >= static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
return is_negative() ? -1 : 1;
using UInt = make_unsigned_t<Int>;
UInt uint_rhs = rhs < 0 ? -rhs : rhs;
/// Smaller octave: abs(rhs) < abs(float)
if (uint_rhs < (static_cast<UInt>(1) << normalized_exponent()))
return is_negative() ? -1 : 1;
/// Larger octave: abs(rhs) > abs(float)
if (normalized_exponent() + 1 < static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>)
&& uint_rhs >= (static_cast<UInt>(1) << (normalized_exponent() + 1)))
return is_negative() ? 1 : -1;
/// The same octave
/// uint_rhs == 2 ^ normalized_exponent + mantissa * 2 ^ (normalized_exponent - mantissa_bits)
bool large_and_always_integer = normalized_exponent() >= static_cast<int16_t>(Traits::mantissa_bits);
typename Traits::UInt a = large_and_always_integer
? mantissa() << (normalized_exponent() - Traits::mantissa_bits)
: mantissa() >> (Traits::mantissa_bits - normalized_exponent());
typename Traits::UInt b = uint_rhs - (static_cast<UInt>(1) << normalized_exponent());
if (a < b)
return is_negative() ? 1 : -1;
if (a > b)
return is_negative() ? -1 : 1;
/// Float has no fractional part means that the numbers are equal.
if (large_and_always_integer || (mantissa() & ((1ULL << (Traits::mantissa_bits - normalized_exponent())) - 1)) == 0)
return 0;
else
/// Float has fractional part means its abs value is larger.
return is_negative() ? -1 : 1;
}
template <typename Int>
bool equals(Int rhs)
{
return compare(rhs) == 0;
}
template <typename Int>
bool notEquals(Int rhs)
{
return compare(rhs) != 0;
}
template <typename Int>
bool less(Int rhs)
{
return compare(rhs) < 0;
}
template <typename Int>
bool greater(Int rhs)
{
return compare(rhs) > 0;
}
template <typename Int>
bool lessOrEquals(Int rhs)
{
return compare(rhs) <= 0;
}
template <typename Int>
bool greaterOrEquals(Int rhs)
{
return compare(rhs) >= 0;
}
};
using DecomposedFloat64 = DecomposedFloat<double>;
using DecomposedFloat32 = DecomposedFloat<float>;

View File

@ -56,27 +56,33 @@ namespace common
}
template <>
inline bool addOverflow(__int128 x, __int128 y, __int128 & res)
inline bool addOverflow(Int128 x, Int128 y, Int128 & res)
{
static constexpr __int128 min_int128 = minInt128();
static constexpr __int128 max_int128 = maxInt128();
res = addIgnoreOverflow(x, y);
return (y > 0 && x > max_int128 - y) || (y < 0 && x < min_int128 - y);
return (y > 0 && x > std::numeric_limits<Int128>::max() - y) ||
(y < 0 && x < std::numeric_limits<Int128>::min() - y);
}
template <>
inline bool addOverflow(wInt256 x, wInt256 y, wInt256 & res)
inline bool addOverflow(UInt128 x, UInt128 y, UInt128 & res)
{
res = addIgnoreOverflow(x, y);
return (y > 0 && x > std::numeric_limits<wInt256>::max() - y) ||
(y < 0 && x < std::numeric_limits<wInt256>::min() - y);
return x > std::numeric_limits<UInt128>::max() - y;
}
template <>
inline bool addOverflow(wUInt256 x, wUInt256 y, wUInt256 & res)
inline bool addOverflow(Int256 x, Int256 y, Int256 & res)
{
res = addIgnoreOverflow(x, y);
return x > std::numeric_limits<wUInt256>::max() - y;
return (y > 0 && x > std::numeric_limits<Int256>::max() - y) ||
(y < 0 && x < std::numeric_limits<Int256>::min() - y);
}
template <>
inline bool addOverflow(UInt256 x, UInt256 y, UInt256 & res)
{
res = addIgnoreOverflow(x, y);
return x > std::numeric_limits<UInt256>::max() - y;
}
template <typename T>
@ -104,24 +110,30 @@ namespace common
}
template <>
inline bool subOverflow(__int128 x, __int128 y, __int128 & res)
inline bool subOverflow(Int128 x, Int128 y, Int128 & res)
{
static constexpr __int128 min_int128 = minInt128();
static constexpr __int128 max_int128 = maxInt128();
res = subIgnoreOverflow(x, y);
return (y < 0 && x > max_int128 + y) || (y > 0 && x < min_int128 + y);
return (y < 0 && x > std::numeric_limits<Int128>::max() + y) ||
(y > 0 && x < std::numeric_limits<Int128>::min() + y);
}
template <>
inline bool subOverflow(wInt256 x, wInt256 y, wInt256 & res)
inline bool subOverflow(UInt128 x, UInt128 y, UInt128 & res)
{
res = subIgnoreOverflow(x, y);
return (y < 0 && x > std::numeric_limits<wInt256>::max() + y) ||
(y > 0 && x < std::numeric_limits<wInt256>::min() + y);
return x < y;
}
template <>
inline bool subOverflow(wUInt256 x, wUInt256 y, wUInt256 & res)
inline bool subOverflow(Int256 x, Int256 y, Int256 & res)
{
res = subIgnoreOverflow(x, y);
return (y < 0 && x > std::numeric_limits<Int256>::max() + y) ||
(y > 0 && x < std::numeric_limits<Int256>::min() + y);
}
template <>
inline bool subOverflow(UInt256 x, UInt256 y, UInt256 & res)
{
res = subIgnoreOverflow(x, y);
return x < y;
@ -151,36 +163,33 @@ namespace common
return __builtin_smulll_overflow(x, y, &res);
}
/// Overflow check is not implemented for big integers.
template <>
inline bool mulOverflow(__int128 x, __int128 y, __int128 & res)
inline bool mulOverflow(Int128 x, Int128 y, Int128 & res)
{
res = mulIgnoreOverflow(x, y);
if (!x || !y)
return false;
unsigned __int128 a = (x > 0) ? x : -x;
unsigned __int128 b = (y > 0) ? y : -y;
return mulIgnoreOverflow(a, b) / b != a;
return false;
}
template <>
inline bool mulOverflow(wInt256 x, wInt256 y, wInt256 & res)
inline bool mulOverflow(Int256 x, Int256 y, Int256 & res)
{
res = mulIgnoreOverflow(x, y);
if (!x || !y)
return false;
wInt256 a = (x > 0) ? x : -x;
wInt256 b = (y > 0) ? y : -y;
return mulIgnoreOverflow(a, b) / b != a;
return false;
}
template <>
inline bool mulOverflow(wUInt256 x, wUInt256 y, wUInt256 & res)
inline bool mulOverflow(UInt128 x, UInt128 y, UInt128 & res)
{
res = mulIgnoreOverflow(x, y);
if (!x || !y)
return false;
return res / y != x;
return false;
}
template <>
inline bool mulOverflow(UInt256 x, UInt256 y, UInt256 & res)
{
res = mulIgnoreOverflow(x, y);
return false;
}
}

View File

@ -5,16 +5,14 @@
#include <common/types.h>
#include <common/wide_integer.h>
using Int128 = __int128;
using wInt256 = wide::integer<256, signed>;
using wUInt256 = wide::integer<256, unsigned>;
using Int128 = wide::integer<128, signed>;
using UInt128 = wide::integer<128, unsigned>;
using Int256 = wide::integer<256, signed>;
using UInt256 = wide::integer<256, unsigned>;
static_assert(sizeof(wInt256) == 32);
static_assert(sizeof(wUInt256) == 32);
static constexpr __int128 minInt128() { return static_cast<unsigned __int128>(1) << 127; }
static constexpr __int128 maxInt128() { return (static_cast<unsigned __int128>(1) << 127) - 1; }
static_assert(sizeof(Int256) == 32);
static_assert(sizeof(UInt256) == 32);
/// The standard library type traits, such as std::is_arithmetic, with one exception
/// (std::common_type), are "set in stone". Attempting to specialize them causes undefined behavior.
@ -26,7 +24,7 @@ struct is_signed
};
template <> struct is_signed<Int128> { static constexpr bool value = true; };
template <> struct is_signed<wInt256> { static constexpr bool value = true; };
template <> struct is_signed<Int256> { static constexpr bool value = true; };
template <typename T>
inline constexpr bool is_signed_v = is_signed<T>::value;
@ -37,7 +35,8 @@ struct is_unsigned
static constexpr bool value = std::is_unsigned_v<T>;
};
template <> struct is_unsigned<wUInt256> { static constexpr bool value = true; };
template <> struct is_unsigned<UInt128> { static constexpr bool value = true; };
template <> struct is_unsigned<UInt256> { static constexpr bool value = true; };
template <typename T>
inline constexpr bool is_unsigned_v = is_unsigned<T>::value;
@ -51,8 +50,9 @@ struct is_integer
};
template <> struct is_integer<Int128> { static constexpr bool value = true; };
template <> struct is_integer<wInt256> { static constexpr bool value = true; };
template <> struct is_integer<wUInt256> { static constexpr bool value = true; };
template <> struct is_integer<UInt128> { static constexpr bool value = true; };
template <> struct is_integer<Int256> { static constexpr bool value = true; };
template <> struct is_integer<UInt256> { static constexpr bool value = true; };
template <typename T>
inline constexpr bool is_integer_v = is_integer<T>::value;
@ -64,7 +64,11 @@ struct is_arithmetic
static constexpr bool value = std::is_arithmetic_v<T>;
};
template <> struct is_arithmetic<__int128> { static constexpr bool value = true; };
template <> struct is_arithmetic<Int128> { static constexpr bool value = true; };
template <> struct is_arithmetic<UInt128> { static constexpr bool value = true; };
template <> struct is_arithmetic<Int256> { static constexpr bool value = true; };
template <> struct is_arithmetic<UInt256> { static constexpr bool value = true; };
template <typename T>
inline constexpr bool is_arithmetic_v = is_arithmetic<T>::value;
@ -75,9 +79,10 @@ struct make_unsigned
typedef std::make_unsigned_t<T> type;
};
template <> struct make_unsigned<Int128> { using type = unsigned __int128; };
template <> struct make_unsigned<wInt256> { using type = wUInt256; };
template <> struct make_unsigned<wUInt256> { using type = wUInt256; };
template <> struct make_unsigned<Int128> { using type = UInt128; };
template <> struct make_unsigned<UInt128> { using type = UInt128; };
template <> struct make_unsigned<Int256> { using type = UInt256; };
template <> struct make_unsigned<UInt256> { using type = UInt256; };
template <typename T> using make_unsigned_t = typename make_unsigned<T>::type;
@ -87,8 +92,10 @@ struct make_signed
typedef std::make_signed_t<T> type;
};
template <> struct make_signed<wInt256> { using type = wInt256; };
template <> struct make_signed<wUInt256> { using type = wInt256; };
template <> struct make_signed<Int128> { using type = Int128; };
template <> struct make_signed<UInt128> { using type = Int128; };
template <> struct make_signed<Int256> { using type = Int256; };
template <> struct make_signed<UInt256> { using type = Int256; };
template <typename T> using make_signed_t = typename make_signed<T>::type;
@ -98,8 +105,10 @@ struct is_big_int
static constexpr bool value = false;
};
template <> struct is_big_int<wInt256> { static constexpr bool value = true; };
template <> struct is_big_int<wUInt256> { static constexpr bool value = true; };
template <> struct is_big_int<Int128> { static constexpr bool value = true; };
template <> struct is_big_int<UInt128> { static constexpr bool value = true; };
template <> struct is_big_int<Int256> { static constexpr bool value = true; };
template <> struct is_big_int<UInt256> { static constexpr bool value = true; };
template <typename T>
inline constexpr bool is_big_int_v = is_big_int<T>::value;

View File

@ -30,9 +30,8 @@
#include <cstddef>
#include <cstring>
#include <type_traits>
#include <common/extended_types.h>
using int128_t = __int128;
using uint128_t = unsigned __int128;
namespace impl
{
@ -106,7 +105,7 @@ using UnsignedOfSize = typename SelectType
uint16_t,
uint32_t,
uint64_t,
uint128_t
__uint128_t
>::Result;
/// Holds the result of dividing an unsigned N-byte variable by 10^N resulting in
@ -313,7 +312,8 @@ namespace convert
}
}
static inline int digits10(uint128_t x)
template <typename T>
static inline int digits10(T x)
{
if (x < 10ULL)
return 1;
@ -346,8 +346,11 @@ static inline int digits10(uint128_t x)
return 12 + digits10(x / 1000000000000ULL);
}
static inline char * writeUIntText(uint128_t x, char * p)
template <typename T>
static inline char * writeUIntText(T x, char * p)
{
static_assert(is_unsigned_v<T>);
int len = digits10(x);
auto pp = p + len;
while (x >= 100)
@ -370,14 +373,28 @@ static inline char * writeLeadingMinus(char * pos)
return pos + 1;
}
static inline char * writeSIntText(int128_t x, char * pos)
template <typename T>
static inline char * writeSIntText(T x, char * pos)
{
static constexpr int128_t min_int128 = uint128_t(1) << 127;
static_assert(std::is_same_v<T, Int128> || std::is_same_v<T, Int256>);
if (unlikely(x == min_int128))
using UnsignedT = make_unsigned_t<T>;
static constexpr T min_int = UnsignedT(1) << (sizeof(T) * 8 - 1);
if (unlikely(x == min_int))
{
memcpy(pos, "-170141183460469231731687303715884105728", 40);
return pos + 40;
if constexpr (std::is_same_v<T, Int128>)
{
const char * res = "-170141183460469231731687303715884105728";
memcpy(pos, res, strlen(res));
return pos + strlen(res);
}
else if constexpr (std::is_same_v<T, Int256>)
{
const char * res = "-57896044618658097711785492504343953926634992332820282019728792003956564819968";
memcpy(pos, res, strlen(res));
return pos + strlen(res);
}
}
if (x < 0)
@ -385,7 +402,7 @@ static inline char * writeSIntText(int128_t x, char * pos)
x = -x;
pos = writeLeadingMinus(pos);
}
return writeUIntText(static_cast<uint128_t>(x), pos);
return writeUIntText(UnsignedT(x), pos);
}
}
@ -403,13 +420,25 @@ inline char * itoa(char8_t i, char * p)
}
template <>
inline char * itoa<uint128_t>(uint128_t i, char * p)
inline char * itoa(UInt128 i, char * p)
{
return impl::writeUIntText(i, p);
}
template <>
inline char * itoa<int128_t>(int128_t i, char * p)
inline char * itoa(Int128 i, char * p)
{
return impl::writeSIntText(i, p);
}
template <>
inline char * itoa(UInt256 i, char * p)
{
return impl::writeUIntText(i, p);
}
template <>
inline char * itoa(Int256 i, char * p)
{
return impl::writeSIntText(i, p);
}

View File

@ -4,7 +4,8 @@
#include <type_traits>
#include <utility>
template <class T, class Tag>
template <typename T, typename Tag>
struct StrongTypedef
{
private:
@ -38,14 +39,16 @@ public:
bool operator==(const Self & rhs) const { return t == rhs.t; }
bool operator<(const Self & rhs) const { return t < rhs.t; }
bool operator>(const Self & rhs) const { return t > rhs.t; }
T & toUnderType() { return t; }
const T & toUnderType() const { return t; }
};
namespace std
{
template <class T, class Tag>
template <typename T, typename Tag>
struct hash<StrongTypedef<T, Tag>>
{
size_t operator()(const StrongTypedef<T, Tag> & x) const

View File

@ -1,13 +1,15 @@
#pragma once
#include <stdexcept>
/// Throw DB::Exception-like exception before its definition.
/// DB::Exception derived from Poco::Exception derived from std::exception.
/// DB::Exception generally cought as Poco::Exception. std::exception generally has other catch blocks and could lead to other outcomes.
/// DB::Exception generally caught as Poco::Exception. std::exception generally has other catch blocks and could lead to other outcomes.
/// DB::Exception is not defined yet. It'd better to throw Poco::Exception but we do not want to include any big header here, even <string>.
/// So we throw some std::exception instead in the hope its catch block is the same as DB::Exception one.
template <typename T>
inline void throwError(const T & err)
[[noreturn]] inline void throwError(const T & err)
{
throw std::runtime_error(err);
}

View File

@ -58,9 +58,11 @@ public:
using signed_base_type = int64_t;
// ctors
constexpr integer() noexcept;
constexpr integer() noexcept = default;
template <typename T>
constexpr integer(T rhs) noexcept;
template <typename T>
constexpr integer(std::initializer_list<T> il) noexcept;
@ -108,9 +110,9 @@ public:
constexpr explicit operator bool() const noexcept;
template <class T>
using __integral_not_wide_integer_class = typename std::enable_if<std::is_arithmetic<T>::value, T>::type;
using _integral_not_wide_integer_class = typename std::enable_if<std::is_arithmetic<T>::value, T>::type;
template <class T, class = __integral_not_wide_integer_class<T>>
template <class T, class = _integral_not_wide_integer_class<T>>
constexpr operator T() const noexcept;
constexpr operator long double() const noexcept;
@ -119,25 +121,27 @@ public:
struct _impl;
base_type items[_impl::item_count];
private:
template <size_t Bits2, typename Signed2>
friend class integer;
friend class std::numeric_limits<integer<Bits, signed>>;
friend class std::numeric_limits<integer<Bits, unsigned>>;
base_type items[_impl::item_count];
};
template <typename T>
static constexpr bool ArithmeticConcept() noexcept;
template <class T1, class T2>
using __only_arithmetic = typename std::enable_if<ArithmeticConcept<T1>() && ArithmeticConcept<T2>()>::type;
using _only_arithmetic = typename std::enable_if<ArithmeticConcept<T1>() && ArithmeticConcept<T2>()>::type;
template <typename T>
static constexpr bool IntegralConcept() noexcept;
template <class T, class T2>
using __only_integer = typename std::enable_if<IntegralConcept<T>() && IntegralConcept<T2>()>::type;
using _only_integer = typename std::enable_if<IntegralConcept<T>() && IntegralConcept<T2>()>::type;
// Unary operators
template <size_t Bits, typename Signed>
@ -153,54 +157,55 @@ constexpr integer<Bits, Signed> operator+(const integer<Bits, Signed> & lhs) noe
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator*(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
std::common_type_t<Arithmetic, Arithmetic2> constexpr operator*(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator/(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
std::common_type_t<Arithmetic, Arithmetic2> constexpr operator/(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator+(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
std::common_type_t<Arithmetic, Arithmetic2> constexpr operator+(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator-(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
std::common_type_t<Arithmetic, Arithmetic2> constexpr operator-(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator%(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
template <typename Integral, typename Integral2, class = _only_integer<Integral, Integral2>>
std::common_type_t<Integral, Integral2> constexpr operator%(const Integral & rhs, const Integral2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator&(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
template <typename Integral, typename Integral2, class = _only_integer<Integral, Integral2>>
std::common_type_t<Integral, Integral2> constexpr operator&(const Integral & rhs, const Integral2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator|(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
template <typename Integral, typename Integral2, class = _only_integer<Integral, Integral2>>
std::common_type_t<Integral, Integral2> constexpr operator|(const Integral & rhs, const Integral2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator^(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
template <typename Integral, typename Integral2, class = _only_integer<Integral, Integral2>>
std::common_type_t<Integral, Integral2> constexpr operator^(const Integral & rhs, const Integral2 & lhs);
// TODO: Integral
template <size_t Bits, typename Signed>
constexpr integer<Bits, Signed> operator<<(const integer<Bits, Signed> & lhs, int n) noexcept;
template <size_t Bits, typename Signed>
constexpr integer<Bits, Signed> operator>>(const integer<Bits, Signed> & lhs, int n) noexcept;
@ -217,32 +222,32 @@ constexpr integer<Bits, Signed> operator>>(const integer<Bits, Signed> & lhs, In
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator<(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
constexpr bool operator<(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator>(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
constexpr bool operator>(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator<=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
constexpr bool operator<=(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator>=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
constexpr bool operator>=(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator==(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
constexpr bool operator==(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator!=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
constexpr bool operator!=(const Arithmetic & rhs, const Arithmetic2 & lhs);
}

View File

@ -5,6 +5,7 @@
/// (See at http://www.boost.org/LICENSE_1_0.txt)
#include "throwError.h"
#include <cmath>
#include <cfloat>
#include <cassert>
@ -81,7 +82,7 @@ public:
res.items[T::_impl::big(0)] = std::numeric_limits<typename wide::integer<Bits, Signed>::signed_base_type>::min();
return res;
}
return 0;
return wide::integer<Bits, Signed>(0);
}
static constexpr wide::integer<Bits, Signed> max() noexcept
@ -176,7 +177,7 @@ struct integer<Bits, Signed>::_impl
constexpr static bool is_negative(const integer<B, T> & n) noexcept
{
if constexpr (std::is_same_v<T, signed>)
return static_cast<signed_base_type>(n.items[big(0)]) < 0;
return static_cast<signed_base_type>(n.items[integer<B, T>::_impl::big(0)]) < 0;
else
return false;
}
@ -193,40 +194,36 @@ struct integer<Bits, Signed>::_impl
template <size_t B, class S>
constexpr static integer<B, S> make_positive(const integer<B, S> & n) noexcept
{
return is_negative(n) ? operator_unary_minus(n) : n;
return is_negative(n) ? integer<B, S>(operator_unary_minus(n)) : n;
}
template <typename T>
__attribute__((no_sanitize("undefined"))) constexpr static auto to_Integral(T f) noexcept
{
if constexpr (std::is_same_v<T, __int128>)
return f;
else if constexpr (std::is_signed_v<T>)
if constexpr (std::is_signed_v<T>)
return static_cast<int64_t>(f);
else
return static_cast<uint64_t>(f);
}
template <typename Integral>
constexpr static void wide_integer_from_bultin(integer<Bits, Signed> & self, Integral rhs) noexcept
constexpr static void wide_integer_from_builtin(integer<Bits, Signed> & self, Integral rhs) noexcept
{
self.items[0] = _impl::to_Integral(rhs);
if constexpr (std::is_same_v<Integral, __int128>)
self.items[1] = rhs >> base_bits;
static_assert(sizeof(Integral) <= sizeof(base_type));
constexpr const unsigned start = (sizeof(Integral) == 16) ? 2 : 1;
self.items[0] = _impl::to_Integral(rhs);
if constexpr (std::is_signed_v<Integral>)
{
if (rhs < 0)
{
for (unsigned i = start; i < item_count; ++i)
for (size_t i = 1; i < item_count; ++i)
self.items[i] = -1;
return;
}
}
for (unsigned i = start; i < item_count; ++i)
for (size_t i = 1; i < item_count; ++i)
self.items[i] = 0;
}
@ -239,7 +236,8 @@ struct integer<Bits, Signed>::_impl
* a_(n - 1) = a_n * max_int + b2, a_n <= max_int <- base case.
*/
template <class T>
constexpr static void set_multiplier(integer<Bits, Signed> & self, T t) noexcept {
constexpr static void set_multiplier(integer<Bits, Signed> & self, T t) noexcept
{
constexpr uint64_t max_int = std::numeric_limits<uint64_t>::max();
/// Implementation specific behaviour on overflow (if we don't check here, stack overflow will triggered in bigint_cast).
@ -260,7 +258,8 @@ struct integer<Bits, Signed>::_impl
self += static_cast<uint64_t>(t - alpha * static_cast<T>(max_int)); // += b_i
}
constexpr static void wide_integer_from_bultin(integer<Bits, Signed>& self, double rhs) noexcept {
constexpr static void wide_integer_from_builtin(integer<Bits, Signed>& self, double rhs) noexcept
{
constexpr int64_t max_int = std::numeric_limits<int64_t>::max();
constexpr int64_t min_int = std::numeric_limits<int64_t>::min();
@ -383,13 +382,13 @@ struct integer<Bits, Signed>::_impl
if (bit_shift)
lhs.items[big(items_shift)] |= std::numeric_limits<base_type>::max() << (base_bits - bit_shift);
for (unsigned i = item_count - items_shift; i < items_shift; ++i)
lhs.items[little(i)] = std::numeric_limits<base_type>::max();
for (unsigned i = 0; i < items_shift; ++i)
lhs.items[big(i)] = std::numeric_limits<base_type>::max();
}
else
{
for (unsigned i = item_count - items_shift; i < items_shift; ++i)
lhs.items[little(i)] = 0;
for (unsigned i = 0; i < items_shift; ++i)
lhs.items[big(i)] = 0;
}
return lhs;
@ -397,23 +396,23 @@ struct integer<Bits, Signed>::_impl
private:
template <typename T>
constexpr static base_type get_item(const T & x, unsigned number)
constexpr static base_type get_item(const T & x, unsigned idx)
{
if constexpr (IsWideInteger<T>::value)
{
if (number < T::_impl::item_count)
return x.items[number];
if (idx < T::_impl::item_count)
return x.items[idx];
return 0;
}
else
{
if constexpr (sizeof(T) <= sizeof(base_type))
{
if (!number)
if (0 == idx)
return x;
}
else if (number * sizeof(base_type) < sizeof(T))
return x >> (number * base_bits); // & std::numeric_limits<base_type>::max()
else if (idx * sizeof(base_type) < sizeof(T))
return x >> (idx * base_bits); // & std::numeric_limits<base_type>::max()
return 0;
}
}
@ -439,7 +438,7 @@ private:
for (unsigned i = 1; i < item_count; ++i)
{
if (underflows[i-1])
if (underflows[i - 1])
{
base_type & res_item = res.items[little(i)];
if (res_item == 0)
@ -472,7 +471,7 @@ private:
for (unsigned i = 1; i < item_count; ++i)
{
if (overflows[i-1])
if (overflows[i - 1])
{
base_type & res_item = res.items[little(i)];
++res_item;
@ -532,6 +531,17 @@ private:
res.items[little(2)] = r12 >> 64;
return res;
}
else if constexpr (Bits == 128 && sizeof(base_type) == 8)
{
using CompilerUInt128 = unsigned __int128;
CompilerUInt128 a = (CompilerUInt128(lhs.items[1]) << 64) + lhs.items[0];
CompilerUInt128 b = (CompilerUInt128(rhs.items[1]) << 64) + rhs.items[0];
CompilerUInt128 c = a * b;
integer<Bits, Signed> res;
res.items[0] = c;
res.items[1] = c >> 64;
return res;
}
else
{
integer<Bits, Signed> res{};
@ -657,7 +667,7 @@ public:
}
template <typename T>
constexpr static bool operator_more(const integer<Bits, Signed> & lhs, const T & rhs) noexcept
constexpr static bool operator_greater(const integer<Bits, Signed> & lhs, const T & rhs) noexcept
{
if constexpr (should_keep_size<T>())
{
@ -677,7 +687,7 @@ public:
else
{
static_assert(IsWideInteger<T>::value);
return std::common_type_t<integer<Bits, Signed>, T>::_impl::operator_more(T(lhs), rhs);
return std::common_type_t<integer<Bits, Signed>, T>::_impl::operator_greater(T(lhs), rhs);
}
}
@ -764,7 +774,6 @@ public:
}
}
private:
template <typename T>
constexpr static bool is_zero(const T & x)
{
@ -781,46 +790,65 @@ private:
}
/// returns quotient as result and remainder in numerator.
template <typename T>
constexpr static T divide(T & numerator, T && denominator)
template <size_t Bits2>
constexpr static integer<Bits2, unsigned> divide(integer<Bits2, unsigned> & numerator, integer<Bits2, unsigned> denominator)
{
if (is_zero(denominator))
throwError("divide by zero");
static_assert(std::is_unsigned_v<Signed>);
T & n = numerator;
T & d = denominator;
T x = 1;
T quotient = 0;
while (!operator_more(d, n) && operator_eq(operator_amp(shift_right(d, base_bits * item_count - 1), 1), 0))
if constexpr (Bits == 128 && sizeof(base_type) == 8)
{
x = shift_left(x, 1);
d = shift_left(d, 1);
using CompilerUInt128 = unsigned __int128;
CompilerUInt128 a = (CompilerUInt128(numerator.items[1]) << 64) + numerator.items[0];
CompilerUInt128 b = (CompilerUInt128(denominator.items[1]) << 64) + denominator.items[0];
CompilerUInt128 c = a / b;
integer<Bits, Signed> res;
res.items[0] = c;
res.items[1] = c >> 64;
CompilerUInt128 remainder = a - b * c;
numerator.items[0] = remainder;
numerator.items[1] = remainder >> 64;
return res;
}
while (!operator_eq(x, 0))
if (is_zero(denominator))
throwError("Division by zero");
integer<Bits2, unsigned> x = 1;
integer<Bits2, unsigned> quotient = 0;
while (!operator_greater(denominator, numerator) && is_zero(operator_amp(shift_right(denominator, Bits2 - 1), 1)))
{
if (!operator_more(d, n))
x = shift_left(x, 1);
denominator = shift_left(denominator, 1);
}
while (!is_zero(x))
{
if (!operator_greater(denominator, numerator))
{
n = operator_minus(n, d);
numerator = operator_minus(numerator, denominator);
quotient = operator_pipe(quotient, x);
}
x = shift_right(x, 1);
d = shift_right(d, 1);
denominator = shift_right(denominator, 1);
}
return quotient;
}
public:
template <typename T>
constexpr static auto operator_slash(const integer<Bits, Signed> & lhs, const T & rhs)
{
if constexpr (should_keep_size<T>())
{
integer<Bits, Signed> numerator = make_positive(lhs);
integer<Bits, Signed> quotient = divide(numerator, make_positive(integer<Bits, Signed>(rhs)));
integer<Bits, unsigned> numerator = make_positive(lhs);
integer<Bits, unsigned> denominator = make_positive(integer<Bits, Signed>(rhs));
integer<Bits, unsigned> quotient = integer<Bits, unsigned>::_impl::divide(numerator, std::move(denominator));
if (std::is_same_v<Signed, signed> && is_negative(rhs) != is_negative(lhs))
quotient = operator_unary_minus(quotient);
@ -838,8 +866,9 @@ public:
{
if constexpr (should_keep_size<T>())
{
integer<Bits, Signed> remainder = make_positive(lhs);
divide(remainder, make_positive(integer<Bits, Signed>(rhs)));
integer<Bits, unsigned> remainder = make_positive(lhs);
integer<Bits, unsigned> denominator = make_positive(integer<Bits, Signed>(rhs));
integer<Bits, unsigned>::_impl::divide(remainder, std::move(denominator));
if (std::is_same_v<Signed, signed> && is_negative(lhs))
remainder = operator_unary_minus(remainder);
@ -905,7 +934,7 @@ public:
++c;
}
else
throwError("invalid char from");
throwError("Invalid char from");
}
}
else
@ -913,7 +942,7 @@ public:
while (*c)
{
if (*c < '0' || *c > '9')
throwError("invalid char from");
throwError("Invalid char from");
res = multiply(res, 10U);
res = plus(res, *c - '0');
@ -930,11 +959,6 @@ public:
// Members
template <size_t Bits, typename Signed>
constexpr integer<Bits, Signed>::integer() noexcept
: items{}
{}
template <size_t Bits, typename Signed>
template <typename T>
constexpr integer<Bits, Signed>::integer(T rhs) noexcept
@ -943,7 +967,7 @@ constexpr integer<Bits, Signed>::integer(T rhs) noexcept
if constexpr (IsWideInteger<T>::value)
_impl::wide_integer_from_wide_integer(*this, rhs);
else
_impl::wide_integer_from_bultin(*this, rhs);
_impl::wide_integer_from_builtin(*this, rhs);
}
template <size_t Bits, typename Signed>
@ -956,10 +980,19 @@ constexpr integer<Bits, Signed>::integer(std::initializer_list<T> il) noexcept
if constexpr (IsWideInteger<T>::value)
_impl::wide_integer_from_wide_integer(*this, *il.begin());
else
_impl::wide_integer_from_bultin(*this, *il.begin());
_impl::wide_integer_from_builtin(*this, *il.begin());
}
else if (il.size() == 0)
{
_impl::wide_integer_from_builtin(*this, 0);
}
else
_impl::wide_integer_from_bultin(*this, 0);
{
auto it = il.begin();
for (size_t i = 0; i < _impl::item_count; ++i)
if (it < il.end())
items[i] = *it;
}
}
template <size_t Bits, typename Signed>
@ -974,7 +1007,7 @@ template <size_t Bits, typename Signed>
template <typename T>
constexpr integer<Bits, Signed> & integer<Bits, Signed>::operator=(T rhs) noexcept
{
_impl::wide_integer_from_bultin(*this, rhs);
_impl::wide_integer_from_builtin(*this, rhs);
return *this;
}
@ -1057,7 +1090,7 @@ constexpr integer<Bits, Signed> & integer<Bits, Signed>::operator>>=(int n) noex
{
if (static_cast<size_t>(n) >= Bits)
{
if (is_negative(*this))
if (_impl::is_negative(*this))
*this = -1;
else
*this = 0;
@ -1107,16 +1140,17 @@ template <size_t Bits, typename Signed>
template <class T, class>
constexpr integer<Bits, Signed>::operator T() const noexcept
{
if constexpr (std::is_same_v<T, __int128>)
{
static_assert(Bits >= 128);
return (__int128(items[1]) << 64) | items[0];
}
else
{
static_assert(std::numeric_limits<T>::is_integer);
return items[0];
}
static_assert(std::numeric_limits<T>::is_integer);
/// NOTE: memcpy will suffice, but unfortunately, this function is constexpr.
using UnsignedT = std::make_unsigned_t<T>;
UnsignedT res{};
for (unsigned i = 0; i < _impl::item_count && i < (sizeof(T) + sizeof(base_type) - 1) / sizeof(base_type); ++i)
res += UnsignedT(items[i]) << (sizeof(base_type) * 8 * i);
return res;
}
template <size_t Bits, typename Signed>
@ -1280,7 +1314,7 @@ template <size_t Bits, typename Signed>
constexpr integer<Bits, Signed> operator<<(const integer<Bits, Signed> & lhs, int n) noexcept
{
if (static_cast<size_t>(n) >= Bits)
return 0;
return integer<Bits, Signed>(0);
if (n <= 0)
return lhs;
return integer<Bits, Signed>::_impl::shift_left(lhs, n);
@ -1289,7 +1323,7 @@ template <size_t Bits, typename Signed>
constexpr integer<Bits, Signed> operator>>(const integer<Bits, Signed> & lhs, int n) noexcept
{
if (static_cast<size_t>(n) >= Bits)
return 0;
return integer<Bits, Signed>(0);
if (n <= 0)
return lhs;
return integer<Bits, Signed>::_impl::shift_right(lhs, n);
@ -1309,7 +1343,7 @@ constexpr bool operator<(const Arithmetic & lhs, const Arithmetic2 & rhs)
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator>(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs)
{
return std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_more(lhs, rhs);
return std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_greater(lhs, rhs);
}
template <typename Arithmetic, typename Arithmetic2, class>
constexpr bool operator>(const Arithmetic & lhs, const Arithmetic2 & rhs)
@ -1332,7 +1366,7 @@ constexpr bool operator<=(const Arithmetic & lhs, const Arithmetic2 & rhs)
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator>=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs)
{
return std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_more(lhs, rhs)
return std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_greater(lhs, rhs)
|| std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_eq(lhs, rhs);
}
template <typename Arithmetic, typename Arithmetic2, class>

View File

@ -1,9 +1,12 @@
#pragma once
#include <string>
#include <ostream>
#include <fmt/format.h>
#include "wide_integer.h"
namespace wide
{
@ -33,3 +36,34 @@ inline std::string to_string(const integer<Bits, Signed> & n)
}
}
template <size_t Bits, typename Signed>
std::ostream & operator<<(std::ostream & out, const wide::integer<Bits, Signed> & value)
{
return out << to_string(value);
}
/// See https://fmt.dev/latest/api.html#formatting-user-defined-types
template <size_t Bits, typename Signed>
struct fmt::formatter<wide::integer<Bits, Signed>>
{
constexpr auto parse(format_parse_context & ctx)
{
auto it = ctx.begin();
auto end = ctx.end();
/// Only support {}.
if (it != end && *it != '}')
throw format_error("invalid format");
return it;
}
template <typename FormatContext>
auto format(const wide::integer<Bits, Signed> & value, FormatContext & ctx)
{
return format_to(ctx.out(), "{}", to_string(value));
}
};

@ -1 +1 @@
Subproject commit f915d35b2de676683493c86c585141a1e1c83334
Subproject commit 7d73d7610db31d4e1ecde0fb3a7ee90ef371207f

View File

@ -308,6 +308,8 @@ function run_tests
01354_order_by_tuple_collate_const
01355_ilike
01411_bayesian_ab_testing
01798_uniq_theta_sketch
01799_long_uniq_theta_sketch
collate
collation
_orc_

View File

@ -38,3 +38,4 @@ We recommend using this function in almost all scenarios.
- [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64)
- [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniqhll12.md#agg_function-uniqhll12)
- [uniqExact](../../../sql-reference/aggregate-functions/reference/uniqexact.md#agg_function-uniqexact)
- [uniqThetaSketch](../../../sql-reference/aggregate-functions/reference/uniqthetasketch.md#agg_function-uniqthetasketch)

View File

@ -49,3 +49,4 @@ Compared to the [uniq](../../../sql-reference/aggregate-functions/reference/uniq
- [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64)
- [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniqhll12.md#agg_function-uniqhll12)
- [uniqExact](../../../sql-reference/aggregate-functions/reference/uniqexact.md#agg_function-uniqexact)
- [uniqThetaSketch](../../../sql-reference/aggregate-functions/reference/uniqthetasketch.md#agg_function-uniqthetasketch)

View File

@ -23,3 +23,4 @@ The function takes a variable number of parameters. Parameters can be `Tuple`, `
- [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq)
- [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniqcombined)
- [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniqhll12)
- [uniqThetaSketch](../../../sql-reference/aggregate-functions/reference/uniqthetasketch.md#agg_function-uniqthetasketch)

View File

@ -37,3 +37,4 @@ We dont recommend using this function. In most cases, use the [uniq](../../..
- [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq)
- [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md#agg_function-uniqcombined)
- [uniqExact](../../../sql-reference/aggregate-functions/reference/uniqexact.md#agg_function-uniqexact)
- [uniqThetaSketch](../../../sql-reference/aggregate-functions/reference/uniqthetasketch.md#agg_function-uniqthetasketch)

View File

@ -0,0 +1,39 @@
---
toc_priority: 195
---
# uniqThetaSketch {#agg_function-uniqthetasketch}
Calculates the approximate number of different argument values, using the [Theta Sketch Framework](https://datasketches.apache.org/docs/Theta/ThetaSketchFramework.html).
``` sql
uniqThetaSketch(x[, ...])
```
**Arguments**
The function takes a variable number of parameters. Parameters can be `Tuple`, `Array`, `Date`, `DateTime`, `String`, or numeric types.
**Returned value**
- A [UInt64](../../../sql-reference/data-types/int-uint.md)-type number.
**Implementation details**
Function:
- Calculates a hash for all parameters in the aggregate, then uses it in calculations.
- Uses the [KMV](https://datasketches.apache.org/docs/Theta/InverseEstimate.html) algorithm to approximate the number of different argument values.
4096(2^12) 64-bit sketch are used. The size of the state is about 41 KB.
- The relative error is 3.125% (95% confidence), see the [relative error table](https://datasketches.apache.org/docs/Theta/ThetaErrorTable.html) for detail.
**See Also**
- [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq)
- [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md#agg_function-uniqcombined)
- [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64)
- [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniqhll12.md#agg_function-uniqhll12)
- [uniqExact](../../../sql-reference/aggregate-functions/reference/uniqexact.md#agg_function-uniqexact)

View File

@ -95,7 +95,9 @@ LAYOUT(FLAT(INITIAL_ARRAY_SIZE 50000 MAX_ARRAY_SIZE 5000000))
The dictionary is completely stored in memory in the form of a hash table. The dictionary can contain any number of elements with any identifiers In practice, the number of keys can reach tens of millions of items.
The hash table will be preallocated (this will make dictionary load faster), if the is approx number of total rows is known, this is supported only if the source is `clickhouse` without any `<where>` (since in case of `<where>` you can filter out too much rows and the dictionary will allocate too much memory, that will not be used eventually).
If `preallocate` is `true` (default is `false`) the hash table will be preallocated (this will make dictionary load faster). But note that you should use it only if:
- the source support approximate number of elements (for now it is supported only by the `ClickHouse` source)
- there is no duplicates in the data (otherwise it may increase memory usage for the hashtable)
All types of sources are supported. When updating, data (from a file or from a table) is read in its entirety.
@ -103,21 +105,23 @@ Configuration example:
``` xml
<layout>
<hashed />
<hashed>
<preallocate>0</preallocate>
</hashed>
</layout>
```
or
``` sql
LAYOUT(HASHED())
LAYOUT(HASHED(PREALLOCATE 0))
```
### sparse_hashed {#dicts-external_dicts_dict_layout-sparse_hashed}
Similar to `hashed`, but uses less memory in favor more CPU usage.
It will be also preallocated so as `hashed`, note that it is even more significant for `sparse_hashed`.
It will be also preallocated so as `hashed` (with `preallocate` set to `true`), and note that it is even more significant for `sparse_hashed`.
Configuration example:
@ -127,8 +131,10 @@ Configuration example:
</layout>
```
or
``` sql
LAYOUT(SPARSE_HASHED())
LAYOUT(SPARSE_HASHED([PREALLOCATE 0]))
```
### complex_key_hashed {#complex-key-hashed}

View File

@ -1544,3 +1544,52 @@ SELECT arrayCumSumNonNegative([1, 1, -4, 1]) AS res
```
Note that the `arraySumNonNegative` is a [higher-order function](../../sql-reference/functions/index.md#higher-order-functions). You can pass a lambda function to it as the first argument.
## arrayProduct {#arrayproduct}
Multiplies elements of an [array](../../sql-reference/data-types/array.md).
**Syntax**
``` sql
arrayProduct(arr)
```
**Arguments**
- `arr` — [Array](../../sql-reference/data-types/array.md) of numeric values.
**Returned value**
- A product of array's elements.
Type: [Float64](../../sql-reference/data-types/float.md).
**Examples**
Query:
``` sql
SELECT arrayProduct([1,2,3,4,5,6]) as res;
```
Result:
``` text
┌─res───┐
│ 720 │
└───────┘
```
Query:
``` sql
SELECT arrayProduct([toDecimal64(1,8), toDecimal64(2,8), toDecimal64(3,8)]) as res, toTypeName(res);
```
Return value type is always [Float64](../../sql-reference/data-types/float.md). Result:
``` text
┌─res─┬─toTypeName(arrayProduct(array(toDecimal64(1, 8), toDecimal64(2, 8), toDecimal64(3, 8))))─┐
│ 6 │ Float64 │
└─────┴──────────────────────────────────────────────────────────────────────────────────────────┘
```

View File

@ -47,9 +47,9 @@ Union
### EXPLAIN AST {#explain-ast}
Dump query AST.
Dump query AST. Supports all types of queries, not only `SELECT`.
Example:
Examples:
```sql
EXPLAIN AST SELECT 1;
@ -63,6 +63,22 @@ SelectWithUnionQuery (children 1)
Literal UInt64_1
```
```sql
EXPLAIN AST ALTER TABLE t1 DELETE WHERE date = today();
```
```sql
explain
AlterQuery t1 (children 1)
ExpressionList (children 1)
AlterCommand 27 (children 1)
Function equals (children 1)
ExpressionList (children 2)
Identifier date
Function today (children 1)
ExpressionList
```
### EXPLAIN SYNTAX {#explain-syntax}
Returns query after syntax optimizations.

View File

@ -105,7 +105,7 @@ SELECT * FROM s3_engine_table LIMIT 2;
## Примеры использования {#usage-examples}
Предположим, у нас есть несколько файлов в формате TSV со следующими URL-адресами в HDFS:
Предположим, у нас есть несколько файлов в формате TSV со следующими URL-адресами в S3:
- 'https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_1.csv'
- 'https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_2.csv'

View File

@ -1,248 +0,0 @@
---
toc_priority: 58
toc_title: "Советы по эксплуатации"
---
# Советы по эксплуатации {#sovety-po-ekspluatatsii}
## CPU Scaling Governor {#cpu-scaling-governor}
Всегда используйте `performance` scaling governor. `ondemand` scaling governor работает намного хуже при постоянно высоком спросе.
``` bash
$ echo 'performance' | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
```
## Ограничение CPU {#ogranichenie-cpu}
Процессоры могут перегреваться. С помощью `dmesg` можно увидеть, если тактовая частота процессора была ограничена из-за перегрева.
Также ограничение может устанавливаться снаружи на уровне дата-центра. С помощью `turbostat` можно за этим наблюдать под нагрузкой.
## Оперативная память {#operativnaia-pamiat}
Для небольших объёмов данных (до ~200 Гб в сжатом виде) лучше всего использовать столько памяти не меньше, чем объём данных.
Для больших объёмов данных, при выполнении интерактивных (онлайн) запросов, стоит использовать разумный объём оперативной памяти (128 Гб или более) для того, чтобы горячее подмножество данных поместилось в кеше страниц.
Даже для объёмов данных в ~50 Тб на сервер, использование 128 Гб оперативной памяти намного лучше для производительности выполнения запросов, чем 64 Гб.
Не выключайте overcommit. Значение `cat /proc/sys/vm/overcommit_memory` должно быть 0 or 1. Выполните:
``` bash
$ echo 0 | sudo tee /proc/sys/vm/overcommit_memory
```
## Huge Pages {#huge-pages}
Механизм прозрачных huge pages нужно отключить. Он мешает работе аллокаторов памяти, что приводит к значительной деградации производительности.
``` bash
$ echo 'madvise' | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
```
С помощью `perf top` можно наблюдать за временем, проведенном в ядре операционной системы для управления памятью.
Постоянные huge pages так же не нужно аллоцировать.
## Подсистема хранения {#podsistema-khraneniia}
Если ваш бюджет позволяет использовать SSD, используйте SSD.
В противном случае используйте HDD. SATA HDDs 7200 RPM подойдут.
Предпочитайте много серверов с локальными жесткими дисками вместо меньшего числа серверов с подключенными дисковыми полками.
Но для хранения архивов с редкими запросами полки всё же подходят.
## RAID {#raid}
При использовании HDD можно объединить их RAID-10, RAID-5, RAID-6 или RAID-50.
Лучше использовать программный RAID в Linux (`mdadm`). Лучше не использовать LVM.
При создании RAID-10, нужно выбрать `far` расположение.
Если бюджет позволяет, лучше выбрать RAID-10.
На более чем 4 дисках вместо RAID-5 нужно использовать RAID-6 (предпочтительнее) или RAID-50.
При использовании RAID-5, RAID-6 или RAID-50, нужно всегда увеличивать stripe_cache_size, так как значение по умолчанию выбрано не самым удачным образом.
``` bash
$ echo 4096 | sudo tee /sys/block/md2/md/stripe_cache_size
```
Точное число стоит вычислять из числа устройств и размер блока по формуле: `2 * num_devices * chunk_size_in_bytes / 4096`.
Размер блока в 1024 Кб подходит для всех конфигураций RAID.
Никогда не указывайте слишком маленький или слишком большой размер блока.
На SSD можно использовать RAID-0.
Вне зависимости от использования RAID, всегда используйте репликацию для безопасности данных.
Включите NCQ с длинной очередью. Для HDD стоит выбрать планировщик CFQ, а для SSD — noop. Не стоит уменьшать настройку readahead.
На HDD стоит включать кеш записи.
## Файловая система {#failovaia-sistema}
Ext4 самый проверенный вариант. Укажите опции монтирования `noatime,nobarrier`.
XFS также подходит, но не так тщательно протестирована в сочетании с ClickHouse.
Большинство других файловых систем также должны нормально работать. Файловые системы с отложенной аллокацией работают лучше.
## Ядро Linux {#iadro-linux}
Не используйте слишком старое ядро Linux.
## Сеть {#set}
При использовании IPv6, стоит увеличить размер кеша маршрутов.
Ядра Linux до 3.2 имели массу проблем в реализации IPv6.
Предпочитайте как минимум 10 Гбит сеть. 1 Гбит также будет работать, но намного хуже для починки реплик с десятками терабайт данных или для обработки распределенных запросов с большим объёмом промежуточных данных.
## ZooKeeper {#zookeeper}
Вероятно вы уже используете ZooKeeper для других целей. Можно использовать ту же инсталляцию ZooKeeper, если она не сильно перегружена.
Лучше использовать свежую версию ZooKeeper, как минимум 3.4.9. Версия в стабильных дистрибутивах Linux может быть устаревшей.
Никогда не используете написанные вручную скрипты для переноса данных между разными ZooKeeper кластерами, потому что результат будет некорректный для sequential нод. Никогда не используйте утилиту «zkcopy», по той же причине: https://github.com/ksprojects/zkcopy/issues/15
Если вы хотите разделить существующий ZooKeeper кластер на два, правильный способ - увеличить количество его реплик, а затем переконфигурировать его как два независимых кластера.
Не запускайте ZooKeeper на тех же серверах, что и ClickHouse. Потому что ZooKeeper очень чувствителен к задержкам, а ClickHouse может использовать все доступные системные ресурсы.
С настройками по умолчанию, ZooKeeper является бомбой замедленного действия:
> Сервер ZooKeeper не будет удалять файлы со старыми снепшоты и логами при использовании конфигурации по умолчанию (см. autopurge), это является ответственностью оператора.
Эту бомбу нужно обезвредить.
Далее описана конфигурация ZooKeeper (3.5.1), используемая в боевом окружении Яндекс.Метрики на момент 20 мая 2017 года:
zoo.cfg:
``` bash
# http://hadoop.apache.org/zookeeper/docs/current/zookeeperAdmin.html
# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial
# synchronization phase can take
initLimit=30000
# The number of ticks that can pass between
# sending a request and getting an acknowledgement
syncLimit=10
maxClientCnxns=2000
maxSessionTimeout=60000000
# the directory where the snapshot is stored.
dataDir=/opt/zookeeper/{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/data
# Place the dataLogDir to a separate physical disc for better performance
dataLogDir=/opt/zookeeper/{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/logs
autopurge.snapRetainCount=10
autopurge.purgeInterval=1
# To avoid seeks ZooKeeper allocates space in the transaction log file in
# blocks of preAllocSize kilobytes. The default block size is 64M. One reason
# for changing the size of the blocks is to reduce the block size if snapshots
# are taken more often. (Also, see snapCount).
preAllocSize=131072
# Clients can submit requests faster than ZooKeeper can process them,
# especially if there are a lot of clients. To prevent ZooKeeper from running
# out of memory due to queued requests, ZooKeeper will throttle clients so that
# there is no more than globalOutstandingLimit outstanding requests in the
# system. The default limit is 1,000.ZooKeeper logs transactions to a
# transaction log. After snapCount transactions are written to a log file a
# snapshot is started and a new transaction log file is started. The default
# snapCount is 10,000.
snapCount=3000000
# If this option is defined, requests will be will logged to a trace file named
# traceFile.year.month.day.
#traceFile=
# Leader accepts client connections. Default value is "yes". The leader machine
# coordinates updates. For higher update throughput at thes slight expense of
# read throughput the leader can be configured to not accept clients and focus
# on coordination.
leaderServes=yes
standaloneEnabled=false
dynamicConfigFile=/etc/zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/conf/zoo.cfg.dynamic
```
Версия Java:
``` text
Java(TM) SE Runtime Environment (build 1.8.0_25-b17)
Java HotSpot(TM) 64-Bit Server VM (build 25.25-b02, mixed mode)
```
Параметры JVM:
``` bash
NAME=zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}
ZOOCFGDIR=/etc/$NAME/conf
# TODO this is really ugly
# How to find out, which jars are needed?
# seems, that log4j requires the log4j.properties file to be in the classpath
CLASSPATH="$ZOOCFGDIR:/usr/build/classes:/usr/build/lib/*.jar:/usr/share/zookeeper/zookeeper-3.5.1-metrika.jar:/usr/share/zookeeper/slf4j-log4j12-1.7.5.jar:/usr/share/zookeeper/slf4j-api-1.7.5.jar:/usr/share/zookeeper/servlet-api-2.5-20081211.jar:/usr/share/zookeeper/netty-3.7.0.Final.jar:/usr/share/zookeeper/log4j-1.2.16.jar:/usr/share/zookeeper/jline-2.11.jar:/usr/share/zookeeper/jetty-util-6.1.26.jar:/usr/share/zookeeper/jetty-6.1.26.jar:/usr/share/zookeeper/javacc.jar:/usr/share/zookeeper/jackson-mapper-asl-1.9.11.jar:/usr/share/zookeeper/jackson-core-asl-1.9.11.jar:/usr/share/zookeeper/commons-cli-1.2.jar:/usr/src/java/lib/*.jar:/usr/etc/zookeeper"
ZOOCFG="$ZOOCFGDIR/zoo.cfg"
ZOO_LOG_DIR=/var/log/$NAME
USER=zookeeper
GROUP=zookeeper
PIDDIR=/var/run/$NAME
PIDFILE=$PIDDIR/$NAME.pid
SCRIPTNAME=/etc/init.d/$NAME
JAVA=/usr/bin/java
ZOOMAIN="org.apache.zookeeper.server.quorum.QuorumPeerMain"
ZOO_LOG4J_PROP="INFO,ROLLINGFILE"
JMXLOCALONLY=false
JAVA_OPTS="-Xms{{ '{{' }} cluster.get('xms','128M') {{ '{{' }} '}}' }} \
-Xmx{{ '{{' }} cluster.get('xmx','1G') {{ '{{' }} '}}' }} \
-Xloggc:/var/log/$NAME/zookeeper-gc.log \
-XX:+UseGCLogFileRotation \
-XX:NumberOfGCLogFiles=16 \
-XX:GCLogFileSize=16M \
-verbose:gc \
-XX:+PrintGCTimeStamps \
-XX:+PrintGCDateStamps \
-XX:+PrintGCDetails
-XX:+PrintTenuringDistribution \
-XX:+PrintGCApplicationStoppedTime \
-XX:+PrintGCApplicationConcurrentTime \
-XX:+PrintSafepointStatistics \
-XX:+UseParNewGC \
-XX:+UseConcMarkSweepGC \
-XX:+CMSParallelRemarkEnabled"
```
Salt init:
``` text
description "zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }} centralized coordination service"
start on runlevel [2345]
stop on runlevel [!2345]
respawn
limit nofile 8192 8192
pre-start script
[ -r "/etc/zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/conf/environment" ] || exit 0
. /etc/zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/conf/environment
[ -d $ZOO_LOG_DIR ] || mkdir -p $ZOO_LOG_DIR
chown $USER:$GROUP $ZOO_LOG_DIR
end script
script
. /etc/zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/conf/environment
[ -r /etc/default/zookeeper ] && . /etc/default/zookeeper
if [ -z "$JMXDISABLE" ]; then
JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY"
fi
exec start-stop-daemon --start -c $USER --exec $JAVA --name zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }} \
-- -cp $CLASSPATH $JAVA_OPTS -Dzookeeper.log.dir=${ZOO_LOG_DIR} \
-Dzookeeper.root.logger=${ZOO_LOG4J_PROP} $ZOOMAIN $ZOOCFG
end script
```

1
docs/ru/operations/tips.md Symbolic link
View File

@ -0,0 +1 @@
../../en/operations/tips.md

View File

@ -1528,3 +1528,52 @@ SELECT arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]);
└────────────────────────────────────────---──┘
```
## arrayProduct {#arrayproduct}
Возвращает произведение элементов [массива](../../sql-reference/data-types/array.md).
**Синтаксис**
``` sql
arrayProduct(arr)
```
**Аргументы**
- `arr` — [массив](../../sql-reference/data-types/array.md) числовых значений.
**Возвращаемое значение**
- Произведение элементов массива.
Тип: [Float64](../../sql-reference/data-types/float.md).
**Примеры**
Запрос:
``` sql
SELECT arrayProduct([1,2,3,4,5,6]) as res;
```
Результат:
``` text
┌─res───┐
│ 720 │
└───────┘
```
Запрос:
``` sql
SELECT arrayProduct([toDecimal64(1,8), toDecimal64(2,8), toDecimal64(3,8)]) as res, toTypeName(res);
```
Возвращаемое значение всегда имеет тип [Float64](../../sql-reference/data-types/float.md). Результат:
``` text
┌─res─┬─toTypeName(arrayProduct(array(toDecimal64(1, 8), toDecimal64(2, 8), toDecimal64(3, 8))))─┐
│ 6 │ Float64 │
└─────┴──────────────────────────────────────────────────────────────────────────────────────────┘
```

View File

@ -47,9 +47,9 @@ Union
### EXPLAIN AST {#explain-ast}
Дамп AST запроса.
Дамп AST запроса. Поддерживает все типы запросов, не только `SELECT`.
Пример:
Примеры:
```sql
EXPLAIN AST SELECT 1;
@ -63,6 +63,22 @@ SelectWithUnionQuery (children 1)
Literal UInt64_1
```
```sql
EXPLAIN AST ALTER TABLE t1 DELETE WHERE date = today();
```
```sql
explain
AlterQuery t1 (children 1)
ExpressionList (children 1)
AlterCommand 27 (children 1)
Function equals (children 1)
ExpressionList (children 2)
Identifier date
Function today (children 1)
ExpressionList
```
### EXPLAIN SYNTAX {#explain-syntax}
Возвращает текст запроса после применения синтаксических оптимизаций.

View File

@ -1461,18 +1461,25 @@ private:
// when `lambda()` function gets substituted into a wrong place.
// To avoid dealing with these cases, run the check only for the
// queries we were able to successfully execute.
// The final caveat is that sometimes WITH queries are not executed,
// Another caveat is that sometimes WITH queries are not executed,
// if they are not referenced by the main SELECT, so they can still
// have the aforementioned problems. Disable this check for such
// queries, for lack of a better solution.
if (!have_error && queryHasWithClause(parsed_query.get()))
// There is also a problem that fuzzer substitutes positive Int64
// literals or Decimal literals, which are then parsed back as
// UInt64, and suddenly duplicate alias substitition starts or stops
// working (ASTWithAlias::formatImpl) or something like that.
// So we compare not even the first and second formatting of the
// query, but second and third.
// If you have to add any more workarounds to this check, just remove
// it altogether, it's not so useful.
if (!have_error && !queryHasWithClause(parsed_query.get()))
{
ASTPtr parsed_formatted_query;
ASTPtr ast_2;
try
{
const auto * tmp_pos = query_to_send.c_str();
parsed_formatted_query = parseQuery(tmp_pos,
tmp_pos + query_to_send.size(),
ast_2 = parseQuery(tmp_pos, tmp_pos + query_to_send.size(),
false /* allow_multi_statements */);
}
catch (Exception & e)
@ -1483,25 +1490,31 @@ private:
}
}
if (parsed_formatted_query)
if (ast_2)
{
const auto formatted_twice
= parsed_formatted_query->formatForErrorMessage();
const auto text_2 = ast_2->formatForErrorMessage();
const auto * tmp_pos = text_2.c_str();
const auto ast_3 = parseQuery(tmp_pos, tmp_pos + text_2.size(),
false /* allow_multi_statements */);
const auto text_3 = ast_3->formatForErrorMessage();
if (formatted_twice != query_to_send)
if (text_3 != text_2)
{
fmt::print(stderr, "The query formatting is broken.\n");
printChangedSettings();
fmt::print(stderr, "Got the following (different) text after formatting the fuzzed query and parsing it back:\n'{}'\n, expected:\n'{}'\n",
formatted_twice, query_to_send);
fmt::print(stderr,
"Got the following (different) text after formatting the fuzzed query and parsing it back:\n'{}'\n, expected:\n'{}'\n",
text_3, text_2);
fmt::print(stderr, "In more detail:\n");
fmt::print(stderr, "AST-1:\n'{}'\n", parsed_query->dumpTree());
fmt::print(stderr, "AST-1 (generated by fuzzer):\n'{}'\n", parsed_query->dumpTree());
fmt::print(stderr, "Text-1 (AST-1 formatted):\n'{}'\n", query_to_send);
fmt::print(stderr, "AST-2 (Text-1 parsed):\n'{}'\n", parsed_formatted_query->dumpTree());
fmt::print(stderr, "Text-2 (AST-2 formatted):\n'{}'\n", formatted_twice);
fmt::print(stderr, "Text-1 must be equal to Text-2, but it is not.\n");
fmt::print(stderr, "AST-2 (Text-1 parsed):\n'{}'\n", ast_2->dumpTree());
fmt::print(stderr, "Text-2 (AST-2 formatted):\n'{}'\n", text_2);
fmt::print(stderr, "AST-3 (Text-2 parsed):\n'{}'\n", ast_3->dumpTree());
fmt::print(stderr, "Text-3 (AST-3 formatted):\n'{}'\n", text_3);
fmt::print(stderr, "Text-3 must be equal to Text-2, but it is not.\n");
exit(1);
}
@ -1518,6 +1531,11 @@ private:
server_exception.reset();
client_exception.reset();
have_error = false;
// We have to reinitialize connection after errors, because it
// might have gotten into a wrong state and we'll get false
// positives about "Unknown packet from server".
connection->forceConnected(connection_parameters.timeouts);
}
else if (ast_to_process->formatForErrorMessage().size() > 500)
{

View File

@ -4,11 +4,14 @@
#include <unordered_map>
#include <vector>
#include <pcg-random/pcg_random.hpp>
#include <Common/randomSeed.h>
#include <Common/Stopwatch.h>
#include <Core/Field.h>
#include <Parsers/IAST.h>
namespace DB
{

View File

@ -774,7 +774,7 @@ UInt128 diffHash(const CommitDiff & file_changes)
}
UInt128 hash_of_diff;
hasher.get128(hash_of_diff.low, hash_of_diff.high);
hasher.get128(hash_of_diff.items[0], hash_of_diff.items[1]);
return hash_of_diff;
}

View File

@ -365,16 +365,20 @@ static void transformFixedString(const UInt8 * src, UInt8 * dst, size_t size, UI
}
}
static void transformUUID(const UInt128 & src, UInt128 & dst, UInt64 seed)
static void transformUUID(const UUID & src_uuid, UUID & dst_uuid, UInt64 seed)
{
const UInt128 & src = src_uuid.toUnderType();
UInt128 & dst = dst_uuid.toUnderType();
SipHash hash;
hash.update(seed);
hash.update(reinterpret_cast<const char *>(&src), sizeof(UInt128));
hash.update(reinterpret_cast<const char *>(&src), sizeof(UUID));
/// Saving version and variant from an old UUID
hash.get128(reinterpret_cast<char *>(&dst));
dst.high = (dst.high & 0x1fffffffffffffffull) | (src.high & 0xe000000000000000ull);
dst.low = (dst.low & 0xffffffffffff0fffull) | (src.low & 0x000000000000f000ull);
dst.items[1] = (dst.items[1] & 0x1fffffffffffffffull) | (src.items[1] & 0xe000000000000000ull);
dst.items[0] = (dst.items[0] & 0xffffffffffff0fffull) | (src.items[0] & 0x000000000000f000ull);
}
class FixedStringModel : public IModel
@ -426,10 +430,10 @@ public:
ColumnPtr generate(const IColumn & column) override
{
const ColumnUInt128 & src_column = assert_cast<const ColumnUInt128 &>(column);
const ColumnUUID & src_column = assert_cast<const ColumnUUID &>(column);
const auto & src_data = src_column.getData();
auto res_column = ColumnUInt128::create();
auto res_column = ColumnUUID::create();
auto & res_data = res_column->getData();
res_data.resize(src_data.size());

View File

@ -1,5 +1,6 @@
#include "ODBCBlockOutputStream.h"
#include <Common/hex.h>
#include <common/logger_useful.h>
#include <Core/Field.h>
#include <common/LocalDate.h>
@ -37,7 +38,6 @@ namespace
query.IAST::format(settings);
return buf.str();
}
}
ODBCBlockOutputStream::ODBCBlockOutputStream(nanodbc::ConnectionHolderPtr connection_,

View File

@ -1,7 +1,7 @@
#pragma once
#include <Access/IAccessEntity.h>
#include <common/types.h>
#include <Core/Types.h>
#include <Core/UUID.h>
#include <ext/scope_guard.h>
#include <functional>

View File

@ -1,6 +1,7 @@
#pragma once
#include <Core/UUID.h>
#include <Core/Types.h>
#include <boost/container/flat_set.hpp>
#include <memory>
#include <optional>

View File

@ -2,6 +2,7 @@
#include <Access/IAccessEntity.h>
#include <Access/RolesOrUsersSet.h>
#include <Core/Types.h>
#include <array>

View File

@ -7,7 +7,7 @@
#include <DataTypes/DataTypesDecimal.h>
#include <DataTypes/DataTypesNumber.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include "Core/DecimalFunctions.h"
#include <Core/DecimalFunctions.h>
namespace DB

View File

@ -5,18 +5,18 @@
namespace DB
{
template <class T>
template <typename T>
using AvgWeightedFieldType = std::conditional_t<IsDecimalNumber<T>,
std::conditional_t<std::is_same_v<T, Decimal256>, Decimal256, Decimal128>,
std::conditional_t<DecimalOrExtendedInt<T>,
Float64, // no way to do UInt128 * UInt128, better cast to Float64
NearestFieldType<T>>>;
template <class T, class U>
template <typename T, typename U>
using MaxFieldType = std::conditional_t<(sizeof(AvgWeightedFieldType<T>) > sizeof(AvgWeightedFieldType<U>)),
AvgWeightedFieldType<T>, AvgWeightedFieldType<U>>;
template <class Value, class Weight>
template <typename Value, typename Weight>
class AggregateFunctionAvgWeighted final :
public AggregateFunctionAvgBase<
MaxFieldType<Value, Weight>, AvgWeightedFieldType<Weight>, AggregateFunctionAvgWeighted<Value, Weight>>

View File

@ -2,7 +2,6 @@
#include <DataTypes/DataTypesNumber.h>
#include <Columns/ColumnsNumber.h>
#include <Common/FieldVisitors.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <AggregateFunctions/Helpers.h>

View File

@ -10,7 +10,6 @@
#include <Common/HashTable/HashMap.h>
#include <Common/SipHash.h>
#include <Common/FieldVisitors.h>
namespace DB
{

View File

@ -296,12 +296,7 @@ public:
if (size)
{
typename ColumnVector<T>::Container & data_to = assert_cast<ColumnVector<T> &>(arr_to.getData()).getData();
if constexpr (is_big_int_v<T>)
// is data_to empty? we should probably use std::vector::insert then
for (auto it = this->data(place).value.begin(); it != this->data(place).value.end(); it++)
data_to.push_back(*it);
else
data_to.insert(this->data(place).value.begin(), this->data(place).value.end());
data_to.insert(this->data(place).value.begin(), this->data(place).value.end());
}
}

View File

@ -21,6 +21,7 @@ namespace ErrorCodes
namespace
{
/// TODO Proper support for Decimal256.
template <typename T, typename LimitNumberOfElements>
struct MovingSum
{

View File

@ -38,7 +38,7 @@ struct MovingData
using Array = PODArray<T, 32, Allocator>;
Array value; /// Prefix sums.
T sum = 0;
T sum{};
void NO_SANITIZE_UNDEFINED add(T val, Arena * arena)
{
@ -69,9 +69,9 @@ struct MovingAvgData : public MovingData<T>
T NO_SANITIZE_UNDEFINED get(size_t idx, UInt64 window_size) const
{
if (idx < window_size)
return this->value[idx] / window_size;
return this->value[idx] / T(window_size);
else
return (this->value[idx] - this->value[idx - window_size]) / window_size;
return (this->value[idx] - this->value[idx - window_size]) / T(window_size);
}
};

View File

@ -6,7 +6,6 @@
#include <Columns/ColumnVector.h>
#include <Columns/ColumnTuple.h>
#include <Common/assert_cast.h>
#include <Common/FieldVisitors.h>
#include <Common/PODArray_fwd.h>
#include <common/types.h>
#include <DataTypes/DataTypesDecimal.h>
@ -21,7 +20,7 @@
#include <Common/ArenaAllocator.h>
#include <iostream>
namespace DB
{

View File

@ -100,13 +100,14 @@ AggregateFunctionPtr createAggregateFunctionQuantile(const std::string & name, c
if (which.idx == TypeIndex::Decimal32) return std::make_shared<Function<Decimal32, false>>(argument_types, params);
if (which.idx == TypeIndex::Decimal64) return std::make_shared<Function<Decimal64, false>>(argument_types, params);
if (which.idx == TypeIndex::Decimal128) return std::make_shared<Function<Decimal128, false>>(argument_types, params);
if (which.idx == TypeIndex::Decimal256) return std::make_shared<Function<Decimal256, false>>(argument_types, params);
if (which.idx == TypeIndex::DateTime64) return std::make_shared<Function<DateTime64, false>>(argument_types, params);
//if (which.idx == TypeIndex::Decimal256) return std::make_shared<Function<Decimal256, false>>(argument_types, params);
}
if constexpr (supportBigInt<Function>())
{
if (which.idx == TypeIndex::Int128) return std::make_shared<Function<Int128, true>>(argument_types, params);
if (which.idx == TypeIndex::UInt128) return std::make_shared<Function<Int128, true>>(argument_types, params);
if (which.idx == TypeIndex::Int256) return std::make_shared<Function<Int256, true>>(argument_types, params);
if (which.idx == TypeIndex::UInt256) return std::make_shared<Function<UInt256, true>>(argument_types, params);
}

View File

@ -36,7 +36,7 @@ AggregateFunctionPtr createAggregateFunctionStatisticsBinary(const std::string &
assertNoParameters(name, parameters);
assertBinary(name, argument_types);
AggregateFunctionPtr res(createWithTwoNumericTypes<FunctionTemplate>(*argument_types[0], *argument_types[1], argument_types));
AggregateFunctionPtr res(createWithTwoBasicNumericTypes<FunctionTemplate>(*argument_types[0], *argument_types[1], argument_types));
if (!res)
throw Exception("Illegal types " + argument_types[0]->getName() + " and " + argument_types[1]->getName()
+ " of arguments for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);

View File

@ -41,7 +41,7 @@ AggregateFunctionPtr createAggregateFunctionStatisticsBinary(const std::string &
assertNoParameters(name, parameters);
assertBinary(name, argument_types);
AggregateFunctionPtr res(createWithTwoNumericTypes<FunctionTemplate>(*argument_types[0], *argument_types[1], argument_types));
AggregateFunctionPtr res(createWithTwoBasicNumericTypes<FunctionTemplate>(*argument_types[0], *argument_types[1], argument_types));
if (!res)
throw Exception("Illegal types " + argument_types[0]->getName() + " and " + argument_types[1]->getName()
+ " of arguments for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);

View File

@ -131,11 +131,10 @@ public:
static_cast<ResultType>(static_cast<const ColVecT2 &>(*columns[1]).getData()[row_num]));
else
{
if constexpr (std::is_same_v<T1, Decimal256>)
if constexpr (IsDecimalNumber<T1>)
{
this->data(place).add(static_cast<ResultType>(
static_cast<const ColVecT1 &>(*columns[0]).getData()[row_num].value
));
static_cast<const ColVecT1 &>(*columns[0]).getData()[row_num].value));
}
else
this->data(place).add(

View File

@ -2,9 +2,10 @@
#include <AggregateFunctions/AggregateFunctionTopK.h>
#include <AggregateFunctions/Helpers.h>
#include <AggregateFunctions/FactoryHelpers.h>
#include <Common/FieldVisitors.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include "registerAggregateFunctions.h"
#define TOP_K_MAX_SIZE 0xFFFFFF

View File

@ -10,7 +10,6 @@
#include <Columns/ColumnArray.h>
#include <Common/SpaceSaving.h>
#include <Common/FieldVisitors.h>
#include <Common/assert_cast.h>
#include <AggregateFunctions/IAggregateFunction.h>

View File

@ -132,6 +132,12 @@ void registerAggregateFunctionsUniq(AggregateFunctionFactory & factory)
factory.registerFunction("uniqExact",
{createAggregateFunctionUniq<true, AggregateFunctionUniqExactData, AggregateFunctionUniqExactData<String>>, properties});
#if USE_DATASKETCHES
factory.registerFunction("uniqThetaSketch",
{createAggregateFunctionUniq<AggregateFunctionUniqThetaSketchData, AggregateFunctionUniqThetaSketchData>, properties});
#endif
}
}

View File

@ -22,6 +22,7 @@
#include <AggregateFunctions/UniquesHashSet.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include <AggregateFunctions/ThetaSketchData.h>
#include <AggregateFunctions/UniqVariadicHash.h>
@ -69,7 +70,7 @@ struct AggregateFunctionUniqHLL12Data<String>
};
template <>
struct AggregateFunctionUniqHLL12Data<UInt128>
struct AggregateFunctionUniqHLL12Data<UUID>
{
using Set = HyperLogLogWithSmallSetOptimization<UInt64, 16, 12>;
Set set;
@ -124,6 +125,19 @@ struct AggregateFunctionUniqExactData<String>
};
/// uniqThetaSketch
#if USE_DATASKETCHES
struct AggregateFunctionUniqThetaSketchData
{
using Set = ThetaSketchData<UInt64>;
Set set;
static String getName() { return "uniqThetaSketch"; }
};
#endif
namespace detail
{
@ -133,16 +147,14 @@ template <typename T> struct AggregateFunctionUniqTraits
{
static UInt64 hash(T x)
{
if constexpr (std::is_same_v<T, UInt128>)
{
return sipHash64(x);
}
else if constexpr (std::is_same_v<T, Float32> || std::is_same_v<T, Float64>)
if constexpr (std::is_same_v<T, Float32> || std::is_same_v<T, Float64>)
{
return ext::bit_cast<UInt64>(x);
}
else if constexpr (sizeof(T) <= sizeof(UInt64))
{
return x;
}
else
return DefaultHash64<T>(x);
}
@ -184,11 +196,17 @@ struct OneAdder
UInt128 key;
SipHash hash;
hash.update(value.data, value.size);
hash.get128(key.low, key.high);
hash.get128(key);
data.set.insert(key);
}
}
#if USE_DATASKETCHES
else if constexpr (std::is_same_v<Data, AggregateFunctionUniqThetaSketchData>)
{
data.set.insertOriginal(column.getDataAt(row_num));
}
#endif
}
};

View File

@ -3,11 +3,13 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/Helpers.h>
#include <Common/FieldVisitors.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include <functional>
#include "registerAggregateFunctions.h"
namespace DB
{

View File

@ -1,7 +1,6 @@
#pragma once
#include <Common/CombinedCardinalityEstimator.h>
#include <Common/FieldVisitors.h>
#include <Common/SipHash.h>
#include <Common/typeid_cast.h>
#include <Common/assert_cast.h>

View File

@ -1,11 +1,11 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/Helpers.h>
#include <AggregateFunctions/AggregateFunctionUniqUpTo.h>
#include <Common/FieldVisitors.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypeFixedString.h>
#include "registerAggregateFunctions.h"
namespace DB

View File

@ -1,6 +1,7 @@
#pragma once
#include <Common/FieldVisitors.h>
#include <common/unaligned.h>
#include <Common/typeid_cast.h>
#include <Common/assert_cast.h>
@ -35,7 +36,7 @@ namespace DB
*/
template <typename T>
struct __attribute__((__packed__)) AggregateFunctionUniqUpToData
struct AggregateFunctionUniqUpToData
{
/** If count == threshold + 1 - this means that it is "overflowed" (values greater than threshold).
* In this case (for example, after calling the merge function), the `data` array does not necessarily contain the initialized values
@ -43,7 +44,17 @@ struct __attribute__((__packed__)) AggregateFunctionUniqUpToData
* then set count to `threshold + 1`, and values from another state are not copied.
*/
UInt8 count = 0;
T data[0];
char data_ptr[0];
T load(size_t i) const
{
return unalignedLoad<T>(data_ptr + i * sizeof(T));
}
void store(size_t i, const T & x)
{
unalignedStore<T>(data_ptr + i * sizeof(T), x);
}
size_t size() const
{
@ -60,12 +71,12 @@ struct __attribute__((__packed__)) AggregateFunctionUniqUpToData
/// Linear search for the matching element.
for (size_t i = 0; i < count; ++i)
if (data[i] == x)
if (load(i) == x)
return;
/// Did not find the matching element. If there is room for one more element, insert it.
if (count < threshold)
data[count] = x;
store(count, x);
/// After increasing count, the state may be overflowed.
++count;
@ -84,7 +95,7 @@ struct __attribute__((__packed__)) AggregateFunctionUniqUpToData
}
for (size_t i = 0; i < rhs.count; ++i)
insert(rhs.data[i], threshold);
insert(rhs.load(i), threshold);
}
void write(WriteBuffer & wb, UInt8 threshold) const
@ -93,7 +104,7 @@ struct __attribute__((__packed__)) AggregateFunctionUniqUpToData
/// Write values only if the state is not overflowed. Otherwise, they are not needed, and only the fact that the state is overflowed is important.
if (count <= threshold)
wb.write(reinterpret_cast<const char *>(data), count * sizeof(data[0]));
wb.write(data_ptr, count * sizeof(T));
}
void read(ReadBuffer & rb, UInt8 threshold)
@ -101,7 +112,7 @@ struct __attribute__((__packed__)) AggregateFunctionUniqUpToData
readBinary(count, rb);
if (count <= threshold)
rb.read(reinterpret_cast<char *>(data), count * sizeof(data[0]));
rb.read(data_ptr, count * sizeof(T));
}
/// ALWAYS_INLINE is required to have better code layout for uniqUpTo function

View File

@ -15,12 +15,12 @@
M(Float32) \
M(Float64)
// No UInt128 here because of the name conflict
#define FOR_NUMERIC_TYPES(M) \
M(UInt8) \
M(UInt16) \
M(UInt32) \
M(UInt64) \
M(UInt128) \
M(UInt256) \
M(Int8) \
M(Int16) \
@ -109,6 +109,8 @@ static IAggregateFunction * createWithUnsignedIntegerType(const IDataType & argu
if (which.idx == TypeIndex::UInt16) return new AggregateFunctionTemplate<UInt16, Data<UInt16>>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::UInt32) return new AggregateFunctionTemplate<UInt32, Data<UInt32>>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::UInt64) return new AggregateFunctionTemplate<UInt64, Data<UInt64>>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::UInt128) return new AggregateFunctionTemplate<UInt128, Data<UInt128>>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::UInt256) return new AggregateFunctionTemplate<UInt256, Data<UInt256>>(std::forward<TArgs>(args)...);
return nullptr;
}
@ -119,11 +121,11 @@ static IAggregateFunction * createWithNumericBasedType(const IDataType & argumen
if (f)
return f;
/// expects that DataTypeDate based on UInt16, DataTypeDateTime based on UInt32 and UUID based on UInt128
/// expects that DataTypeDate based on UInt16, DataTypeDateTime based on UInt32
WhichDataType which(argument_type);
if (which.idx == TypeIndex::Date) return new AggregateFunctionTemplate<UInt16>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::DateTime) return new AggregateFunctionTemplate<UInt32>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::UUID) return new AggregateFunctionTemplate<UInt128>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::UUID) return new AggregateFunctionTemplate<UUID>(std::forward<TArgs>(args)...);
return nullptr;
}
@ -184,6 +186,29 @@ static IAggregateFunction * createWithTwoNumericTypes(const IDataType & first_ty
return nullptr;
}
template <typename FirstType, template <typename, typename> class AggregateFunctionTemplate, typename... TArgs>
static IAggregateFunction * createWithTwoBasicNumericTypesSecond(const IDataType & second_type, TArgs && ... args)
{
WhichDataType which(second_type);
#define DISPATCH(TYPE) \
if (which.idx == TypeIndex::TYPE) return new AggregateFunctionTemplate<FirstType, TYPE>(std::forward<TArgs>(args)...);
FOR_BASIC_NUMERIC_TYPES(DISPATCH)
#undef DISPATCH
return nullptr;
}
template <template <typename, typename> class AggregateFunctionTemplate, typename... TArgs>
static IAggregateFunction * createWithTwoBasicNumericTypes(const IDataType & first_type, const IDataType & second_type, TArgs && ... args)
{
WhichDataType which(first_type);
#define DISPATCH(TYPE) \
if (which.idx == TypeIndex::TYPE) \
return createWithTwoBasicNumericTypesSecond<TYPE, AggregateFunctionTemplate>(second_type, std::forward<TArgs>(args)...);
FOR_BASIC_NUMERIC_TYPES(DISPATCH)
#undef DISPATCH
return nullptr;
}
template <typename FirstType, template <typename, typename> class AggregateFunctionTemplate, typename... TArgs>
static IAggregateFunction * createWithTwoNumericOrDateTypesSecond(const IDataType & second_type, TArgs && ... args)
{
@ -195,7 +220,7 @@ static IAggregateFunction * createWithTwoNumericOrDateTypesSecond(const IDataTyp
if (which.idx == TypeIndex::Enum8) return new AggregateFunctionTemplate<FirstType, Int8>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::Enum16) return new AggregateFunctionTemplate<FirstType, Int16>(std::forward<TArgs>(args)...);
/// expects that DataTypeDate based on UInt16, DataTypeDateTime based on UInt32 and UUID based on UInt128
/// expects that DataTypeDate based on UInt16, DataTypeDateTime based on UInt32
if (which.idx == TypeIndex::Date) return new AggregateFunctionTemplate<FirstType, UInt16>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::DateTime) return new AggregateFunctionTemplate<FirstType, UInt32>(std::forward<TArgs>(args)...);
@ -216,7 +241,7 @@ static IAggregateFunction * createWithTwoNumericOrDateTypes(const IDataType & fi
if (which.idx == TypeIndex::Enum16)
return createWithTwoNumericOrDateTypesSecond<Int16, AggregateFunctionTemplate>(second_type, std::forward<TArgs>(args)...);
/// expects that DataTypeDate based on UInt16, DataTypeDateTime based on UInt32 and UUID based on UInt128
/// expects that DataTypeDate based on UInt16, DataTypeDateTime based on UInt32
if (which.idx == TypeIndex::Date)
return createWithTwoNumericOrDateTypesSecond<UInt16, AggregateFunctionTemplate>(second_type, std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::DateTime)

View File

@ -54,7 +54,7 @@ struct QuantileReservoirSampler
/// Get the value of the `level` quantile. The level must be between 0 and 1.
Value get(Float64 level)
{
return data.quantileInterpolated(level);
return Value(data.quantileInterpolated(level));
}
/// Get the `size` values of `levels` quantiles. Write `size` results starting with `result` address.
@ -62,7 +62,7 @@ struct QuantileReservoirSampler
void getMany(const Float64 * levels, const size_t * indices, size_t size, Value * result)
{
for (size_t i = 0; i < size; ++i)
result[indices[i]] = data.quantileInterpolated(levels[indices[i]]);
result[indices[i]] = Value(data.quantileInterpolated(levels[indices[i]]));
}
/// The same, but in the case of an empty state, NaN is returned.

View File

@ -131,12 +131,20 @@ public:
size_t left_index = static_cast<size_t>(index);
size_t right_index = left_index + 1;
if (right_index == samples.size())
return static_cast<double>(samples[left_index]);
{
if constexpr (DB::IsDecimalNumber<T>)
return static_cast<double>(samples[left_index].value);
else
return static_cast<double>(samples[left_index]);
}
double left_coef = right_index - index;
double right_coef = index - left_index;
return static_cast<double>(samples[left_index]) * left_coef + static_cast<double>(samples[right_index]) * right_coef;
if constexpr (DB::IsDecimalNumber<T>)
return static_cast<double>(samples[left_index].value) * left_coef + static_cast<double>(samples[right_index].value) * right_coef;
else
return static_cast<double>(samples[left_index]) * left_coef + static_cast<double>(samples[right_index]) * right_coef;
}
void merge(const ReservoirSampler<T, OnEmpty> & b)

View File

@ -0,0 +1,119 @@
#pragma once
#if !defined(ARCADIA_BUILD)
# include <Common/config.h>
#endif
#if USE_DATASKETCHES
#include <boost/noncopyable.hpp>
#include <memory>
#include <theta_sketch.hpp> // Y_IGNORE
#include <theta_union.hpp> // Y_IGNORE
namespace DB
{
template <typename Key>
class ThetaSketchData : private boost::noncopyable
{
private:
std::unique_ptr<datasketches::update_theta_sketch> sk_update;
std::unique_ptr<datasketches::theta_union> sk_union;
inline datasketches::update_theta_sketch * getSkUpdate()
{
if (!sk_update)
sk_update = std::make_unique<datasketches::update_theta_sketch>(datasketches::update_theta_sketch::builder().build());
return sk_update.get();
}
inline datasketches::theta_union * getSkUnion()
{
if (!sk_union)
sk_union = std::make_unique<datasketches::theta_union>(datasketches::theta_union::builder().build());
return sk_union.get();
}
public:
using value_type = Key;
ThetaSketchData() = default;
~ThetaSketchData() = default;
/// Insert original value without hash, as `datasketches::update_theta_sketch.update` will do the hash internal.
void insertOriginal(const StringRef & value)
{
getSkUpdate()->update(value.data, value.size);
}
/// Note that `datasketches::update_theta_sketch.update` will do the hash again.
void insert(Key value)
{
getSkUpdate()->update(value);
}
UInt64 size() const
{
if (sk_union)
return static_cast<UInt64>(sk_union->get_result().get_estimate());
else if (sk_update)
return static_cast<UInt64>(sk_update->get_estimate());
else
return 0;
}
void merge(const ThetaSketchData & rhs)
{
datasketches::theta_union * u = getSkUnion();
if (sk_update)
{
u->update(*sk_update);
sk_update.reset(nullptr);
}
if (rhs.sk_update)
u->update(*rhs.sk_update);
else if (rhs.sk_union)
u->update(rhs.sk_union->get_result());
}
/// You can only call for an empty object.
void read(DB::ReadBuffer & in)
{
datasketches::compact_theta_sketch::vector_bytes bytes;
readVectorBinary(bytes, in);
if (!bytes.empty())
{
auto sk = datasketches::compact_theta_sketch::deserialize(bytes.data(), bytes.size());
getSkUnion()->update(sk);
}
}
void write(DB::WriteBuffer & out) const
{
if (sk_update)
{
auto bytes = sk_update->compact().serialize();
writeVectorBinary(bytes, out);
}
else if (sk_union)
{
auto bytes = sk_union->get_result().serialize();
writeVectorBinary(bytes, out);
}
else
{
datasketches::compact_theta_sketch::vector_bytes bytes;
writeVectorBinary(bytes, out);
}
}
};
}
#endif

View File

@ -3,7 +3,6 @@
#include <city.h>
#include <Core/Defines.h>
#include <Common/SipHash.h>
#include <Common/UInt128.h>
#include <Common/assert_cast.h>
#include <Columns/ColumnTuple.h>
@ -107,7 +106,7 @@ struct UniqVariadicHash<true, false>
}
UInt128 key;
hash.get128(key.low, key.high);
hash.get128(key);
return key;
}
};
@ -131,7 +130,7 @@ struct UniqVariadicHash<true, true>
}
UInt128 key;
hash.get128(key.low, key.high);
hash.get128(key);
return key;
}
};

View File

@ -85,8 +85,8 @@ private:
{}
public:
const char * getFamilyName() const override { return TypeName<T>::get(); }
TypeIndex getDataType() const override { return TypeId<T>::value; }
const char * getFamilyName() const override { return TypeName<T>; }
TypeIndex getDataType() const override { return TypeId<T>; }
bool isNumeric() const override { return false; }
bool canBeInsideNullable() const override { return true; }

View File

@ -665,7 +665,7 @@ UInt128 ColumnUnique<ColumnType>::IncrementalHash::getHash(const ColumnType & co
column.updateHashWithValue(i, sip_hash);
std::lock_guard lock(mutex);
sip_hash.get128(hash.low, hash.high);
sip_hash.get128(hash);
cur_hash = hash;
num_added_rows.store(column_size);
}

View File

@ -33,6 +33,7 @@ namespace ErrorCodes
extern const int PARAMETER_OUT_OF_BOUND;
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
extern const int LOGICAL_ERROR;
extern const int NOT_IMPLEMENTED;
}
template <typename T>
@ -154,7 +155,7 @@ void ColumnVector<T>::getPermutation(bool reverse, size_t limit, int nan_directi
else
{
/// A case for radix sort
if constexpr (is_arithmetic_v<T> && !std::is_same_v<T, UInt128>)
if constexpr (is_arithmetic_v<T> && !is_big_int_v<T>)
{
/// Thresholds on size. Lower threshold is arbitrary. Upper threshold is chosen by the type for histogram counters.
if (s >= 256 && s <= std::numeric_limits<UInt32>::max())
@ -292,28 +293,37 @@ MutableColumnPtr ColumnVector<T>::cloneResized(size_t size) const
memcpy(new_col.data.data(), data.data(), count * sizeof(data[0]));
if (size > count)
memset(static_cast<void *>(&new_col.data[count]), static_cast<int>(ValueType()), (size - count) * sizeof(ValueType));
memset(static_cast<void *>(&new_col.data[count]), 0, (size - count) * sizeof(ValueType));
}
return res;
}
template <typename T>
UInt64 ColumnVector<T>::get64(size_t n) const
UInt64 ColumnVector<T>::get64(size_t n [[maybe_unused]]) const
{
return ext::bit_cast<UInt64>(data[n]);
if constexpr (is_arithmetic_v<T>)
return ext::bit_cast<UInt64>(data[n]);
else
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get the value of {} as UInt64", TypeName<T>);
}
template <typename T>
inline Float64 ColumnVector<T>::getFloat64(size_t n) const
inline Float64 ColumnVector<T>::getFloat64(size_t n [[maybe_unused]]) const
{
return static_cast<Float64>(data[n]);
if constexpr (is_arithmetic_v<T>)
return static_cast<Float64>(data[n]);
else
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get the value of {} as Float64", TypeName<T>);
}
template <typename T>
Float32 ColumnVector<T>::getFloat32(size_t n) const
Float32 ColumnVector<T>::getFloat32(size_t n [[maybe_unused]]) const
{
return static_cast<Float32>(data[n]);
if constexpr (is_arithmetic_v<T>)
return static_cast<Float32>(data[n]);
else
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get the value of {} as Float32", TypeName<T>);
}
template <typename T>
@ -482,8 +492,6 @@ void ColumnVector<T>::gather(ColumnGathererStream & gatherer)
template <typename T>
void ColumnVector<T>::getExtremes(Field & min, Field & max) const
{
using FastRefT = std::conditional_t<is_big_int_v<T>, const T &, const T>;
size_t size = data.size();
if (size == 0)
@ -504,7 +512,7 @@ void ColumnVector<T>::getExtremes(Field & min, Field & max) const
T cur_min = NaNOrZero<T>();
T cur_max = NaNOrZero<T>();
for (FastRefT x : data)
for (const T & x : data)
{
if (isNaN(x))
continue;
@ -569,5 +577,6 @@ template class ColumnVector<Int128>;
template class ColumnVector<Int256>;
template class ColumnVector<Float32>;
template class ColumnVector<Float64>;
template class ColumnVector<UUID>;
}

View File

@ -12,6 +12,12 @@
namespace DB
{
namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
}
/** Stuff for comparing numbers.
* Integer values are compared as usual.
* Floating-point numbers are compared this way that NaNs always end up at the end
@ -90,6 +96,7 @@ struct FloatCompareHelper
template <class U> struct CompareHelper<Float32, U> : public FloatCompareHelper<Float32> {};
template <class U> struct CompareHelper<Float64, U> : public FloatCompareHelper<Float64> {};
/** A template for columns that use a simple array to store.
*/
template <typename T>
@ -118,7 +125,7 @@ private:
ColumnVector(std::initializer_list<T> il) : data{il} {}
public:
bool isNumeric() const override { return IsNumber<T>; }
bool isNumeric() const override { return is_arithmetic_v<T>; }
size_t size() const override
{
@ -221,8 +228,8 @@ public:
data.reserve(n);
}
const char * getFamilyName() const override { return TypeName<T>::get(); }
TypeIndex getDataType() const override { return TypeId<T>::value; }
const char * getFamilyName() const override { return TypeName<T>; }
TypeIndex getDataType() const override { return TypeId<T>; }
MutableColumnPtr cloneResized(size_t size) const override;
@ -245,18 +252,27 @@ public:
/// Out of range conversion is permitted.
UInt64 NO_SANITIZE_UNDEFINED getUInt(size_t n) const override
{
return UInt64(data[n]);
if constexpr (is_arithmetic_v<T>)
return UInt64(data[n]);
else
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get the value of {} as UInt", TypeName<T>);
}
/// Out of range conversion is permitted.
Int64 NO_SANITIZE_UNDEFINED getInt(size_t n) const override
{
return Int64(data[n]);
if constexpr (is_arithmetic_v<T>)
return Int64(data[n]);
else
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get the value of {} as Int", TypeName<T>);
}
bool getBool(size_t n) const override
{
return bool(data[n]);
if constexpr (is_arithmetic_v<T>)
return bool(data[n]);
else
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get the value of {} as bool", TypeName<T>);
}
void insert(const Field & x) override
@ -370,5 +386,6 @@ extern template class ColumnVector<Int128>;
extern template class ColumnVector<Int256>;
extern template class ColumnVector<Float32>;
extern template class ColumnVector<Float64>;
extern template class ColumnVector<UUID>;
}

View File

@ -26,6 +26,6 @@ using ColumnInt256 = ColumnVector<Int256>;
using ColumnFloat32 = ColumnVector<Float32>;
using ColumnFloat64 = ColumnVector<Float64>;
using ColumnUUID = ColumnVector<UInt128>;
using ColumnUUID = ColumnVector<UUID>;
}

View File

@ -1,7 +1,6 @@
#pragma once
#include <optional>
#include <Columns/IColumn.h>
#include <Common/UInt128.h>
namespace DB
{

View File

@ -14,11 +14,11 @@
#include <DataTypes/DataTypesNumber.h>
#include <Common/WeakHash.h>
#include <Common/hex.h>
#include <unordered_map>
#include <iostream>
#include <sstream>
#include <Common/hex.h>
using namespace DB;
@ -27,20 +27,11 @@ template <typename T>
void checkColumn(
const WeakHash32::Container & hash,
const PaddedPODArray<T> & eq_class,
const std::function<std::string(size_t)> & print_function,
size_t allowed_collisions = 0,
size_t max_collisions_to_print = 10)
{
ASSERT_EQ(hash.size(), eq_class.size());
auto print_for_row = [&](size_t row)
{
std::string res = "row: " + std::to_string(row);
res += " hash: " + std::to_string(hash[row]);
res += " value: " + print_function(row);
return res;
};
/// Check equal rows has equal hash.
{
std::unordered_map<T, UInt32> map;
@ -55,11 +46,7 @@ void checkColumn(
else
{
if (it->second != hash[i])
{
std::cout << "Different hashes for the same equivalent class (" << size_t(val) << "):\n";
std::cout << print_for_row(it->first) << '\n';
std::cout << print_for_row(i) << std::endl;
}
std::cout << "Different hashes for the same equivalent class (" << toString(val) << ")\n";
ASSERT_EQ(it->second, hash[i]);
}
@ -88,7 +75,6 @@ void checkColumn(
if (num_collisions <= max_collisions_to_print)
{
collisions_str << "Collision:\n";
collisions_str << print_for_row(i) << std::endl;
}
if (num_collisions > allowed_collisions)
@ -117,7 +103,7 @@ TEST(WeakHash32, ColumnVectorU8)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnVectorI8)
@ -134,7 +120,7 @@ TEST(WeakHash32, ColumnVectorI8)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnVectorU16)
@ -151,7 +137,7 @@ TEST(WeakHash32, ColumnVectorU16)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnVectorI16)
@ -168,7 +154,7 @@ TEST(WeakHash32, ColumnVectorI16)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnVectorU32)
@ -185,7 +171,7 @@ TEST(WeakHash32, ColumnVectorU32)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnVectorI32)
@ -202,7 +188,7 @@ TEST(WeakHash32, ColumnVectorI32)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnVectorU64)
@ -219,7 +205,7 @@ TEST(WeakHash32, ColumnVectorU64)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnVectorI64)
@ -236,7 +222,7 @@ TEST(WeakHash32, ColumnVectorI64)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnVectorU128)
@ -250,7 +236,9 @@ TEST(WeakHash32, ColumnVectorU128)
{
for (uint64_t i = 0; i < 65536; ++i)
{
UInt128 val(i << 32u, i << 32u);
UInt128 val;
val.items[0] = i << 32u;
val.items[1] = i << 32u;
data.push_back(val);
eq_data.push_back(i);
}
@ -259,7 +247,7 @@ TEST(WeakHash32, ColumnVectorU128)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), eq_data, [&](size_t row) { return col->getElement(row).toHexString(); });
checkColumn(hash.getData(), eq_data);
}
TEST(WeakHash32, ColumnVectorI128)
@ -276,7 +264,7 @@ TEST(WeakHash32, ColumnVectorI128)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(Int64(col->getElement(row))); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnDecimal32)
@ -293,7 +281,7 @@ TEST(WeakHash32, ColumnDecimal32)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnDecimal64)
@ -310,7 +298,7 @@ TEST(WeakHash32, ColumnDecimal64)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnDecimal128)
@ -327,7 +315,7 @@ TEST(WeakHash32, ColumnDecimal128)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(Int64(col->getElement(row))); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnString1)
@ -349,7 +337,7 @@ TEST(WeakHash32, ColumnString1)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), data, [&](size_t row) { return col->getDataAt(row).toString(); });
checkColumn(hash.getData(), data);
}
TEST(WeakHash32, ColumnString2)
@ -389,7 +377,7 @@ TEST(WeakHash32, ColumnString2)
/// Now there is single collision between 'k' * 544 and 'q' * 2512 (which is calculated twice)
size_t allowed_collisions = 4;
checkColumn(hash.getData(), data, [&](size_t row) { return col->getDataAt(row).toString(); }, allowed_collisions);
checkColumn(hash.getData(), data, allowed_collisions);
}
TEST(WeakHash32, ColumnString3)
@ -427,7 +415,7 @@ TEST(WeakHash32, ColumnString3)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), data, [&](size_t row) { return col->getDataAt(row).toString(); });
checkColumn(hash.getData(), data);
}
TEST(WeakHash32, ColumnFixedString)
@ -455,7 +443,7 @@ TEST(WeakHash32, ColumnFixedString)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), data, [&](size_t row) { return col->getDataAt(row).toString(); });
checkColumn(hash.getData(), data);
}
TEST(WeakHash32, ColumnArray)
@ -502,15 +490,7 @@ TEST(WeakHash32, ColumnArray)
WeakHash32 hash(col_arr->size());
col_arr->updateWeakHash32(hash);
auto print_function = [&col_arr](size_t row)
{
auto & offsets = col_arr->getOffsets();
size_t s = offsets[row] - offsets[row - 1];
auto value = col_arr->getData().getUInt(offsets[row]);
return std::string("[array of size ") + std::to_string(s) + " with values " + std::to_string(value) + "]";
};
checkColumn(hash.getData(), eq_data, print_function);
checkColumn(hash.getData(), eq_data);
}
TEST(WeakHash32, ColumnArray2)
@ -545,15 +525,7 @@ TEST(WeakHash32, ColumnArray2)
WeakHash32 hash(col_arr->size());
col_arr->updateWeakHash32(hash);
auto print_function = [&col_arr](size_t row)
{
auto & offsets = col_arr->getOffsets();
auto value1 = col_arr->getData().getUInt(offsets[row]);
auto value2 = col_arr->getData().getUInt(offsets[row] + 1);
return std::string("[") + std::to_string(value1) + ", " + std::to_string(value2) + "]";
};
checkColumn(hash.getData(), eq_data, print_function);
checkColumn(hash.getData(), eq_data);
}
TEST(WeakHash32, ColumnArrayArray)
@ -610,20 +582,7 @@ TEST(WeakHash32, ColumnArrayArray)
WeakHash32 hash(col_arr_arr->size());
col_arr_arr->updateWeakHash32(hash);
auto print_function = [&col_arr_arr](size_t row2)
{
auto & offsets2 = col_arr_arr->getOffsets();
size_t s2 = offsets2[row2] - offsets2[row2 - 1];
const auto & arr2 = typeid_cast<const ColumnArray &>(col_arr_arr->getData());
const auto & offsets = arr2.getOffsets();
size_t row = offsets2[row2];
size_t s = offsets[row] - offsets[row - 1];
auto value = arr2.getData().getUInt(offsets[row]);
return std::string("[array of size ") + std::to_string(s2) + " with values ["
"[[array of size " + std::to_string(s) + " with values " + std::to_string(value) + "]]";
};
checkColumn(hash.getData(), eq_data, print_function);
checkColumn(hash.getData(), eq_data);
}
TEST(WeakHash32, ColumnConst)
@ -642,7 +601,7 @@ TEST(WeakHash32, ColumnConst)
WeakHash32 hash(col_const->size());
col_const->updateWeakHash32(hash);
checkColumn(hash.getData(), data, [&](size_t) { return std::to_string(0); });
checkColumn(hash.getData(), data);
}
TEST(WeakHash32, ColumnLowcardinality)
@ -663,7 +622,7 @@ TEST(WeakHash32, ColumnLowcardinality)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), data, [&](size_t row) { return std::to_string(col->getUInt(row)); });
checkColumn(hash.getData(), data);
}
TEST(WeakHash32, ColumnNullable)
@ -689,7 +648,7 @@ TEST(WeakHash32, ColumnNullable)
WeakHash32 hash(col_null->size());
col_null->updateWeakHash32(hash);
checkColumn(hash.getData(), eq, [&](size_t row) { return eq[row] == -1 ? "Null" : std::to_string(eq[row]); });
checkColumn(hash.getData(), eq);
}
TEST(WeakHash32, ColumnTupleUInt64UInt64)
@ -720,14 +679,7 @@ TEST(WeakHash32, ColumnTupleUInt64UInt64)
WeakHash32 hash(col_tuple->size());
col_tuple->updateWeakHash32(hash);
auto print_func = [&](size_t row)
{
std::string l = std::to_string(col_tuple->getColumn(0).getUInt(row));
std::string r = std::to_string(col_tuple->getColumn(1).getUInt(row));
return "(" + l + ", " + r + ")";
};
checkColumn(hash.getData(), eq, print_func);
checkColumn(hash.getData(), eq);
}
TEST(WeakHash32, ColumnTupleUInt64String)
@ -765,15 +717,8 @@ TEST(WeakHash32, ColumnTupleUInt64String)
WeakHash32 hash(col_tuple->size());
col_tuple->updateWeakHash32(hash);
auto print_func = [&](size_t row)
{
std::string l = std::to_string(col_tuple->getColumn(0).getUInt(row));
std::string r = col_tuple->getColumn(1).getDataAt(row).toString();
return "(" + l + ", " + r + ")";
};
size_t allowed_collisions = 8;
checkColumn(hash.getData(), eq, print_func, allowed_collisions);
checkColumn(hash.getData(), eq, allowed_collisions);
}
TEST(WeakHash32, ColumnTupleUInt64FixedString)
@ -811,14 +756,7 @@ TEST(WeakHash32, ColumnTupleUInt64FixedString)
WeakHash32 hash(col_tuple->size());
col_tuple->updateWeakHash32(hash);
auto print_func = [&](size_t row)
{
std::string l = std::to_string(col_tuple->getColumn(0).getUInt(row));
std::string r = col_tuple->getColumn(1).getDataAt(row).toString();
return "(" + l + ", " + r + ")";
};
checkColumn(hash.getData(), eq, print_func);
checkColumn(hash.getData(), eq);
}
TEST(WeakHash32, ColumnTupleUInt64Array)
@ -865,23 +803,10 @@ TEST(WeakHash32, ColumnTupleUInt64Array)
WeakHash32 hash(col_tuple->size());
col_tuple->updateWeakHash32(hash);
auto print_func = [&](size_t row)
{
std::string l = std::to_string(col_tuple->getColumn(0).getUInt(row));
const auto * col_arr = typeid_cast<const ColumnArray *>(col_tuple->getColumnPtr(1).get());
const auto & offsets = col_arr->getOffsets();
size_t s = offsets[row] - offsets[row - 1];
auto value = col_arr->getData().getUInt(offsets[row]);
auto r = std::string("[array of size ") + std::to_string(s) + " with values " + std::to_string(value) + "]";
return "(" + l + ", " + r + ")";
};
/// There are 2 collisions right now (repeated 2 times each):
/// (0, [array of size 1212 with values 7]) vs (0, [array of size 2265 with values 17])
/// (0, [array of size 558 with values 5]) vs (1, [array of size 879 with values 21])
size_t allowed_collisions = 8;
checkColumn(hash.getData(), eq_data, print_func, allowed_collisions);
checkColumn(hash.getData(), eq_data, allowed_collisions);
}

View File

@ -166,8 +166,7 @@ public:
size_t operator()(const DictionaryKey & key) const
{
SipHash hash;
hash.update(key.hash.low);
hash.update(key.hash.high);
hash.update(key.hash);
hash.update(key.size);
return hash.get64();
}

View File

@ -1,4 +1,3 @@
#include <Core/UUID.h>
#include <IO/ReadBuffer.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteHelpers.h>
@ -27,7 +26,7 @@ template <typename T>
static inline String formatQuotedWithPrefix(T x, const char * prefix)
{
WriteBufferFromOwnString wb;
wb.write(prefix, strlen(prefix));
writeCString(prefix, wb);
writeQuoted(x, wb);
return wb.str();
}
@ -48,10 +47,11 @@ String FieldVisitorDump::operator() (const DecimalField<Decimal32> & x) const {
String FieldVisitorDump::operator() (const DecimalField<Decimal64> & x) const { return formatQuotedWithPrefix(x, "Decimal64_"); }
String FieldVisitorDump::operator() (const DecimalField<Decimal128> & x) const { return formatQuotedWithPrefix(x, "Decimal128_"); }
String FieldVisitorDump::operator() (const DecimalField<Decimal256> & x) const { return formatQuotedWithPrefix(x, "Decimal256_"); }
String FieldVisitorDump::operator() (const UInt128 & x) const { return formatQuotedWithPrefix(x, "UInt128_"); }
String FieldVisitorDump::operator() (const UInt256 & x) const { return formatQuotedWithPrefix(x, "UInt256_"); }
String FieldVisitorDump::operator() (const Int256 & x) const { return formatQuotedWithPrefix(x, "Int256_"); }
String FieldVisitorDump::operator() (const Int128 & x) const { return formatQuotedWithPrefix(x, "Int128_"); }
String FieldVisitorDump::operator() (const UInt128 & x) const { return formatQuotedWithPrefix(UUID(x), "UUID_"); }
String FieldVisitorDump::operator() (const Int256 & x) const { return formatQuotedWithPrefix(x, "Int256_"); }
String FieldVisitorDump::operator() (const UUID & x) const { return formatQuotedWithPrefix(x, "UUID_"); }
String FieldVisitorDump::operator() (const String & x) const
@ -152,13 +152,11 @@ String FieldVisitorToString::operator() (const DecimalField<Decimal64> & x) cons
String FieldVisitorToString::operator() (const DecimalField<Decimal128> & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const DecimalField<Decimal256> & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const Int128 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const UInt128 & x) const { return formatQuoted(UUID(x)); }
String FieldVisitorToString::operator() (const AggregateFunctionStateData & x) const
{
return formatQuoted(x.data);
}
String FieldVisitorToString::operator() (const UInt128 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const UInt256 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const Int256 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const UUID & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const AggregateFunctionStateData & x) const { return formatQuoted(x.data); }
String FieldVisitorToString::operator() (const Array & x) const
{
@ -228,6 +226,7 @@ void FieldVisitorWriteBinary::operator() (const UInt128 & x, WriteBuffer & buf)
void FieldVisitorWriteBinary::operator() (const Int128 & x, WriteBuffer & buf) const { DB::writeVarInt(x, buf); }
void FieldVisitorWriteBinary::operator() (const UInt256 & x, WriteBuffer & buf) const { DB::writeBinary(x, buf); }
void FieldVisitorWriteBinary::operator() (const Int256 & x, WriteBuffer & buf) const { DB::writeBinary(x, buf); }
void FieldVisitorWriteBinary::operator() (const UUID & x, WriteBuffer & buf) const { DB::writeBinary(x, buf); }
void FieldVisitorWriteBinary::operator() (const DecimalField<Decimal32> & x, WriteBuffer & buf) const { DB::writeBinary(x.getValue(), buf); }
void FieldVisitorWriteBinary::operator() (const DecimalField<Decimal64> & x, WriteBuffer & buf) const { DB::writeBinary(x.getValue(), buf); }
void FieldVisitorWriteBinary::operator() (const DecimalField<Decimal128> & x, WriteBuffer & buf) const { DB::writeBinary(x.getValue(), buf); }
@ -315,6 +314,13 @@ void FieldVisitorHash::operator() (const Int128 & x) const
hash.update(x);
}
void FieldVisitorHash::operator() (const UUID & x) const
{
UInt8 type = Field::Types::UUID;
hash.update(type);
hash.update(x);
}
void FieldVisitorHash::operator() (const Float64 & x) const
{
UInt8 type = Field::Types::Float64;

View File

@ -19,14 +19,6 @@ namespace ErrorCodes
extern const int NOT_IMPLEMENTED;
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wredundant-decls"
// Just dont mess with it. If the redundant redeclaration is removed then ReaderHelpers.h should be included.
// This leads to Arena.h inclusion which has a problem with ASAN stuff included properly and messing macro definition
// which intefrers with... You dont want to know, really.
UInt128 stringToUUID(const String & str);
#pragma GCC diagnostic pop
/** StaticVisitor (and its descendants) - class with overloaded operator() for all types of fields.
* You could call visitor for field using function 'applyVisitor'.
@ -71,8 +63,11 @@ public:
String operator() (const Null & x) const;
String operator() (const UInt64 & x) const;
String operator() (const UInt128 & x) const;
String operator() (const UInt256 & x) const;
String operator() (const Int64 & x) const;
String operator() (const Int128 & x) const;
String operator() (const Int256 & x) const;
String operator() (const UUID & x) const;
String operator() (const Float64 & x) const;
String operator() (const String & x) const;
String operator() (const Array & x) const;
@ -83,9 +78,6 @@ public:
String operator() (const DecimalField<Decimal128> & x) const;
String operator() (const DecimalField<Decimal256> & x) const;
String operator() (const AggregateFunctionStateData & x) const;
String operator() (const UInt256 & x) const;
String operator() (const Int256 & x) const;
};
@ -95,8 +87,11 @@ public:
void operator() (const Null & x, WriteBuffer & buf) const;
void operator() (const UInt64 & x, WriteBuffer & buf) const;
void operator() (const UInt128 & x, WriteBuffer & buf) const;
void operator() (const UInt256 & x, WriteBuffer & buf) const;
void operator() (const Int64 & x, WriteBuffer & buf) const;
void operator() (const Int128 & x, WriteBuffer & buf) const;
void operator() (const Int256 & x, WriteBuffer & buf) const;
void operator() (const UUID & x, WriteBuffer & buf) const;
void operator() (const Float64 & x, WriteBuffer & buf) const;
void operator() (const String & x, WriteBuffer & buf) const;
void operator() (const Array & x, WriteBuffer & buf) const;
@ -107,9 +102,6 @@ public:
void operator() (const DecimalField<Decimal128> & x, WriteBuffer & buf) const;
void operator() (const DecimalField<Decimal256> & x, WriteBuffer & buf) const;
void operator() (const AggregateFunctionStateData & x, WriteBuffer & buf) const;
void operator() (const UInt256 & x, WriteBuffer & buf) const;
void operator() (const Int256 & x, WriteBuffer & buf) const;
};
@ -120,8 +112,11 @@ public:
String operator() (const Null & x) const;
String operator() (const UInt64 & x) const;
String operator() (const UInt128 & x) const;
String operator() (const UInt256 & x) const;
String operator() (const Int64 & x) const;
String operator() (const Int128 & x) const;
String operator() (const Int256 & x) const;
String operator() (const UUID & x) const;
String operator() (const Float64 & x) const;
String operator() (const String & x) const;
String operator() (const Array & x) const;
@ -132,9 +127,6 @@ public:
String operator() (const DecimalField<Decimal128> & x) const;
String operator() (const DecimalField<Decimal256> & x) const;
String operator() (const AggregateFunctionStateData & x) const;
String operator() (const UInt256 & x) const;
String operator() (const Int256 & x) const;
};
@ -171,6 +163,7 @@ public:
T operator() (const UInt64 & x) const { return T(x); }
T operator() (const Int64 & x) const { return T(x); }
T operator() (const Int128 & x) const { return T(x); }
T operator() (const UUID & x) const { return T(x.toUnderType()); }
T operator() (const Float64 & x) const
{
@ -259,8 +252,11 @@ public:
void operator() (const Null & x) const;
void operator() (const UInt64 & x) const;
void operator() (const UInt128 & x) const;
void operator() (const UInt256 & x) const;
void operator() (const Int64 & x) const;
void operator() (const Int128 & x) const;
void operator() (const Int256 & x) const;
void operator() (const UUID & x) const;
void operator() (const Float64 & x) const;
void operator() (const String & x) const;
void operator() (const Array & x) const;
@ -271,9 +267,6 @@ public:
void operator() (const DecimalField<Decimal128> & x) const;
void operator() (const DecimalField<Decimal256> & x) const;
void operator() (const AggregateFunctionStateData & x) const;
void operator() (const UInt256 & x) const;
void operator() (const Int256 & x) const;
};
@ -309,7 +302,7 @@ public:
bool operator() (Array &) const { throw Exception("Cannot sum Arrays", ErrorCodes::LOGICAL_ERROR); }
bool operator() (Tuple &) const { throw Exception("Cannot sum Tuples", ErrorCodes::LOGICAL_ERROR); }
bool operator() (Map &) const { throw Exception("Cannot sum Maps", ErrorCodes::LOGICAL_ERROR); }
bool operator() (UInt128 &) const { throw Exception("Cannot sum UUIDs", ErrorCodes::LOGICAL_ERROR); }
bool operator() (UUID &) const { throw Exception("Cannot sum UUIDs", ErrorCodes::LOGICAL_ERROR); }
bool operator() (AggregateFunctionStateData &) const { throw Exception("Cannot sum AggregateFunctionStates", ErrorCodes::LOGICAL_ERROR); }
bool operator() (Int128 & x) const

View File

@ -36,41 +36,30 @@ public:
if constexpr (std::is_arithmetic_v<T> && std::is_arithmetic_v<U>)
return accurate::equalsOp(l, r);
/// TODO This is wrong (does not respect scale).
if constexpr (isDecimalField<T>() && isDecimalField<U>())
return l == r;
if constexpr (isDecimalField<T>() && std::is_arithmetic_v<U>)
return l == DecimalField<Decimal128>(r, 0);
return l == DecimalField<Decimal256>(Decimal256(r), 0);
if constexpr (std::is_arithmetic_v<T> && isDecimalField<U>())
return DecimalField<Decimal128>(l, 0) == r;
return DecimalField<Decimal256>(Decimal256(l), 0) == r;
if constexpr (std::is_same_v<T, String>)
if constexpr (std::is_same_v<T, String> && std::is_arithmetic_v<U>)
{
if constexpr (std::is_same_v<U, UInt128>)
return stringToUUID(l) == r;
if constexpr (std::is_arithmetic_v<U>)
{
ReadBufferFromString in(l);
U parsed;
readText(parsed, in);
return operator()(parsed, r);
}
ReadBufferFromString in(l);
U parsed;
readText(parsed, in);
return operator()(parsed, r);
}
if constexpr (std::is_same_v<U, String>)
if constexpr (std::is_same_v<U, String> && std::is_arithmetic_v<T>)
{
if constexpr (std::is_same_v<T, UInt128>)
return l == stringToUUID(r);
if constexpr (std::is_arithmetic_v<T>)
{
ReadBufferFromString in(r);
T parsed;
readText(parsed, in);
return operator()(l, parsed);
}
ReadBufferFromString in(r);
T parsed;
readText(parsed, in);
return operator()(l, parsed);
}
}
@ -96,41 +85,30 @@ public:
if constexpr (std::is_arithmetic_v<T> && std::is_arithmetic_v<U>)
return accurate::lessOp(l, r);
/// TODO This is wrong (does not respect scale).
if constexpr (isDecimalField<T>() && isDecimalField<U>())
return l < r;
if constexpr (isDecimalField<T>() && std::is_arithmetic_v<U>)
return l < DecimalField<Decimal128>(r, 0);
return l < DecimalField<Decimal256>(Decimal256(r), 0);
if constexpr (std::is_arithmetic_v<T> && isDecimalField<U>())
return DecimalField<Decimal128>(l, 0) < r;
return DecimalField<Decimal256>(Decimal256(l), 0) < r;
if constexpr (std::is_same_v<T, String>)
if constexpr (std::is_same_v<T, String> && std::is_arithmetic_v<U>)
{
if constexpr (std::is_same_v<U, UInt128>)
return stringToUUID(l) < r;
if constexpr (std::is_arithmetic_v<U>)
{
ReadBufferFromString in(l);
U parsed;
readText(parsed, in);
return operator()(parsed, r);
}
ReadBufferFromString in(l);
U parsed;
readText(parsed, in);
return operator()(parsed, r);
}
if constexpr (std::is_same_v<U, String>)
if constexpr (std::is_same_v<U, String> && std::is_arithmetic_v<T>)
{
if constexpr (std::is_same_v<T, UInt128>)
return l < stringToUUID(r);
if constexpr (std::is_arithmetic_v<T>)
{
ReadBufferFromString in(r);
T parsed;
readText(parsed, in);
return operator()(l, parsed);
}
ReadBufferFromString in(r);
T parsed;
readText(parsed, in);
return operator()(l, parsed);
}
}

View File

@ -1,9 +1,10 @@
#pragma once
#include <city.h>
#include <Core/Types.h>
#include <common/types.h>
#include <common/unaligned.h>
#include <common/StringRef.h>
#include <Common/UInt128.h>
#include <type_traits>
@ -167,35 +168,36 @@ inline size_t DefaultHash64(std::enable_if_t<(sizeof(T) <= sizeof(UInt64)), T> k
return intHash64(u.out);
}
template <typename T>
static constexpr bool UseDefaultHashForBigInts =
std::is_same_v<T, DB::Int128> ||
std::is_same_v<T, DB::UInt128> ||
(is_big_int_v<T> && sizeof(T) == 32);
template <typename T>
inline size_t DefaultHash64(std::enable_if_t<(sizeof(T) > sizeof(UInt64) && UseDefaultHashForBigInts<T>), T> key)
inline size_t DefaultHash64(std::enable_if_t<(sizeof(T) > sizeof(UInt64)), T> key)
{
if constexpr (std::is_same_v<T, DB::Int128>)
if constexpr (is_big_int_v<T> && sizeof(T) == 16)
{
return intHash64(static_cast<UInt64>(key) ^ static_cast<UInt64>(key >> 64));
/// TODO This is classical antipattern.
return intHash64(
static_cast<UInt64>(key) ^
static_cast<UInt64>(key >> 64));
}
else if constexpr (std::is_same_v<T, DB::UInt128>)
else if constexpr (std::is_same_v<T, DB::UUID>)
{
return intHash64(key.low ^ key.high);
return intHash64(
static_cast<UInt64>(key.toUnderType()) ^
static_cast<UInt64>(key.toUnderType() >> 64));
}
else if constexpr (is_big_int_v<T> && sizeof(T) == 32)
{
return intHash64(static_cast<UInt64>(key) ^
return intHash64(
static_cast<UInt64>(key) ^
static_cast<UInt64>(key >> 64) ^
static_cast<UInt64>(key >> 128) ^
static_cast<UInt64>(key >> 256));
}
assert(false);
__builtin_unreachable();
}
template <typename T, typename Enable = void>
struct DefaultHash;
@ -213,7 +215,7 @@ struct DefaultHash<T, std::enable_if_t<DB::IsDecimalNumber<T>>>
{
size_t operator() (T key) const
{
return DefaultHash64<typename T::NativeType>(key);
return DefaultHash64<typename T::NativeType>(key.value);
}
};
@ -261,15 +263,98 @@ DEFINE_HASH(DB::Int128)
DEFINE_HASH(DB::Int256)
DEFINE_HASH(DB::Float32)
DEFINE_HASH(DB::Float64)
DEFINE_HASH(DB::UUID)
#undef DEFINE_HASH
template <>
struct DefaultHash<DB::UInt128> : public DB::UInt128Hash {};
struct UInt128Hash
{
size_t operator()(UInt128 x) const
{
return CityHash_v1_0_2::Hash128to64({x.items[0], x.items[1]});
}
};
struct UUIDHash
{
size_t operator()(DB::UUID x) const
{
return UInt128Hash()(x.toUnderType());
}
};
#ifdef __SSE4_2__
struct UInt128HashCRC32
{
size_t operator()(UInt128 x) const
{
UInt64 crc = -1ULL;
crc = _mm_crc32_u64(crc, x.items[0]);
crc = _mm_crc32_u64(crc, x.items[1]);
return crc;
}
};
#else
/// On other platforms we do not use CRC32. NOTE This can be confusing.
struct UInt128HashCRC32 : public UInt128Hash {};
#endif
struct UInt128TrivialHash
{
size_t operator()(UInt128 x) const { return x.items[0]; }
};
struct UUIDTrivialHash
{
size_t operator()(DB::UUID x) const { return x.toUnderType().items[0]; }
};
struct UInt256Hash
{
size_t operator()(UInt256 x) const
{
/// NOTE suboptimal
return CityHash_v1_0_2::Hash128to64({
CityHash_v1_0_2::Hash128to64({x.items[0], x.items[1]}),
CityHash_v1_0_2::Hash128to64({x.items[2], x.items[3]})});
}
};
#ifdef __SSE4_2__
struct UInt256HashCRC32
{
size_t operator()(UInt256 x) const
{
UInt64 crc = -1ULL;
crc = _mm_crc32_u64(crc, x.items[0]);
crc = _mm_crc32_u64(crc, x.items[1]);
crc = _mm_crc32_u64(crc, x.items[2]);
crc = _mm_crc32_u64(crc, x.items[3]);
return crc;
}
};
#else
/// We do not need to use CRC32 on other platforms. NOTE This can be confusing.
struct UInt256HashCRC32 : public UInt256Hash {};
#endif
template <>
struct DefaultHash<DB::DummyUInt256> : public DB::UInt256Hash {};
struct DefaultHash<DB::UInt128> : public UInt128Hash {};
template <>
struct DefaultHash<DB::UInt256> : public UInt256Hash {};
template <>
struct DefaultHash<DB::UUID> : public UUIDHash {};
/// It is reasonable to use for UInt8, UInt16 with sufficient hash table size.
@ -322,23 +407,18 @@ struct IntHash32
{
size_t operator() (const T & key) const
{
if constexpr (std::is_same_v<T, DB::Int128>)
if constexpr (is_big_int_v<T> && sizeof(T) == 16)
{
return intHash32<salt>(static_cast<UInt64>(key) ^ static_cast<UInt64>(key >> 64));
}
else if constexpr (std::is_same_v<T, DB::UInt128>)
{
return intHash32<salt>(key.low ^ key.high);
return intHash32<salt>(key.items[0] ^ key.items[1]);
}
else if constexpr (is_big_int_v<T> && sizeof(T) == 32)
{
return intHash32<salt>(static_cast<UInt64>(key) ^
static_cast<UInt64>(key >> 64) ^
static_cast<UInt64>(key >> 128) ^
static_cast<UInt64>(key >> 256));
return intHash32<salt>(key.items[0] ^ key.items[1] ^ key.items[2] ^ key.items[3]);
}
else if constexpr (sizeof(T) <= sizeof(UInt64))
{
return intHash32<salt>(key);
}
assert(false);
__builtin_unreachable();

View File

@ -70,10 +70,10 @@ namespace ZeroTraits
{
template <typename T>
bool check(const T x) { return x == 0; }
bool check(const T x) { return x == T{}; }
template <typename T>
void set(T & x) { x = 0; }
void set(T & x) { x = {}; }
}

View File

@ -28,9 +28,8 @@ struct StringHashMapCell<StringKey16, TMapped> : public HashMapCell<StringKey16,
// Zero means unoccupied cells in hash table. Use key with last word = 0 as
// zero keys, because such keys are unrepresentable (no way to encode length).
static bool isZero(const StringKey16 & key, const HashTableNoState &)
{ return key.high == 0; }
void setZero() { this->value.first.high = 0; }
static bool isZero(const StringKey16 & key, const HashTableNoState &) { return key.items[1] == 0; }
void setZero() { this->value.first.items[1] = 0; }
// external
const StringRef getKey() const { return toStringRef(this->value.first); }

View File

@ -33,8 +33,8 @@ struct StringHashSetCell<StringKey16> : public HashTableCell<StringKey16, String
// Zero means unoccupied cells in hash table. Use key with last word = 0 as
// zero keys, because such keys are unrepresentable (no way to encode length).
static bool isZero(const StringKey16 & key_, const HashTableNoState &)
{ return key_.high == 0; }
void setZero() { this->key.high = 0; }
{ return key_.items[1] == 0; }
void setZero() { this->key.items[1] = 0; }
};
template <>

View File

@ -25,8 +25,8 @@ inline StringRef ALWAYS_INLINE toStringRef(const StringKey8 & n)
}
inline StringRef ALWAYS_INLINE toStringRef(const StringKey16 & n)
{
assert(n.high != 0);
return {reinterpret_cast<const char *>(&n), 16ul - (__builtin_clzll(n.high) >> 3)};
assert(n.items[1] != 0);
return {reinterpret_cast<const char *>(&n), 16ul - (__builtin_clzll(n.items[1]) >> 3)};
}
inline StringRef ALWAYS_INLINE toStringRef(const StringKey24 & n)
{
@ -46,8 +46,8 @@ struct StringHashTableHash
size_t ALWAYS_INLINE operator()(StringKey16 key) const
{
size_t res = -1ULL;
res = _mm_crc32_u64(res, key.low);
res = _mm_crc32_u64(res, key.high);
res = _mm_crc32_u64(res, key.items[0]);
res = _mm_crc32_u64(res, key.items[1]);
return res;
}
size_t ALWAYS_INLINE operator()(StringKey24 key) const

View File

@ -6,12 +6,14 @@
#include <Common/formatReadable.h>
#include <common/logger_useful.h>
#include <Common/ProfileEvents.h>
#include <Common/thread_local_rng.h>
#include <atomic>
#include <cmath>
#include <random>
#include <cstdlib>
#ifdef MEMORY_TRACKER_DEBUG_CHECKS
thread_local bool _memory_tracker_always_throw_logical_error_on_allocation = false;
#endif

View File

@ -6,12 +6,12 @@ namespace DB
// The runtime info we need to create new OpenTelemetry spans.
struct OpenTelemetryTraceContext
{
__uint128_t trace_id = 0;
UUID trace_id{};
UInt64 span_id = 0;
// The incoming tracestate header and the trace flags, we just pass them
// downstream. See https://www.w3.org/TR/trace-context/
String tracestate;
uint8_t trace_flags = 0;
UInt8 trace_flags = 0;
// Parse/compose OpenTelemetry traceparent header.
bool parseTraceparentHeader(const std::string & traceparent, std::string & error);

View File

@ -260,22 +260,12 @@ public:
template <typename ... TAllocatorParams>
void push_back_raw(const void * ptr, TAllocatorParams &&... allocator_params)
{
push_back_raw_many(1, ptr, std::forward<TAllocatorParams>(allocator_params)...);
}
template <typename ... TAllocatorParams>
void push_back_raw_many(size_t number_of_items, const void * ptr, TAllocatorParams &&... allocator_params)
{
size_t required_capacity = size() + number_of_items;
size_t required_capacity = size() + ELEMENT_SIZE;
if (unlikely(required_capacity > capacity()))
reserve(required_capacity, std::forward<TAllocatorParams>(allocator_params)...);
size_t items_byte_size = byte_size(number_of_items);
if (items_byte_size)
{
memcpy(c_end, ptr, items_byte_size);
c_end += items_byte_size;
}
memcpy(c_end, ptr, ELEMENT_SIZE);
c_end += ELEMENT_SIZE;
}
void protect()

View File

@ -155,7 +155,6 @@ public:
unalignedStore<UInt64>(out + 8, v2 ^ v3);
}
/// template for avoiding 'unsigned long long' vs 'unsigned long' problem on old poco in macos
template <typename T>
ALWAYS_INLINE void get128(T & lo, T & hi)
{
@ -165,6 +164,13 @@ public:
hi = v2 ^ v3;
}
template <typename T>
ALWAYS_INLINE void get128(T & dst)
{
static_assert(sizeof(T) == 16);
get128(reinterpret_cast<char *>(&dst));
}
UInt64 get64()
{
finalize();

View File

@ -6,7 +6,6 @@
#include <boost/range/adaptor/reversed.hpp>
#include <Common/ArenaWithFreeLists.h>
#include <Common/UInt128.h>
#include <Common/HashTable/Hash.h>
#include <Common/HashTable/HashMap.h>

View File

@ -24,7 +24,10 @@ namespace
/// Thus upper bound on query_id length should be introduced to avoid buffer overflow in signal handler.
///
/// And it cannot be large, since otherwise it will not fit into PIPE_BUF.
constexpr size_t QUERY_ID_MAX_LEN = sizeof("00000000-0000-0000-0000-000000000000") - 1; // 36
/// The performance test query ids can be surprisingly long like
/// `aggregating_merge_tree_simple_aggregate_function_string.query100.profile100`,
/// so make some allowance for them as well.
constexpr size_t QUERY_ID_MAX_LEN = 128;
}
LazyPipeFDs pipe;

View File

@ -1,292 +0,0 @@
#pragma once
#include <tuple>
#include <iomanip>
#include <city.h>
#include <Core/Types.h>
#include <Common/hex.h>
#ifdef __SSE4_2__
#include <nmmintrin.h>
#endif
namespace DB
{
/// For aggregation by SipHash, UUID type or concatenation of several fields.
struct UInt128
{
/// Suppress gcc7 warnings: 'prev_key.DB::UInt128::low' may be used uninitialized in this function
#if !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
/// This naming assumes little endian.
UInt64 low;
UInt64 high;
/// TODO: Make this constexpr. Currently it is used in unions
/// and union cannot contain member with non trivial constructor
/// constructor must be non user provided but compiler cannot constexpr constructor
/// if members low and high are not initialized, if we default member initialize them
/// constructor becomes non trivial.
UInt128() = default;
explicit constexpr UInt128(const UInt64 low_, const UInt64 high_) : low(low_), high(high_) { }
/// We need Int128 to UInt128 conversion or AccurateComparison will call greaterOp<Int128, UInt64> instead of greaterOp<Int128, UInt128>
explicit constexpr UInt128(const Int128 rhs) : low(rhs), high(rhs >> 64) {}
explicit constexpr UInt128(const Int64 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const Int32 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const Int16 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const Int8 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const UInt8 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const UInt16 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const UInt32 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const UInt64 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const Float32 rhs) : low(rhs), high() {}
explicit constexpr UInt128(const Float64 rhs) : low(rhs), high() {}
constexpr auto tuple() const { return std::tie(high, low); }
String toHexString() const
{
String res(2 * sizeof(UInt128), 0);
writeHexUIntLowercase(*this, res.data());
return res;
}
constexpr bool operator== (const UInt128 rhs) const { return tuple() == rhs.tuple(); }
constexpr bool operator!= (const UInt128 rhs) const { return tuple() != rhs.tuple(); }
constexpr bool operator< (const UInt128 rhs) const { return tuple() < rhs.tuple(); }
constexpr bool operator<= (const UInt128 rhs) const { return tuple() <= rhs.tuple(); }
constexpr bool operator> (const UInt128 rhs) const { return tuple() > rhs.tuple(); }
constexpr bool operator>= (const UInt128 rhs) const { return tuple() >= rhs.tuple(); }
constexpr bool operator == (const Int128 rhs) const { return *this == UInt128(rhs, rhs >> 64) && rhs >= 0; }
constexpr bool operator != (const Int128 rhs) const { return *this != UInt128(rhs, rhs >> 64) || rhs < 0; }
constexpr bool operator >= (const Int128 rhs) const { return *this >= UInt128(rhs, rhs >> 64) || rhs < 0; }
constexpr bool operator > (const Int128 rhs) const { return *this > UInt128(rhs, rhs >> 64) || rhs < 0; }
constexpr bool operator <= (const Int128 rhs) const { return *this <= UInt128(rhs, rhs >> 64) && rhs >= 0; }
constexpr bool operator < (const Int128 rhs) const { return *this < UInt128(rhs, rhs >> 64) && rhs >= 0; }
constexpr bool operator > (const Int256 rhs) const { return (rhs < 0) || ((Int256(high) << 64) + low) > rhs; }
constexpr bool operator > (const UInt256 rhs) const { return ((UInt256(high) << 64) + low) > rhs; }
constexpr bool operator < (const Int256 rhs) const { return (rhs >= 0) && ((Int256(high) << 64) + low) < rhs; }
constexpr bool operator < (const UInt256 rhs) const { return ((UInt256(high) << 64) + low) < rhs; }
template <typename T> constexpr bool operator== (const T rhs) const { return *this == UInt128(rhs); }
template <typename T> constexpr bool operator!= (const T rhs) const { return *this != UInt128(rhs); }
template <typename T> constexpr bool operator>= (const T rhs) const { return *this >= UInt128(rhs); }
template <typename T> constexpr bool operator> (const T rhs) const { return *this > UInt128(rhs); }
template <typename T> constexpr bool operator<= (const T rhs) const { return *this <= UInt128(rhs); }
template <typename T> constexpr bool operator< (const T rhs) const { return *this < UInt128(rhs); }
template <typename T> explicit operator T() const
{
if constexpr (std::is_class_v<T>)
return T();
else
return static_cast<T>(low);
}
#if !defined(__clang__)
#pragma GCC diagnostic pop
#endif
constexpr UInt128 & operator= (const UInt64 rhs) { low = rhs; high = 0; return *this; }
};
template <typename T> constexpr bool operator == (T a, const UInt128 b) { return b.operator==(a); }
template <typename T> constexpr bool operator != (T a, const UInt128 b) { return b.operator!=(a); }
template <typename T> constexpr bool operator >= (T a, const UInt128 b) { return b <= a; }
template <typename T> constexpr bool operator > (T a, const UInt128 b) { return b < a; }
template <typename T> constexpr bool operator <= (T a, const UInt128 b) { return b >= a; }
template <typename T> constexpr bool operator < (T a, const UInt128 b) { return b > a; }
template <> inline constexpr bool IsNumber<UInt128> = true;
template <> struct TypeName<UInt128> { static constexpr const char * get() { return "UInt128"; } };
template <> struct TypeId<UInt128> { static constexpr const TypeIndex value = TypeIndex::UInt128; };
struct UInt128Hash
{
size_t operator()(UInt128 x) const
{
return CityHash_v1_0_2::Hash128to64({x.low, x.high});
}
};
#ifdef __SSE4_2__
struct UInt128HashCRC32
{
size_t operator()(UInt128 x) const
{
UInt64 crc = -1ULL;
crc = _mm_crc32_u64(crc, x.low);
crc = _mm_crc32_u64(crc, x.high);
return crc;
}
};
#else
/// On other platforms we do not use CRC32. NOTE This can be confusing.
struct UInt128HashCRC32 : public UInt128Hash {};
#endif
struct UInt128TrivialHash
{
size_t operator()(UInt128 x) const { return x.low; }
};
/** Used for aggregation, for putting a large number of constant-length keys in a hash table.
*/
struct DummyUInt256
{
/// Suppress gcc7 warnings: 'prev_key.DB::UInt256::a' may be used uninitialized in this function
#if !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
UInt64 a;
UInt64 b;
UInt64 c;
UInt64 d;
bool operator== (const DummyUInt256 rhs) const
{
return a == rhs.a && b == rhs.b && c == rhs.c && d == rhs.d;
/* So it's no better.
return 0xFFFF == _mm_movemask_epi8(_mm_and_si128(
_mm_cmpeq_epi8(
_mm_loadu_si128(reinterpret_cast<const __m128i *>(&a)),
_mm_loadu_si128(reinterpret_cast<const __m128i *>(&rhs.a))),
_mm_cmpeq_epi8(
_mm_loadu_si128(reinterpret_cast<const __m128i *>(&c)),
_mm_loadu_si128(reinterpret_cast<const __m128i *>(&rhs.c)))));*/
}
bool operator!= (const DummyUInt256 rhs) const { return !operator==(rhs); }
bool operator== (const UInt64 rhs) const { return a == rhs && b == 0 && c == 0 && d == 0; }
bool operator!= (const UInt64 rhs) const { return !operator==(rhs); }
#if !defined(__clang__)
#pragma GCC diagnostic pop
#endif
DummyUInt256 & operator = (const UInt64 rhs) { a = rhs; b = 0; c = 0; d = 0; return *this; }
};
struct UInt256Hash
{
size_t operator()(DummyUInt256 x) const
{
/// NOTE suboptimal
return CityHash_v1_0_2::Hash128to64({CityHash_v1_0_2::Hash128to64({x.a, x.b}), CityHash_v1_0_2::Hash128to64({x.c, x.d})});
}
};
#ifdef __SSE4_2__
struct UInt256HashCRC32
{
size_t operator()(DummyUInt256 x) const
{
UInt64 crc = -1ULL;
crc = _mm_crc32_u64(crc, x.a);
crc = _mm_crc32_u64(crc, x.b);
crc = _mm_crc32_u64(crc, x.c);
crc = _mm_crc32_u64(crc, x.d);
return crc;
}
};
#else
/// We do not need to use CRC32 on other platforms. NOTE This can be confusing.
struct UInt256HashCRC32 : public UInt256Hash {};
#endif
}
template <> struct is_signed<DB::UInt128>
{
static constexpr bool value = false;
};
template <> struct is_unsigned<DB::UInt128>
{
static constexpr bool value = true;
};
template <> struct is_integer<DB::UInt128>
{
static constexpr bool value = true;
};
// Operator +, -, /, *, % aren't implemented so it's not an arithmetic type
template <> struct is_arithmetic<DB::UInt128>
{
static constexpr bool value = false;
};
/// Overload hash for type casting
namespace std
{
template <> struct hash<DB::UInt128>
{
size_t operator()(const DB::UInt128 & u) const
{
return CityHash_v1_0_2::Hash128to64({u.low, u.high});
}
};
template<>
class numeric_limits<DB::UInt128>
{
public:
static constexpr bool is_specialized = true;
static constexpr bool is_signed = ::is_signed<DB::UInt128>::value;
static constexpr bool is_integer = ::is_integer<DB::UInt128>::value;
static constexpr bool is_exact = true;
static constexpr bool has_infinity = false;
static constexpr bool has_quiet_NaN = false;
static constexpr bool has_signaling_NaN = false;
static constexpr std::float_denorm_style has_denorm = std::denorm_absent;
static constexpr bool has_denorm_loss = false;
static constexpr std::float_round_style round_style = std::round_toward_zero;
static constexpr bool is_iec559 = false;
static constexpr bool is_bounded = true;
static constexpr bool is_modulo = true;
static constexpr int digits = std::numeric_limits<UInt64>::digits * 2;
static constexpr int digits10 = digits * 0.30103 /*std::log10(2)*/;
static constexpr int max_digits10 = 0;
static constexpr int radix = 2;
static constexpr int min_exponent = 0;
static constexpr int min_exponent10 = 0;
static constexpr int max_exponent = 0;
static constexpr int max_exponent10 = 0;
static constexpr bool traps = true;
static constexpr bool tinyness_before = false;
static constexpr DB::UInt128 min() noexcept { return DB::UInt128(0, 0); }
static constexpr DB::UInt128 max() noexcept
{
return DB::UInt128(std::numeric_limits<UInt64>::max(), std::numeric_limits<UInt64>::max());
}
static constexpr DB::UInt128 lowest() noexcept { return min(); }
};
}

View File

@ -15,3 +15,4 @@
#cmakedefine01 USE_GRPC
#cmakedefine01 USE_STATS
#cmakedefine01 CLICKHOUSE_SPLIT_BINARY
#cmakedefine01 USE_DATASKETCHES

View File

@ -135,24 +135,24 @@ struct Dictionary
template <typename Value> using ContainerType = Value[];
template <typename Value> using ContainerPtrType = std::unique_ptr<ContainerType<Value>>;
enum class AttributeUnderlyingType
enum class AttributeUnderlyingTypeTest
{
utUInt8,
utUInt16,
utUInt32,
utUInt64,
utInt8,
utInt16,
utInt32,
utInt64,
utFloat32,
utFloat64,
utString
UInt8,
UInt16,
UInt32,
UInt64,
Int8,
Int16,
Int32,
Int64,
Float32,
Float64,
String
};
struct Attribute final
{
AttributeUnderlyingType type;
AttributeUnderlyingTypeTest type;
std::variant<
UInt8, UInt16, UInt32, UInt64,
Int8, Int16, Int32, Int64,
@ -172,17 +172,17 @@ struct Dictionary
{
switch (attribute.type)
{
case AttributeUnderlyingType::utUInt8: std::get<ContainerPtrType<UInt8>>(attribute.arrays)[idx] = value.get<UInt64>(); break;
case AttributeUnderlyingType::utUInt16: std::get<ContainerPtrType<UInt16>>(attribute.arrays)[idx] = value.get<UInt64>(); break;
case AttributeUnderlyingType::utUInt32: std::get<ContainerPtrType<UInt32>>(attribute.arrays)[idx] = value.get<UInt64>(); break;
case AttributeUnderlyingType::utUInt64: std::get<ContainerPtrType<UInt64>>(attribute.arrays)[idx] = value.get<UInt64>(); break;
case AttributeUnderlyingType::utInt8: std::get<ContainerPtrType<Int8>>(attribute.arrays)[idx] = value.get<Int64>(); break;
case AttributeUnderlyingType::utInt16: std::get<ContainerPtrType<Int16>>(attribute.arrays)[idx] = value.get<Int64>(); break;
case AttributeUnderlyingType::utInt32: std::get<ContainerPtrType<Int32>>(attribute.arrays)[idx] = value.get<Int64>(); break;
case AttributeUnderlyingType::utInt64: std::get<ContainerPtrType<Int64>>(attribute.arrays)[idx] = value.get<Int64>(); break;
case AttributeUnderlyingType::utFloat32: std::get<ContainerPtrType<Float32>>(attribute.arrays)[idx] = value.get<Float64>(); break;
case AttributeUnderlyingType::utFloat64: std::get<ContainerPtrType<Float64>>(attribute.arrays)[idx] = value.get<Float64>(); break;
case AttributeUnderlyingType::utString:
case AttributeUnderlyingTypeTest::UInt8: std::get<ContainerPtrType<UInt8>>(attribute.arrays)[idx] = value.get<UInt64>(); break;
case AttributeUnderlyingTypeTest::UInt16: std::get<ContainerPtrType<UInt16>>(attribute.arrays)[idx] = value.get<UInt64>(); break;
case AttributeUnderlyingTypeTest::UInt32: std::get<ContainerPtrType<UInt32>>(attribute.arrays)[idx] = value.get<UInt64>(); break;
case AttributeUnderlyingTypeTest::UInt64: std::get<ContainerPtrType<UInt64>>(attribute.arrays)[idx] = value.get<UInt64>(); break;
case AttributeUnderlyingTypeTest::Int8: std::get<ContainerPtrType<Int8>>(attribute.arrays)[idx] = value.get<Int64>(); break;
case AttributeUnderlyingTypeTest::Int16: std::get<ContainerPtrType<Int16>>(attribute.arrays)[idx] = value.get<Int64>(); break;
case AttributeUnderlyingTypeTest::Int32: std::get<ContainerPtrType<Int32>>(attribute.arrays)[idx] = value.get<Int64>(); break;
case AttributeUnderlyingTypeTest::Int64: std::get<ContainerPtrType<Int64>>(attribute.arrays)[idx] = value.get<Int64>(); break;
case AttributeUnderlyingTypeTest::Float32: std::get<ContainerPtrType<Float32>>(attribute.arrays)[idx] = value.get<Float64>(); break;
case AttributeUnderlyingTypeTest::Float64: std::get<ContainerPtrType<Float64>>(attribute.arrays)[idx] = value.get<Float64>(); break;
case AttributeUnderlyingTypeTest::String:
{
const auto & string = value.get<String>();
auto & string_ref = std::get<ContainerPtrType<StringRef>>(attribute.arrays)[idx];
@ -308,7 +308,7 @@ int main(int argc, char ** argv)
constexpr size_t cache_size = 1024;
Dictionary::Attribute attr;
attr.type = Dictionary::AttributeUnderlyingType::utString;
attr.type = Dictionary::AttributeUnderlyingTypeTest::String;
std::get<Dictionary::ContainerPtrType<StringRef>>(attr.arrays).reset(new StringRef[cache_size]{}); // NOLINT
while (true)

View File

@ -38,7 +38,7 @@ std::string getHashOfLoadedBinaryHex()
{
SipHash hash = getHashOfLoadedBinary();
std::array<UInt64, 2> checksum;
hash.get128(checksum[0], checksum[1]);
hash.get128(checksum);
return getHexUIntUppercase(checksum);
}

View File

@ -138,9 +138,9 @@ constexpr inline Int128 exp10_i128(int x)
}
inline wInt256 exp10_i256(int x)
inline Int256 exp10_i256(int x)
{
using Int256 = wInt256;
using Int256 = Int256;
static constexpr Int256 i10e18{1000000000000000000ll};
static const Int256 values[] = {
static_cast<Int256>(1ll),

View File

@ -5,11 +5,17 @@
#include <Common/HashTable/HashMap.h>
#include <Common/HashTable/HashSet.h>
#include <Common/HashTable/Hash.h>
#include <IO/ReadBufferFromString.h>
#include <IO/WriteHelpers.h>
#include <gtest/gtest.h>
using namespace DB;
/// To test dump functionality without using other hashes that can change
template <typename T>
struct DummyHash
@ -18,12 +24,12 @@ struct DummyHash
};
template<typename HashTable>
std::set<typename HashTable::value_type> convertToSet(const HashTable& table)
std::set<std::string> convertToSet(const HashTable & table)
{
std::set<typename HashTable::value_type> result;
std::set<std::string> result;
for (auto v: table)
result.emplace(v.getValue());
result.emplace(toString(v.getValue()));
return result;
}
@ -91,8 +97,8 @@ TEST(HashTable, Iteration)
cont.insert(2);
cont.insert(3);
std::set<int> expected = {1, 2, 3};
std::set<int> actual = convertToSet(cont);
std::set<std::string> expected = {"1", "2", "3"};
std::set<std::string> actual = convertToSet(cont);
ASSERT_EQ(actual, expected);
}
@ -251,14 +257,14 @@ TEST(HashTable, SerializationDeserialization)
cont.insert(2);
cont.insert(3);
DB::WriteBufferFromOwnString wb;
WriteBufferFromOwnString wb;
cont.writeText(wb);
std::string expected = "3,1,2,3";
ASSERT_EQ(wb.str(), expected);
DB::ReadBufferFromString rb(expected);
ReadBufferFromString rb(expected);
Cont deserialized;
deserialized.readText(rb);
@ -273,10 +279,10 @@ TEST(HashTable, SerializationDeserialization)
cont.insert(2);
cont.insert(3);
DB::WriteBufferFromOwnString wb;
WriteBufferFromOwnString wb;
cont.write(wb);
DB::ReadBufferFromString rb(wb.str());
ReadBufferFromString rb(wb.str());
Cont deserialized;
deserialized.read(rb);
@ -286,23 +292,23 @@ TEST(HashTable, SerializationDeserialization)
using Cont = HashSet<int, DummyHash<int>, HashTableGrower<1>>;
Cont cont;
DB::WriteBufferFromOwnString wb;
WriteBufferFromOwnString wb;
cont.writeText(wb);
std::string expected = "0";
ASSERT_EQ(wb.str(), expected);
DB::ReadBufferFromString rb(expected);
ReadBufferFromString rb(expected);
Cont deserialized;
deserialized.readText(rb);
ASSERT_EQ(convertToSet(cont), convertToSet(deserialized));
}
{
using Cont = HashSet<DB::UInt128, DB::UInt128TrivialHash>;
using Cont = HashSet<UInt128, UInt128TrivialHash>;
Cont cont;
DB::WriteBufferFromOwnString wb;
WriteBufferFromOwnString wb;
cont.write(wb);
std::string expected;
@ -310,7 +316,7 @@ TEST(HashTable, SerializationDeserialization)
ASSERT_EQ(wb.str(), expected);
DB::ReadBufferFromString rb(expected);
ReadBufferFromString rb(expected);
Cont deserialized;
deserialized.read(rb);
@ -359,8 +365,8 @@ TEST(HashTable, Resize)
cont.insert(3);
cont.insert(1);
std::set<int> expected = {1, 3};
std::set<int> actual = convertToSet(cont);
std::set<std::string> expected = {"1", "3"};
std::set<std::string> actual = convertToSet(cont);
ASSERT_EQ(actual, expected);
}

View File

@ -459,18 +459,6 @@ TEST(Common, PODArrayInsertFromItself)
}
}
TEST(Common, PODPushBackRawMany)
{
PODArray<char> chars;
chars.push_back_raw_many(5, "first");
EXPECT_EQ(std::string("first"), std::string(chars.data(), chars.size()));
EXPECT_EQ(5, chars.size());
EXPECT_LE(chars.capacity() - chars.size(), 10);
chars.push_back_raw_many(10, "0123456789");
EXPECT_EQ(15, chars.size());
EXPECT_EQ(std::string("first0123456789"), std::string(chars.data(), chars.size()));
}
TEST(Common, PODNoOverallocation)
{
/// Check that PaddedPODArray allocates for smaller number of elements than the power of two due to padding.

View File

@ -0,0 +1,281 @@
#include <gtest/gtest.h>
#include <cstdint>
#include <limits>
#include <type_traits>
#include <initializer_list>
#include <Core/Types.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadHelpers.h>
#include <common/demangle.h>
static_assert(is_signed_v<Int128>);
static_assert(!is_unsigned_v<Int128>);
static_assert(is_integer_v<Int128>);
static_assert(sizeof(Int128) == 16);
static_assert(is_signed_v<Int256>);
static_assert(!is_unsigned_v<Int256>);
static_assert(is_integer_v<Int256>);
static_assert(sizeof(Int256) == 32);
static_assert(!is_signed_v<UInt128>);
static_assert(is_unsigned_v<UInt128>);
static_assert(is_integer_v<UInt128>);
static_assert(sizeof(UInt128) == 16);
static_assert(!is_signed_v<UInt256>);
static_assert(is_unsigned_v<UInt256>);
static_assert(is_integer_v<UInt256>);
static_assert(sizeof(UInt256) == 32);
using namespace DB;
GTEST_TEST(WideInteger, Conversions)
{
ASSERT_EQ(toString(UInt128(12345678901234567890ULL)), "12345678901234567890");
ASSERT_EQ(toString(UInt256(12345678901234567890ULL)), "12345678901234567890");
Int128 minus_one = -1;
ASSERT_EQ(minus_one.items[0], -1);
ASSERT_EQ(minus_one.items[1], -1);
ASSERT_EQ(0, memcmp(&minus_one, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", sizeof(minus_one)));
ASSERT_EQ(minus_one, -1);
ASSERT_EQ(minus_one, -1LL);
ASSERT_EQ(minus_one, Int8(-1));
ASSERT_EQ(minus_one, Int16(-1));
ASSERT_EQ(minus_one, Int32(-1));
ASSERT_EQ(minus_one, Int64(-1));
ASSERT_LT(minus_one, 0);
Int128 zero = 0;
zero += -1;
ASSERT_EQ(zero, -1);
ASSERT_EQ(zero, minus_one);
zero += minus_one;
ASSERT_EQ(0, memcmp(&zero, "\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", sizeof(zero)));
zero += 2;
ASSERT_EQ(zero, 0);
ASSERT_EQ(toString(Int128(-1)), "-1");
ASSERT_EQ(toString(Int256(-1)), "-1");
ASSERT_EQ(toString(Int128(-1LL)), "-1");
ASSERT_EQ(toString(Int256(-1LL)), "-1");
ASSERT_EQ(toString(Int128(-1234567890123456789LL)), "-1234567890123456789");
ASSERT_EQ(toString(Int256(-1234567890123456789LL)), "-1234567890123456789");
ASSERT_EQ(UInt64(UInt128(12345678901234567890ULL)), 12345678901234567890ULL);
ASSERT_EQ(UInt64(UInt256(12345678901234567890ULL)), 12345678901234567890ULL);
ASSERT_EQ(__uint128_t(UInt128(12345678901234567890ULL)), 12345678901234567890ULL);
ASSERT_EQ(__uint128_t(UInt256(12345678901234567890ULL)), 12345678901234567890ULL);
ASSERT_EQ(__int128_t(Int128(-1234567890123456789LL)), -1234567890123456789LL);
ASSERT_EQ(__int128_t(Int256(-1234567890123456789LL)), -1234567890123456789LL);
ASSERT_EQ(toString(Int128(-1)), "-1");
ASSERT_EQ(toString(Int256(-1)), "-1");
ASSERT_EQ(toString(UInt128(123.456)), "123");
ASSERT_EQ(toString(UInt256(123.456)), "123");
ASSERT_EQ(toString(Int128(-123.456)), "-123");
ASSERT_EQ(toString(Int256(-123.456)), "-123");
ASSERT_EQ(toString(UInt128(123.456f)), "123");
ASSERT_EQ(toString(UInt256(123.456f)), "123");
ASSERT_EQ(toString(Int128(-123.456f)), "-123");
ASSERT_EQ(toString(Int256(-123.456f)), "-123");
ASSERT_EQ(toString(UInt128(1) * 1000000000 * 1000000000 * 1000000000 * 1000000000), "1000000000000000000000000000000000000");
ASSERT_EQ(Float64(UInt128(1) * 1000000000 * 1000000000 * 1000000000 * 1000000000), 1e36);
ASSERT_EQ(toString(UInt256(1) * 1000000000 * 1000000000 * 1000000000 * 1000000000 * 1000000000 * 1000000000 * 1000000000 * 1000000000),
"1000000000000000000000000000000000000000000000000000000000000000000000000");
ASSERT_EQ(Float64(UInt256(1) * 1000000000 * 1000000000 * 1000000000 * 1000000000 * 1000000000 * 1000000000 * 1000000000 * 1000000000), 1e72);
EXPECT_EQ(toString(parse<Int128>("148873535527910577765226390751398592640")), "148873535527910577765226390751398592640");
EXPECT_EQ(toString(parse<UInt128>("148873535527910577765226390751398592640")), "148873535527910577765226390751398592640");
}
template <typename T>
static T divide(T & numerator, T && denominator)
{
if (!denominator)
throwError("Division by zero");
T & n = numerator;
T & d = denominator;
T x = 1;
T quotient = 0;
/// Multiply d to the power of two until it will be greater than n.
/// The factor will be collected in x.
while (d <= n && ((d >> (sizeof(T) * 8 - 1)) & 1) == 0)
{
x <<= 1;
d <<= 1;
}
std::cerr << toString(x) << ", " << toString(d) << "\n";
while (x)
{
if (d <= n)
{
n -= d;
quotient |= x;
}
x >>= 1;
d >>= 1;
}
return quotient;
}
GTEST_TEST(WideInteger, Arithmetic)
{
Int128 minus_one = -1;
Int128 zero = 0;
zero += -1;
ASSERT_EQ(zero, -1);
ASSERT_EQ(zero, minus_one);
zero += minus_one;
ASSERT_EQ(0, memcmp(&zero, "\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", sizeof(zero)));
zero += 2;
ASSERT_EQ(zero, 0);
ASSERT_EQ(UInt256(12345678901234567890ULL) * 12345678901234567890ULL / 12345678901234567890ULL, 12345678901234567890ULL);
ASSERT_EQ(UInt256(12345678901234567890ULL) * UInt256(12345678901234567890ULL) / 12345678901234567890ULL, 12345678901234567890ULL);
ASSERT_EQ(UInt256(12345678901234567890ULL) * 12345678901234567890ULL / UInt256(12345678901234567890ULL), 12345678901234567890ULL);
ASSERT_EQ(UInt256(12345678901234567890ULL) * 12345678901234567890ULL / 12345678901234567890ULL, UInt256(12345678901234567890ULL));
ASSERT_EQ(UInt128(12345678901234567890ULL) * 12345678901234567890ULL / UInt128(12345678901234567890ULL), 12345678901234567890ULL);
ASSERT_EQ(UInt256(12345678901234567890ULL) * UInt128(12345678901234567890ULL) / 12345678901234567890ULL, 12345678901234567890ULL);
ASSERT_EQ(Int128(0) + Int32(-1), Int128(-1));
Int128 x(parse<Int128>("148873535527910577765226390751398592640"));
Int128 dividend = x / 10;
ASSERT_EQ(toString(dividend), "14887353552791057776522639075139859264");
}
GTEST_TEST(WideInteger, DecimalArithmetic)
{
Decimal128 zero{};
Decimal32 addend = -1000;
zero += Decimal128(addend);
ASSERT_EQ(zero.value, -1000);
zero += addend;
ASSERT_EQ(zero.value, -2000);
}
GTEST_TEST(WideInteger, FromDouble)
{
/// Check that we are being able to convert double to big integer without the help of floating point instructions.
/// (a prototype of a function that we may need)
double f = -123.456;
UInt64 u;
memcpy(&u, &f, sizeof(f));
bool is_negative = u >> 63;
uint16_t exponent = (u >> 52) & (((1ull << 12) - 1) >> 1);
int16_t normalized_exponent = exponent - 1023;
UInt64 mantissa = u & ((1ull << 52) - 1);
// std::cerr << is_negative << ", " << normalized_exponent << ", " << mantissa << "\n";
/// x = sign * (2 ^ normalized_exponent + mantissa * 2 ^ (normalized_exponent - mantissa_bits))
Int128 res = 0;
if (normalized_exponent >= 128)
{
}
else
{
res = mantissa;
if (normalized_exponent > 52)
res <<= (normalized_exponent - 52);
else
res >>= (52 - normalized_exponent);
if (normalized_exponent > 0)
res += Int128(1) << normalized_exponent;
}
if (is_negative)
res = -res;
ASSERT_EQ(toString(res), "-123");
}
GTEST_TEST(WideInteger, Shift)
{
Int128 x = 1;
auto y = x << 64;
ASSERT_EQ(0, memcmp(&y, "\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00", sizeof(Int128)));
auto z = y << 11;
ASSERT_EQ(toString(z), "37778931862957161709568");
auto a = x << 11;
ASSERT_EQ(a, 2048);
z >>= 64;
ASSERT_EQ(z, a);
x = -1;
y = x << 16;
ASSERT_EQ(0, memcmp(&y, "\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", sizeof(Int128)));
y >>= 16;
ASSERT_EQ(0, memcmp(&y, "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", sizeof(Int128)));
y <<= 64;
ASSERT_EQ(0, memcmp(&y, "\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", sizeof(Int128)));
y >>= 32;
ASSERT_EQ(0, memcmp(&y, "\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF", sizeof(Int128)));
y <<= 64;
ASSERT_EQ(0, memcmp(&y, "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xFF\xFF\xFF\xFF", sizeof(Int128)));
}
GTEST_TEST(WideInteger, DecimalFormatting)
{
Decimal128 x(parse<Int128>("148873535527910577765226390751398592640"));
EXPECT_EQ(toString(x.value), "148873535527910577765226390751398592640");
EXPECT_EQ(toString(x.value / 10), "14887353552791057776522639075139859264");
EXPECT_EQ(toString(x.value % 10), "0");
Int128 fractional = DecimalUtils::getFractionalPart(x, 2);
EXPECT_EQ(fractional, 40);
EXPECT_EQ(decimalFractional(fractional, 2), "40");
}

View File

@ -6,8 +6,6 @@
#include <Common/PODArray.h>
#include <common/logger_useful.h>
#include <Common/UInt128.h>
namespace DB
{

View File

@ -2,11 +2,11 @@
#include <cmath>
#include <limits>
#include "Defines.h"
#include "Types.h"
#include <common/DecomposedFloat.h>
#include <Core/Defines.h>
#include <Core/Types.h>
#include <common/extended_types.h>
#include <Common/NaNUtils.h>
#include <Common/UInt128.h>
/** Preceptually-correct number comparisons.
* Example: Int8(-1) != UInt8(255)
@ -15,504 +15,142 @@
namespace accurate
{
using DB::UInt64;
using namespace DB;
/** Cases:
1) Safe conversion (in case of default C++ operators)
a) int vs any int
b) uint vs any uint
c) float vs any float
2) int vs uint
a) sizeof(int) <= sizeof(uint). Accurate comparison with MAX_INT tresholds
b) sizeof(int) > sizeof(uint). Casting to int
3) integral_type vs floating_type
a) sizeof(integral_type) <= 4. Comparison via casting arguments to Float64
b) sizeof(integral_type) == 8. Accurate comparison. Consider 3 sets of intervals:
1) interval between adjacent floats less or equal 1
2) interval between adjacent floats greater then 2
3) float is outside [MIN_INT64; MAX_INT64]
*/
// Case 1. Is pair of floats or pair of ints or pair of uints
template <typename A, typename B>
constexpr bool is_safe_conversion = (std::is_floating_point_v<A> && std::is_floating_point_v<B>)
|| (is_integer_v<A> && is_integer_v<B> && !(is_signed_v<A> ^ is_signed_v<B>));
bool lessOp(A a, B b)
{
if constexpr (std::is_same_v<A, B>)
return a < b;
/// float vs float
if constexpr (std::is_floating_point_v<A> && std::is_floating_point_v<B>)
return a < b;
/// anything vs NaN
if (isNaN(a) || isNaN(b))
return false;
/// int vs int
if constexpr (is_integer_v<A> && is_integer_v<B>)
{
/// same signedness
if constexpr (is_signed_v<A> == is_signed_v<B>)
return a < b;
/// different signedness
if constexpr (is_signed_v<A> && !is_signed_v<B>)
return a < 0 || static_cast<make_unsigned_t<A>>(a) < b;
if constexpr (!is_signed_v<A> && is_signed_v<B>)
return b >= 0 && a < static_cast<make_unsigned_t<B>>(b);
}
/// int vs float
if constexpr (is_integer_v<A> && std::is_floating_point_v<B>)
{
if constexpr (sizeof(A) <= 4)
return static_cast<double>(a) < static_cast<double>(b);
return DecomposedFloat<B>(b).greater(a);
}
if constexpr (std::is_floating_point_v<A> && is_integer_v<B>)
{
if constexpr (sizeof(B) <= 4)
return static_cast<double>(a) < static_cast<double>(b);
return DecomposedFloat<A>(a).less(b);
}
static_assert(is_integer_v<A> || std::is_floating_point_v<A>);
static_assert(is_integer_v<B> || std::is_floating_point_v<B>);
__builtin_unreachable();
}
template <typename A, typename B>
using bool_if_safe_conversion = std::enable_if_t<is_safe_conversion<A, B>, bool>;
bool greaterOp(A a, B b)
{
return lessOp(b, a);
}
template <typename A, typename B>
using bool_if_not_safe_conversion = std::enable_if_t<!is_safe_conversion<A, B>, bool>;
/// Case 2. Are params IntXX and UIntYY ?
template <typename TInt, typename TUInt>
constexpr bool is_any_int_vs_uint
= is_integer_v<TInt> && is_integer_v<TUInt> && is_signed_v<TInt> && is_unsigned_v<TUInt>;
// Case 2a. Are params IntXX and UIntYY and sizeof(IntXX) >= sizeof(UIntYY) (in such case will use accurate compare)
template <typename TInt, typename TUInt>
constexpr bool is_le_int_vs_uint = is_any_int_vs_uint<TInt, TUInt> && (sizeof(TInt) <= sizeof(TUInt));
static_assert(is_le_int_vs_uint<Int128, DB::UInt128>);
static_assert(is_le_int_vs_uint<Int128, DB::UInt256>);
template <typename TInt, typename TUInt>
using bool_if_le_int_vs_uint_t = std::enable_if_t<is_le_int_vs_uint<TInt, TUInt>, bool>;
template <typename TInt, typename TUInt>
inline bool_if_le_int_vs_uint_t<TInt, TUInt> greaterOpTmpl(TInt a, TUInt b)
bool greaterOrEqualsOp(A a, B b)
{
return static_cast<TUInt>(a) > b && a >= 0 && b <= static_cast<TUInt>(std::numeric_limits<TInt>::max());
if (isNaN(a) || isNaN(b))
return false;
return !lessOp(a, b);
}
template <typename TUInt, typename TInt>
inline bool_if_le_int_vs_uint_t<TInt, TUInt> greaterOpTmpl(TUInt a, TInt b)
template <typename A, typename B>
bool lessOrEqualsOp(A a, B b)
{
return a > static_cast<TUInt>(b) || b < 0 || a > static_cast<TUInt>(std::numeric_limits<TInt>::max());
if (isNaN(a) || isNaN(b))
return false;
return !lessOp(b, a);
}
template <typename TInt, typename TUInt>
inline bool_if_le_int_vs_uint_t<TInt, TUInt> equalsOpTmpl(TInt a, TUInt b)
template <typename A, typename B>
bool equalsOp(A a, B b)
{
return static_cast<TUInt>(a) == b && a >= 0 && b <= static_cast<TUInt>(std::numeric_limits<TInt>::max());
}
if constexpr (std::is_same_v<A, B>)
return a == b;
template <typename TUInt, typename TInt>
inline bool_if_le_int_vs_uint_t<TInt, TUInt> equalsOpTmpl(TUInt a, TInt b)
{
return a == static_cast<TUInt>(b) && b >= 0 && a <= static_cast<TUInt>(std::numeric_limits<TInt>::max());
}
/// float vs float
if constexpr (std::is_floating_point_v<A> && std::is_floating_point_v<B>)
return a == b;
/// anything vs NaN
if (isNaN(a) || isNaN(b))
return false;
// Case 2b. Are params IntXX and UIntYY and sizeof(IntXX) > sizeof(UIntYY) (in such case will cast UIntYY to IntXX and compare)
template <typename TInt, typename TUInt>
constexpr bool is_gt_int_vs_uint = is_any_int_vs_uint<TInt, TUInt> && (sizeof(TInt) > sizeof(TUInt));
/// int vs int
if constexpr (is_integer_v<A> && is_integer_v<B>)
{
/// same signedness
if constexpr (is_signed_v<A> == is_signed_v<B>)
return a == b;
template <typename TInt, typename TUInt>
using bool_if_gt_int_vs_uint = std::enable_if_t<is_gt_int_vs_uint<TInt, TUInt>, bool>;
/// different signedness
template <typename TInt, typename TUInt>
inline bool_if_gt_int_vs_uint<TInt, TUInt> greaterOpTmpl(TInt a, TUInt b)
{
return static_cast<TInt>(a) > static_cast<TInt>(b);
}
if constexpr (is_signed_v<A> && !is_signed_v<B>)
return a >= 0 && static_cast<make_unsigned_t<A>>(a) == b;
template <typename TInt, typename TUInt>
inline bool_if_gt_int_vs_uint<TInt, TUInt> greaterOpTmpl(TUInt a, TInt b)
{
using CastA = std::conditional_t<is_big_int_v<TInt> && std::is_same_v<TUInt, DB::UInt128>, DB::UInt256, TInt>;
if constexpr (!is_signed_v<A> && is_signed_v<B>)
return b >= 0 && a == static_cast<make_unsigned_t<B>>(b);
}
return static_cast<CastA>(a) > b;
}
/// int vs float
if constexpr (is_integer_v<A> && std::is_floating_point_v<B>)
{
if constexpr (sizeof(A) <= 4)
return static_cast<double>(a) == static_cast<double>(b);
template <typename TInt, typename TUInt>
inline bool_if_gt_int_vs_uint<TInt, TUInt> equalsOpTmpl(TInt a, TUInt b)
{
return static_cast<TInt>(a) == static_cast<TInt>(b);
}
return DecomposedFloat<B>(b).equals(a);
}
template <typename TInt, typename TUInt>
inline bool_if_gt_int_vs_uint<TInt, TUInt> equalsOpTmpl(TUInt a, TInt b)
{
return static_cast<TInt>(a) == static_cast<TInt>(b);
}
if constexpr (std::is_floating_point_v<A> && is_integer_v<B>)
{
if constexpr (sizeof(B) <= 4)
return static_cast<double>(a) == static_cast<double>(b);
return DecomposedFloat<A>(a).equals(b);
}
// Case 3a. Comparison via conversion to double.
template <typename TAInt, typename TAFloat>
using bool_if_double_can_be_used
= std::enable_if_t<is_integer_v<TAInt> && (sizeof(TAInt) <= 4) && std::is_floating_point_v<TAFloat>, bool>;
template <typename TAInt, typename TAFloat>
inline bool_if_double_can_be_used<TAInt, TAFloat> greaterOpTmpl(TAInt a, TAFloat b)
{
return static_cast<double>(a) > static_cast<double>(b);
}
template <typename TAInt, typename TAFloat>
inline bool_if_double_can_be_used<TAInt, TAFloat> greaterOpTmpl(TAFloat a, TAInt b)
{
return static_cast<double>(a) > static_cast<double>(b);
}
template <typename TAInt, typename TAFloat>
inline bool_if_double_can_be_used<TAInt, TAFloat> equalsOpTmpl(TAInt a, TAFloat b)
{
return static_cast<double>(a) == static_cast<double>(b);
}
template <typename TAInt, typename TAFloat>
inline bool_if_double_can_be_used<TAInt, TAFloat> equalsOpTmpl(TAFloat a, TAInt b)
{
return static_cast<double>(a) == static_cast<double>(b);
}
// Big integers vs Float (not equal in any case for now, until big floats are introduced?)
template <typename TABigInt, typename TAFloat>
constexpr bool if_big_int_vs_float = (is_big_int_v<TABigInt> && std::is_floating_point_v<TAFloat>);
template <typename TABigInt, typename TAFloat>
using bool_if_big_int_vs_float = std::enable_if_t<if_big_int_vs_float<TABigInt, TAFloat>, bool>;
template <typename TABigInt, typename TAFloat>
inline bool_if_big_int_vs_float<TABigInt, TAFloat> greaterOpTmpl(TAFloat, TABigInt)
{
/// e.g comparing UUID with integer.
return false;
}
template <typename TABigInt, typename TAFloat>
inline bool_if_big_int_vs_float<TABigInt, TAFloat> greaterOpTmpl(TABigInt, TAFloat)
{
return false;
}
template <typename TABigInt, typename TAFloat>
inline bool_if_big_int_vs_float<TABigInt, TAFloat> equalsOpTmpl(TAFloat, TABigInt)
{
return false;
}
template <typename TABigInt, typename TAFloat>
inline bool_if_big_int_vs_float<TABigInt, TAFloat> equalsOpTmpl(TABigInt, TAFloat)
{
return false;
}
/* Final implementations */
template <typename A, typename B>
inline bool_if_not_safe_conversion<A, B> greaterOp(A a, B b)
{
return greaterOpTmpl(a, b);
}
template <typename A, typename B>
inline bool_if_safe_conversion<A, B> greaterOp(A a, B b)
{
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
return static_cast<CastA>(a) > static_cast<CastB>(b);
else
return a > b;
}
// Case 3b. 64-bit integers vs floats comparison.
// See hint at https://github.com/JuliaLang/julia/issues/257 (but it doesn't work properly for -2**63)
constexpr DB::Int64 MAX_INT64_WITH_EXACT_FLOAT64_REPR = 9007199254740992LL; // 2^53
template <>
inline bool greaterOp<DB::Float64, DB::Int64>(DB::Float64 f, DB::Int64 i)
{
if (-MAX_INT64_WITH_EXACT_FLOAT64_REPR <= i && i <= MAX_INT64_WITH_EXACT_FLOAT64_REPR)
return f > static_cast<DB::Float64>(i);
return (f >= static_cast<DB::Float64>(std::numeric_limits<DB::Int64>::max())) // rhs is 2**63 (not 2^63 - 1)
|| (f > static_cast<DB::Float64>(std::numeric_limits<DB::Int64>::min()) && static_cast<DB::Int64>(f) > i);
}
template <>
inline bool greaterOp<DB::Int64, DB::Float64>(DB::Int64 i, DB::Float64 f)
{
if (-MAX_INT64_WITH_EXACT_FLOAT64_REPR <= i && i <= MAX_INT64_WITH_EXACT_FLOAT64_REPR)
return f < static_cast<DB::Float64>(i);
return (f < static_cast<DB::Float64>(std::numeric_limits<DB::Int64>::min()))
|| (f < static_cast<DB::Float64>(std::numeric_limits<DB::Int64>::max()) && i > static_cast<DB::Int64>(f));
}
template <>
inline bool greaterOp<DB::Float64, DB::UInt64>(DB::Float64 f, DB::UInt64 u)
{
if (u <= static_cast<DB::UInt64>(MAX_INT64_WITH_EXACT_FLOAT64_REPR))
return f > static_cast<DB::Float64>(u);
return (f >= static_cast<DB::Float64>(std::numeric_limits<DB::UInt64>::max()))
|| (f >= 0 && static_cast<DB::UInt64>(f) > u);
}
template <>
inline bool greaterOp<DB::UInt64, DB::Float64>(DB::UInt64 u, DB::Float64 f)
{
if (u <= static_cast<DB::UInt64>(MAX_INT64_WITH_EXACT_FLOAT64_REPR))
return static_cast<DB::Float64>(u) > f;
return (f < 0)
|| (f < static_cast<DB::Float64>(std::numeric_limits<DB::UInt64>::max()) && u > static_cast<UInt64>(f));
}
// Case 3b for float32
template <>
inline bool greaterOp<DB::Float32, DB::Int64>(DB::Float32 f, DB::Int64 i)
{
return greaterOp(static_cast<DB::Float64>(f), i);
}
template <>
inline bool greaterOp<DB::Int64, DB::Float32>(DB::Int64 i, DB::Float32 f)
{
return greaterOp(i, static_cast<DB::Float64>(f));
}
template <>
inline bool greaterOp<DB::Float32, DB::UInt64>(DB::Float32 f, DB::UInt64 u)
{
return greaterOp(static_cast<DB::Float64>(f), u);
}
template <>
inline bool greaterOp<DB::UInt64, DB::Float32>(DB::UInt64 u, DB::Float32 f)
{
return greaterOp(u, static_cast<DB::Float64>(f));
}
template <>
inline bool greaterOp<DB::Float64, DB::UInt128>(DB::Float64 f, DB::UInt128 u)
{
/// TODO: This is wrong.
return u.low == 0 && greaterOp(f, u.high);
}
template <>
inline bool greaterOp<DB::UInt128, DB::Float64>(DB::UInt128 u, DB::Float64 f)
{
/// TODO: This is wrong.
return u.low != 0 || greaterOp(u.high, f);
}
template <>
inline bool greaterOp<DB::Float32, DB::UInt128>(DB::Float32 f, DB::UInt128 u)
{
return greaterOp(static_cast<DB::Float64>(f), u);
}
template <>
inline bool greaterOp<DB::UInt128, DB::Float32>(DB::UInt128 u, DB::Float32 f)
{
return greaterOp(u, static_cast<DB::Float64>(f));
}
template <typename A, typename B>
inline bool_if_not_safe_conversion<A, B> equalsOp(A a, B b)
{
return equalsOpTmpl(a, b);
}
template <typename A, typename B>
inline bool_if_safe_conversion<A, B> equalsOp(A a, B b)
{
using LargestType = std::conditional_t<(sizeof(A) > sizeof(B)) || ((sizeof(A) == sizeof(B)) && !std::is_same_v<A, DB::UInt128>), A, B>;
return static_cast<LargestType>(a) == static_cast<LargestType>(b);
}
template <>
inline bool equalsOp<DB::Float64, DB::UInt64>(DB::Float64 f, DB::UInt64 u)
{
/// Maximum exactly representable integer.
return u <= (1ULL << std::numeric_limits<DB::Float64>::digits)
&& f == static_cast<DB::Float64>(u);
}
template <>
inline bool equalsOp<DB::UInt64, DB::Float64>(DB::UInt64 u, DB::Float64 f)
{
return equalsOp(f, u);
}
template <>
inline bool equalsOp<DB::Float64, DB::Int64>(DB::Float64 f, DB::Int64 u)
{
return u <= (1LL << std::numeric_limits<DB::Float64>::digits)
&& u >= -(1LL << std::numeric_limits<DB::Float64>::digits)
&& f == static_cast<DB::Float64>(u);
}
template <>
inline bool equalsOp<DB::Int64, DB::Float64>(DB::Int64 u, DB::Float64 f)
{
return equalsOp(f, u);
}
template <>
inline bool equalsOp<DB::Float32, DB::UInt64>(DB::Float32 f, DB::UInt64 u)
{
return u <= (1ULL << std::numeric_limits<DB::Float32>::digits)
&& f == static_cast<DB::Float32>(u);
}
template <>
inline bool equalsOp<DB::UInt64, DB::Float32>(DB::UInt64 u, DB::Float32 f)
{
return equalsOp(f, u);
}
template <>
inline bool equalsOp<DB::Float32, DB::Int64>(DB::Float32 f, DB::Int64 u)
{
return u <= (1LL << std::numeric_limits<DB::Float32>::digits)
&& u >= -(1LL << std::numeric_limits<DB::Float32>::digits)
&& f == static_cast<DB::Float32>(u);
}
template <>
inline bool equalsOp<DB::Int64, DB::Float32>(DB::Int64 u, DB::Float32 f)
{
return equalsOp(f, u);
}
template <>
inline bool equalsOp<DB::UInt128, DB::Float64>(DB::UInt128 u, DB::Float64 f)
{
/// TODO: This is wrong.
return u.low == 0 && equalsOp(static_cast<UInt64>(u.high), f);
}
template <>
inline bool equalsOp<DB::UInt128, DB::Float32>(DB::UInt128 u, DB::Float32 f)
{
return equalsOp(u, static_cast<DB::Float64>(f));
}
template <>
inline bool equalsOp<DB::Float64, DB::UInt128>(DB::Float64 f, DB::UInt128 u)
{
return equalsOp(u, f);
}
template <>
inline bool equalsOp<DB::Float32, DB::UInt128>(DB::Float32 f, DB::UInt128 u)
{
return equalsOp(static_cast<DB::Float64>(f), u);
}
inline bool NO_SANITIZE_UNDEFINED greaterOp(DB::Int128 i, DB::Float64 f)
{
static constexpr Int128 min_int128 = minInt128();
static constexpr Int128 max_int128 = maxInt128();
if (-MAX_INT64_WITH_EXACT_FLOAT64_REPR <= i && i <= MAX_INT64_WITH_EXACT_FLOAT64_REPR)
return static_cast<DB::Float64>(i) > f;
return (f < static_cast<DB::Float64>(min_int128))
|| (f < static_cast<DB::Float64>(max_int128) && i > static_cast<DB::Int128>(f));
}
inline bool NO_SANITIZE_UNDEFINED greaterOp(DB::Float64 f, DB::Int128 i)
{
static constexpr Int128 min_int128 = minInt128();
static constexpr Int128 max_int128 = maxInt128();
if (-MAX_INT64_WITH_EXACT_FLOAT64_REPR <= i && i <= MAX_INT64_WITH_EXACT_FLOAT64_REPR)
return f > static_cast<DB::Float64>(i);
return (f >= static_cast<DB::Float64>(max_int128))
|| (f > static_cast<DB::Float64>(min_int128) && static_cast<DB::Int128>(f) > i);
}
inline bool greaterOp(DB::Int128 i, DB::Float32 f) { return greaterOp(i, static_cast<DB::Float64>(f)); }
inline bool greaterOp(DB::Float32 f, DB::Int128 i) { return greaterOp(static_cast<DB::Float64>(f), i); }
inline bool NO_SANITIZE_UNDEFINED equalsOp(DB::Int128 i, DB::Float64 f) { return i == static_cast<DB::Int128>(f) && static_cast<DB::Float64>(i) == f; }
inline bool NO_SANITIZE_UNDEFINED equalsOp(DB::Int128 i, DB::Float32 f) { return i == static_cast<DB::Int128>(f) && static_cast<DB::Float32>(i) == f; }
inline bool equalsOp(DB::Float64 f, DB::Int128 i) { return equalsOp(i, f); }
inline bool equalsOp(DB::Float32 f, DB::Int128 i) { return equalsOp(i, f); }
template <typename A, typename B>
inline bool_if_not_safe_conversion<A, B> notEqualsOp(A a, B b)
bool notEqualsOp(A a, B b)
{
return !equalsOp(a, b);
}
template <typename A, typename B>
inline bool_if_safe_conversion<A, B> notEqualsOp(A a, B b)
{
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
return static_cast<CastA>(a) != static_cast<CastB>(b);
else
return a != b;
}
template <typename A, typename B>
inline bool_if_not_safe_conversion<A, B> lessOp(A a, B b)
{
return greaterOp(b, a);
}
template <typename A, typename B>
inline bool_if_safe_conversion<A, B> lessOp(A a, B b)
{
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
return static_cast<CastA>(a) < static_cast<CastB>(b);
else
return a < b;
}
template <typename A, typename B>
inline bool_if_not_safe_conversion<A, B> lessOrEqualsOp(A a, B b)
{
if (isNaN(a) || isNaN(b))
return false;
return !greaterOp(a, b);
}
template <typename A, typename B>
inline bool_if_safe_conversion<A, B> lessOrEqualsOp(A a, B b)
{
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
return static_cast<CastA>(a) <= static_cast<CastB>(b);
else
return a <= b;
}
template <typename A, typename B>
inline bool_if_not_safe_conversion<A, B> greaterOrEqualsOp(A a, B b)
{
if (isNaN(a) || isNaN(b))
return false;
return !greaterOp(b, a);
}
template <typename A, typename B>
inline bool_if_safe_conversion<A, B> greaterOrEqualsOp(A a, B b)
{
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
return static_cast<CastA>(a) >= static_cast<CastB>(b);
else
return a >= b;
}
/// Converts numeric to an equal numeric of other type.
template <typename From, typename To>
@ -547,8 +185,8 @@ inline bool NO_SANITIZE_UNDEFINED convertNumeric(From value, To & result)
}
}
if (accurate::greaterOp(value, std::numeric_limits<To>::max())
|| accurate::greaterOp(std::numeric_limits<To>::lowest(), value))
if (greaterOp(value, std::numeric_limits<To>::max())
|| lessOp(value, std::numeric_limits<To>::lowest()))
{
return false;
}

View File

@ -2,7 +2,6 @@
#include <Core/SettingsFields.h>
#include <Common/SettingsChanges.h>
#include <Common/FieldVisitors.h>
#include <ext/range.h>
#include <boost/blank.hpp>
#include <unordered_map>

View File

@ -36,10 +36,13 @@ inline bool allowDecimalComparison(const DataTypePtr & left_type, const DataType
return false;
}
template <size_t > struct ConstructDecInt { using Type = Int32; };
template <size_t> struct ConstructDecInt;
template <> struct ConstructDecInt<1> { using Type = Int32; };
template <> struct ConstructDecInt<2> { using Type = Int32; };
template <> struct ConstructDecInt<4> { using Type = Int32; };
template <> struct ConstructDecInt<8> { using Type = Int64; };
template <> struct ConstructDecInt<16> { using Type = Int128; };
template <> struct ConstructDecInt<48> { using Type = Int256; };
template <> struct ConstructDecInt<32> { using Type = Int256; };
template <typename T, typename U>
struct DecCompareInt
@ -249,7 +252,7 @@ private:
overflow |= common::mulOverflow(y, scale, y);
if (overflow)
throw Exception("Can't compare", ErrorCodes::DECIMAL_OVERFLOW);
throw Exception("Can't compare decimal number due to overflow", ErrorCodes::DECIMAL_OVERFLOW);
}
else
{

View File

@ -276,9 +276,7 @@ template <typename To, typename DecimalType>
To convertTo(const DecimalType & decimal, size_t scale)
{
To result;
convertToImpl<To, DecimalType, void>(decimal, scale, result);
return result;
}

View File

@ -238,6 +238,13 @@ Field Field::restoreFromDump(const std::string_view & dump_)
return value;
}
prefix = std::string_view{"UInt128_"};
if (dump.starts_with(prefix))
{
UInt128 value = parseFromString<UInt128>(dump.substr(prefix.length()));
return value;
}
prefix = std::string_view{"Int256_"};
if (dump.starts_with(prefix))
{
@ -295,15 +302,6 @@ Field Field::restoreFromDump(const std::string_view & dump_)
return decimal;
}
prefix = std::string_view{"UUID_"};
if (dump.starts_with(prefix))
{
UUID uuid;
ReadBufferFromString buf{dump.substr(prefix.length())};
readQuoted(uuid, buf);
return uuid;
}
if (dump.starts_with("\'"))
{
String str;
@ -410,45 +408,45 @@ Field Field::restoreFromDump(const std::string_view & dump_)
template <typename T>
static bool decEqual(T x, T y, UInt32 x_scale, UInt32 y_scale)
bool decimalEqual(T x, T y, UInt32 x_scale, UInt32 y_scale)
{
using Comparator = DecimalComparison<T, T, EqualsOp>;
return Comparator::compare(x, y, x_scale, y_scale);
}
template <typename T>
static bool decLess(T x, T y, UInt32 x_scale, UInt32 y_scale)
bool decimalLess(T x, T y, UInt32 x_scale, UInt32 y_scale)
{
using Comparator = DecimalComparison<T, T, LessOp>;
return Comparator::compare(x, y, x_scale, y_scale);
}
template <typename T>
static bool decLessOrEqual(T x, T y, UInt32 x_scale, UInt32 y_scale)
bool decimalLessOrEqual(T x, T y, UInt32 x_scale, UInt32 y_scale)
{
using Comparator = DecimalComparison<T, T, LessOrEqualsOp>;
return Comparator::compare(x, y, x_scale, y_scale);
}
template <> bool decimalEqual(Decimal32 x, Decimal32 y, UInt32 x_scale, UInt32 y_scale) { return decEqual(x, y, x_scale, y_scale); }
template <> bool decimalLess(Decimal32 x, Decimal32 y, UInt32 x_scale, UInt32 y_scale) { return decLess(x, y, x_scale, y_scale); }
template <> bool decimalLessOrEqual(Decimal32 x, Decimal32 y, UInt32 x_scale, UInt32 y_scale) { return decLessOrEqual(x, y, x_scale, y_scale); }
template <> bool decimalEqual(Decimal64 x, Decimal64 y, UInt32 x_scale, UInt32 y_scale) { return decEqual(x, y, x_scale, y_scale); }
template <> bool decimalLess(Decimal64 x, Decimal64 y, UInt32 x_scale, UInt32 y_scale) { return decLess(x, y, x_scale, y_scale); }
template <> bool decimalLessOrEqual(Decimal64 x, Decimal64 y, UInt32 x_scale, UInt32 y_scale) { return decLessOrEqual(x, y, x_scale, y_scale); }
template bool decimalEqual<Decimal32>(Decimal32 x, Decimal32 y, UInt32 x_scale, UInt32 y_scale);
template bool decimalEqual<Decimal64>(Decimal64 x, Decimal64 y, UInt32 x_scale, UInt32 y_scale);
template bool decimalEqual<Decimal128>(Decimal128 x, Decimal128 y, UInt32 x_scale, UInt32 y_scale);
template bool decimalEqual<Decimal256>(Decimal256 x, Decimal256 y, UInt32 x_scale, UInt32 y_scale);
template bool decimalEqual<DateTime64>(DateTime64 x, DateTime64 y, UInt32 x_scale, UInt32 y_scale);
template <> bool decimalEqual(Decimal128 x, Decimal128 y, UInt32 x_scale, UInt32 y_scale) { return decEqual(x, y, x_scale, y_scale); }
template <> bool decimalLess(Decimal128 x, Decimal128 y, UInt32 x_scale, UInt32 y_scale) { return decLess(x, y, x_scale, y_scale); }
template <> bool decimalLessOrEqual(Decimal128 x, Decimal128 y, UInt32 x_scale, UInt32 y_scale) { return decLessOrEqual(x, y, x_scale, y_scale); }
template bool decimalLess<Decimal32>(Decimal32 x, Decimal32 y, UInt32 x_scale, UInt32 y_scale);
template bool decimalLess<Decimal64>(Decimal64 x, Decimal64 y, UInt32 x_scale, UInt32 y_scale);
template bool decimalLess<Decimal128>(Decimal128 x, Decimal128 y, UInt32 x_scale, UInt32 y_scale);
template bool decimalLess<Decimal256>(Decimal256 x, Decimal256 y, UInt32 x_scale, UInt32 y_scale);
template bool decimalLess<DateTime64>(DateTime64 x, DateTime64 y, UInt32 x_scale, UInt32 y_scale);
template <> bool decimalEqual(Decimal256 x, Decimal256 y, UInt32 x_scale, UInt32 y_scale) { return decEqual(x, y, x_scale, y_scale); }
template <> bool decimalLess(Decimal256 x, Decimal256 y, UInt32 x_scale, UInt32 y_scale) { return decLess(x, y, x_scale, y_scale); }
template <> bool decimalLessOrEqual(Decimal256 x, Decimal256 y, UInt32 x_scale, UInt32 y_scale) { return decLessOrEqual(x, y, x_scale, y_scale); }
template bool decimalLessOrEqual<Decimal32>(Decimal32 x, Decimal32 y, UInt32 x_scale, UInt32 y_scale);
template bool decimalLessOrEqual<Decimal64>(Decimal64 x, Decimal64 y, UInt32 x_scale, UInt32 y_scale);
template bool decimalLessOrEqual<Decimal128>(Decimal128 x, Decimal128 y, UInt32 x_scale, UInt32 y_scale);
template bool decimalLessOrEqual<Decimal256>(Decimal256 x, Decimal256 y, UInt32 x_scale, UInt32 y_scale);
template bool decimalLessOrEqual<DateTime64>(DateTime64 x, DateTime64 y, UInt32 x_scale, UInt32 y_scale);
template <> bool decimalEqual(DateTime64 x, DateTime64 y, UInt32 x_scale, UInt32 y_scale) { return decEqual(x, y, x_scale, y_scale); }
template <> bool decimalLess(DateTime64 x, DateTime64 y, UInt32 x_scale, UInt32 y_scale) { return decLess(x, y, x_scale, y_scale); }
template <> bool decimalLessOrEqual(DateTime64 x, DateTime64 y, UInt32 x_scale, UInt32 y_scale) { return decLessOrEqual(x, y, x_scale, y_scale); }
inline void writeText(const Null &, WriteBuffer & buf)
{

View File

@ -7,7 +7,6 @@
#include <functional>
#include <Common/Exception.h>
#include <Common/UInt128.h>
#include <Common/AllocatorWithMemoryTracking.h>
#include <Core/Types.h>
#include <Core/Defines.h>
@ -177,8 +176,7 @@ template <> struct NearestFieldTypeImpl<UInt16> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<UInt32> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<DayNum> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<UInt128> { using Type = UInt128; };
template <> struct NearestFieldTypeImpl<UUID> { using Type = UInt128; };
template <> struct NearestFieldTypeImpl<UUID> { using Type = UUID; };
template <> struct NearestFieldTypeImpl<Int16> { using Type = Int64; };
template <> struct NearestFieldTypeImpl<Int32> { using Type = Int64; };
@ -191,8 +189,9 @@ template <> struct NearestFieldTypeImpl<unsigned long long> { using Type = UInt6
template <> struct NearestFieldTypeImpl<UInt256> { using Type = UInt256; };
template <> struct NearestFieldTypeImpl<Int256> { using Type = Int256; };
template <> struct NearestFieldTypeImpl<UInt128> { using Type = UInt128; };
template <> struct NearestFieldTypeImpl<Int128> { using Type = Int128; };
template <> struct NearestFieldTypeImpl<Decimal32> { using Type = DecimalField<Decimal32>; };
template <> struct NearestFieldTypeImpl<Decimal64> { using Type = DecimalField<Decimal64>; };
template <> struct NearestFieldTypeImpl<Decimal128> { using Type = DecimalField<Decimal128>; };
@ -235,7 +234,7 @@ struct NearestFieldTypeImpl<T, std::enable_if_t<std::is_enum_v<T>>>
* but somewhat more efficient, and simpler.
*
* Used to represent a single value of one of several types in memory.
* Warning! Prefer to use chunks of columns instead of single values. See Column.h
* Warning! Prefer to use chunks of columns instead of single values. See IColumn.h
*/
class Field
{
@ -252,8 +251,6 @@ public:
UInt128 = 4,
Int128 = 5,
/// Non-POD types.
String = 16,
Array = 17,
Tuple = 18,
@ -265,10 +262,9 @@ public:
UInt256 = 24,
Int256 = 25,
Map = 26,
UUID = 27,
};
static const int MIN_NON_POD = 16;
static const char * toString(Which which)
{
switch (which)
@ -276,8 +272,11 @@ public:
case Null: return "Null";
case UInt64: return "UInt64";
case UInt128: return "UInt128";
case UInt256: return "UInt256";
case Int64: return "Int64";
case Int128: return "Int128";
case Int256: return "Int256";
case UUID: return "UUID";
case Float64: return "Float64";
case String: return "String";
case Array: return "Array";
@ -288,8 +287,6 @@ public:
case Decimal128: return "Decimal128";
case Decimal256: return "Decimal256";
case AggregateFunctionState: return "AggregateFunctionState";
case UInt256: return "UInt256";
case Int256: return "Int256";
}
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
@ -303,12 +300,17 @@ public:
static bool IsDecimal(Types::Which which)
{
return (which >= Types::Decimal32 && which <= Types::Decimal128) || which == Types::Decimal256;
return which == Types::Decimal32
|| which == Types::Decimal64
|| which == Types::Decimal128
|| which == Types::Decimal256;
}
/// Templates to avoid ambiguity.
template <typename T, typename Z = void *>
using enable_if_not_field_or_stringlike_t = std::enable_if_t<!std::is_same_v<std::decay_t<T>, Field> && !std::is_same_v<NearestFieldType<std::decay_t<T>>, String>, Z>;
using enable_if_not_field_or_stringlike_t = std::enable_if_t<
!std::is_same_v<std::decay_t<T>, Field>
&& !std::is_same_v<NearestFieldType<std::decay_t<T>>, String>, Z>;
Field() //-V730
: which(Types::Null)
@ -382,10 +384,10 @@ public:
enable_if_not_field_or_stringlike_t<T, Field> &
operator=(T && rhs);
Field & operator =(const std::string_view & str);
Field & operator =(const String & str) { return *this = std::string_view{str}; }
Field & operator =(String && str);
Field & operator =(const char * str) { return *this = std::string_view{str}; }
Field & operator= (const std::string_view & str);
Field & operator= (const String & str) { return *this = std::string_view{str}; }
Field & operator= (String && str);
Field & operator= (const char * str) { return *this = std::string_view{str}; }
~Field()
{
@ -454,8 +456,11 @@ public:
case Types::Null: return false;
case Types::UInt64: return get<UInt64>() < rhs.get<UInt64>();
case Types::UInt128: return get<UInt128>() < rhs.get<UInt128>();
case Types::UInt256: return get<UInt256>() < rhs.get<UInt256>();
case Types::Int64: return get<Int64>() < rhs.get<Int64>();
case Types::Int128: return get<Int128>() < rhs.get<Int128>();
case Types::Int256: return get<Int256>() < rhs.get<Int256>();
case Types::UUID: return get<UUID>() < rhs.get<UUID>();
case Types::Float64: return get<Float64>() < rhs.get<Float64>();
case Types::String: return get<String>() < rhs.get<String>();
case Types::Array: return get<Array>() < rhs.get<Array>();
@ -466,8 +471,6 @@ public:
case Types::Decimal128: return get<DecimalField<Decimal128>>() < rhs.get<DecimalField<Decimal128>>();
case Types::Decimal256: return get<DecimalField<Decimal256>>() < rhs.get<DecimalField<Decimal256>>();
case Types::AggregateFunctionState: return get<AggregateFunctionStateData>() < rhs.get<AggregateFunctionStateData>();
case Types::UInt256: return get<UInt256>() < rhs.get<UInt256>();
case Types::Int256: return get<Int256>() < rhs.get<Int256>();
}
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
@ -490,8 +493,11 @@ public:
case Types::Null: return true;
case Types::UInt64: return get<UInt64>() <= rhs.get<UInt64>();
case Types::UInt128: return get<UInt128>() <= rhs.get<UInt128>();
case Types::UInt256: return get<UInt256>() <= rhs.get<UInt256>();
case Types::Int64: return get<Int64>() <= rhs.get<Int64>();
case Types::Int128: return get<Int128>() <= rhs.get<Int128>();
case Types::Int256: return get<Int256>() <= rhs.get<Int256>();
case Types::UUID: return get<UUID>().toUnderType() <= rhs.get<UUID>().toUnderType();
case Types::Float64: return get<Float64>() <= rhs.get<Float64>();
case Types::String: return get<String>() <= rhs.get<String>();
case Types::Array: return get<Array>() <= rhs.get<Array>();
@ -502,8 +508,6 @@ public:
case Types::Decimal128: return get<DecimalField<Decimal128>>() <= rhs.get<DecimalField<Decimal128>>();
case Types::Decimal256: return get<DecimalField<Decimal256>>() <= rhs.get<DecimalField<Decimal256>>();
case Types::AggregateFunctionState: return get<AggregateFunctionStateData>() <= rhs.get<AggregateFunctionStateData>();
case Types::UInt256: return get<UInt256>() <= rhs.get<UInt256>();
case Types::Int256: return get<Int256>() <= rhs.get<Int256>();
}
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
@ -531,19 +535,20 @@ public:
// Compare as UInt64 so that NaNs compare as equal.
return reinterpret<UInt64>() == rhs.reinterpret<UInt64>();
}
case Types::UUID: return get<UUID>() == rhs.get<UUID>();
case Types::String: return get<String>() == rhs.get<String>();
case Types::Array: return get<Array>() == rhs.get<Array>();
case Types::Tuple: return get<Tuple>() == rhs.get<Tuple>();
case Types::Map: return get<Map>() == rhs.get<Map>();
case Types::UInt128: return get<UInt128>() == rhs.get<UInt128>();
case Types::UInt256: return get<UInt256>() == rhs.get<UInt256>();
case Types::Int128: return get<Int128>() == rhs.get<Int128>();
case Types::Int256: return get<Int256>() == rhs.get<Int256>();
case Types::Decimal32: return get<DecimalField<Decimal32>>() == rhs.get<DecimalField<Decimal32>>();
case Types::Decimal64: return get<DecimalField<Decimal64>>() == rhs.get<DecimalField<Decimal64>>();
case Types::Decimal128: return get<DecimalField<Decimal128>>() == rhs.get<DecimalField<Decimal128>>();
case Types::Decimal256: return get<DecimalField<Decimal256>>() == rhs.get<DecimalField<Decimal256>>();
case Types::AggregateFunctionState: return get<AggregateFunctionStateData>() == rhs.get<AggregateFunctionStateData>();
case Types::UInt256: return get<UInt256>() == rhs.get<UInt256>();
case Types::Int256: return get<Int256>() == rhs.get<Int256>();
}
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
@ -569,7 +574,11 @@ public:
#endif
case Types::UInt64: return f(field.template get<UInt64>());
case Types::UInt128: return f(field.template get<UInt128>());
case Types::UInt256: return f(field.template get<UInt256>());
case Types::Int64: return f(field.template get<Int64>());
case Types::Int128: return f(field.template get<Int128>());
case Types::Int256: return f(field.template get<Int256>());
case Types::UUID: return f(field.template get<UUID>());
case Types::Float64: return f(field.template get<Float64>());
case Types::String: return f(field.template get<String>());
case Types::Array: return f(field.template get<Array>());
@ -580,19 +589,12 @@ public:
case Types::Decimal128: return f(field.template get<DecimalField<Decimal128>>());
case Types::Decimal256: return f(field.template get<DecimalField<Decimal256>>());
case Types::AggregateFunctionState: return f(field.template get<AggregateFunctionStateData>());
case Types::Int128: return f(field.template get<Int128>());
case Types::UInt256: return f(field.template get<UInt256>());
case Types::Int256: return f(field.template get<Int256>());
#if !defined(__clang__)
#pragma GCC diagnostic pop
#endif
}
// GCC 9 complains that control reaches the end, despite that we handle
// all the cases above (maybe because of throw?). Return something to
// silence it.
Null null{};
return f(null);
__builtin_unreachable();
}
String dump() const;
@ -600,10 +602,9 @@ public:
private:
std::aligned_union_t<DBMS_MIN_FIELD_SIZE - sizeof(Types::Which),
Null, UInt64, UInt128, Int64, Int128, Float64, String, Array, Tuple, Map,
Null, UInt64, UInt128, UInt256, Int64, Int128, Int256, UUID, Float64, String, Array, Tuple, Map,
DecimalField<Decimal32>, DecimalField<Decimal64>, DecimalField<Decimal128>, DecimalField<Decimal256>,
AggregateFunctionStateData,
UInt256, Int256
AggregateFunctionStateData
> storage;
Types::Which which;
@ -685,9 +686,6 @@ private:
ALWAYS_INLINE void destroy()
{
if (which < Types::MIN_NON_POD)
return;
switch (which)
{
case Types::String:
@ -726,8 +724,11 @@ private:
template <> struct Field::TypeToEnum<Null> { static const Types::Which value = Types::Null; };
template <> struct Field::TypeToEnum<UInt64> { static const Types::Which value = Types::UInt64; };
template <> struct Field::TypeToEnum<UInt128> { static const Types::Which value = Types::UInt128; };
template <> struct Field::TypeToEnum<UInt256> { static const Types::Which value = Types::UInt256; };
template <> struct Field::TypeToEnum<Int64> { static const Types::Which value = Types::Int64; };
template <> struct Field::TypeToEnum<Int128> { static const Types::Which value = Types::Int128; };
template <> struct Field::TypeToEnum<Int256> { static const Types::Which value = Types::Int256; };
template <> struct Field::TypeToEnum<UUID> { static const Types::Which value = Types::UUID; };
template <> struct Field::TypeToEnum<Float64> { static const Types::Which value = Types::Float64; };
template <> struct Field::TypeToEnum<String> { static const Types::Which value = Types::String; };
template <> struct Field::TypeToEnum<Array> { static const Types::Which value = Types::Array; };
@ -739,14 +740,15 @@ template <> struct Field::TypeToEnum<DecimalField<Decimal128>>{ static const Typ
template <> struct Field::TypeToEnum<DecimalField<Decimal256>>{ static const Types::Which value = Types::Decimal256; };
template <> struct Field::TypeToEnum<DecimalField<DateTime64>>{ static const Types::Which value = Types::Decimal64; };
template <> struct Field::TypeToEnum<AggregateFunctionStateData>{ static const Types::Which value = Types::AggregateFunctionState; };
template <> struct Field::TypeToEnum<UInt256> { static const Types::Which value = Types::UInt256; };
template <> struct Field::TypeToEnum<Int256> { static const Types::Which value = Types::Int256; };
template <> struct Field::EnumToType<Field::Types::Null> { using Type = Null; };
template <> struct Field::EnumToType<Field::Types::UInt64> { using Type = UInt64; };
template <> struct Field::EnumToType<Field::Types::UInt128> { using Type = UInt128; };
template <> struct Field::EnumToType<Field::Types::UInt256> { using Type = UInt256; };
template <> struct Field::EnumToType<Field::Types::Int64> { using Type = Int64; };
template <> struct Field::EnumToType<Field::Types::Int128> { using Type = Int128; };
template <> struct Field::EnumToType<Field::Types::Int256> { using Type = Int256; };
template <> struct Field::EnumToType<Field::Types::UUID> { using Type = UUID; };
template <> struct Field::EnumToType<Field::Types::Float64> { using Type = Float64; };
template <> struct Field::EnumToType<Field::Types::String> { using Type = String; };
template <> struct Field::EnumToType<Field::Types::Array> { using Type = Array; };
@ -757,10 +759,8 @@ template <> struct Field::EnumToType<Field::Types::Decimal64> { using Type = Dec
template <> struct Field::EnumToType<Field::Types::Decimal128> { using Type = DecimalField<Decimal128>; };
template <> struct Field::EnumToType<Field::Types::Decimal256> { using Type = DecimalField<Decimal256>; };
template <> struct Field::EnumToType<Field::Types::AggregateFunctionState> { using Type = DecimalField<AggregateFunctionStateData>; };
template <> struct Field::EnumToType<Field::Types::UInt256> { using Type = UInt256; };
template <> struct Field::EnumToType<Field::Types::Int256> { using Type = Int256; };
inline constexpr bool isInt64FieldType(Field::Types::Which t)
inline constexpr bool isInt64OrUInt64FieldType(Field::Types::Which t)
{
return t == Field::Types::Int64
|| t == Field::Types::UInt64;
@ -782,7 +782,7 @@ NearestFieldType<std::decay_t<T>> & Field::get()
// Disregard signedness when converting between int64 types.
constexpr Field::Types::Which target = TypeToEnum<StoredType>::value;
if (target != which
&& (!isInt64FieldType(target) || !isInt64FieldType(which)))
&& (!isInt64OrUInt64FieldType(target) || !isInt64OrUInt64FieldType(which)))
throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid Field get from type {} to type {}", Types::toString(which), Types::toString(target));
#endif
@ -834,11 +834,11 @@ T safeGet(Field & field)
return field.template safeGet<T>();
}
template <> inline constexpr const char * TypeName<Array> = "Array";
template <> inline constexpr const char * TypeName<Tuple> = "Tuple";
template <> inline constexpr const char * TypeName<Map> = "Map";
template <> inline constexpr const char * TypeName<AggregateFunctionStateData> = "AggregateFunctionState";
template <> struct TypeName<Array> { static std::string get() { return "Array"; } };
template <> struct TypeName<Tuple> { static std::string get() { return "Tuple"; } };
template <> struct TypeName<Map> { static std::string get() { return "Map"; } };
template <> struct TypeName<AggregateFunctionStateData> { static std::string get() { return "AggregateFunctionState"; } };
template <typename T>
decltype(auto) castToNearestFieldType(T && x)
@ -965,7 +965,7 @@ struct fmt::formatter<DB::Field>
/// Only support {}.
if (it != end && *it != '}')
throw format_error("invalid format");
throw format_error("Invalid format");
return it;
}

View File

@ -1,6 +1,7 @@
#include "MySQLGtid.h"
#include <boost/algorithm/string.hpp>
#include <IO/ReadHelpers.h>
namespace DB
{
@ -38,7 +39,7 @@ void GTIDSets::parse(const String gtid_format)
boost::split(server_ids, gset, [](char c) { return c == ':'; });
GTIDSet set;
set.uuid = stringToUUID(server_ids[0]);
set.uuid = DB::parse<UUID>(server_ids[0]);
for (size_t k = 1; k < server_ids.size(); k++)
{
@ -174,8 +175,8 @@ String GTIDSets::toPayload() const
for (const auto & set : sets)
{
// MySQL UUID is big-endian.
writeBinaryBigEndian(set.uuid.toUnderType().low, buffer);
writeBinaryBigEndian(set.uuid.toUnderType().high, buffer);
writeBinaryBigEndian(set.uuid.toUnderType().items[0], buffer);
writeBinaryBigEndian(set.uuid.toUnderType().items[1], buffer);
UInt64 intervals_size = set.intervals.size();
buffer.write(reinterpret_cast<const char *>(&intervals_size), 8);

View File

@ -658,12 +658,13 @@ namespace MySQLReplication
payload.readStrict(reinterpret_cast<char *>(&commit_flag), 1);
// MySQL UUID is big-endian.
UInt64 high = 0UL, low = 0UL;
UInt64 high = 0UL;
UInt64 low = 0UL;
readBigEndianStrict(payload, reinterpret_cast<char *>(&low), 8);
gtid.uuid.toUnderType().low = low;
gtid.uuid.toUnderType().items[0] = low;
readBigEndianStrict(payload, reinterpret_cast<char *>(&high), 8);
gtid.uuid.toUnderType().high = high;
gtid.uuid.toUnderType().items[1] = high;
payload.readStrict(reinterpret_cast<char *>(&gtid.seq_no), 8);

View File

@ -1,20 +1,18 @@
#pragma once
#include <Core/Types.h>
#include <Common/UInt128.h>
#include <Common/TypeList.h>
namespace DB
{
using TypeListNativeNumbers = TypeList<UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64>;
using TypeListExtendedNumbers = TypeList<Int128, UInt256, Int256>;
using TypeListExtendedNumbers = TypeList<UInt128, Int128, UInt256, Int256>;
using TypeListDecimalNumbers = TypeList<Decimal32, Decimal64, Decimal128, Decimal256>;
using TypeListGeneralNumbers = typename TypeListConcat<TypeListNativeNumbers, TypeListExtendedNumbers>::Type;
using TypeListNumbers = typename TypeListConcat<TypeListGeneralNumbers, TypeListDecimalNumbers>::Type;
/// Currently separate because UInt128 cannot be used in every context where other numbers can be used.
using TypeListNumbersAndUInt128 = typename AppendToTypeList<UInt128, TypeListNumbers>::Type;
using TypeListNumbersAndUUID = typename TypeListConcat<TypeListNumbers, TypeList<UUID>>::Type;
}

View File

@ -3,6 +3,7 @@
#include <cstdint>
#include <string>
#include <vector>
#include <common/strong_typedef.h>
#include <common/extended_types.h>
#include <common/defines.h>
@ -63,60 +64,52 @@ enum class TypeIndex
#pragma GCC diagnostic pop
#endif
/// Other int defines are in common/types.h
using UInt256 = ::wUInt256;
using UInt128 = ::UInt128;
using UInt256 = ::UInt256;
using Int128 = ::Int128;
using Int256 = ::wInt256;
using Int256 = ::Int256;
/** Note that for types not used in DB, IsNumber is false.
*/
template <typename T> constexpr bool IsNumber = false;
STRONG_TYPEDEF(UInt128, UUID)
template <> inline constexpr bool IsNumber<UInt8> = true;
template <> inline constexpr bool IsNumber<UInt16> = true;
template <> inline constexpr bool IsNumber<UInt32> = true;
template <> inline constexpr bool IsNumber<UInt64> = true;
template <> inline constexpr bool IsNumber<UInt256> = true;
template <> inline constexpr bool IsNumber<Int8> = true;
template <> inline constexpr bool IsNumber<Int16> = true;
template <> inline constexpr bool IsNumber<Int32> = true;
template <> inline constexpr bool IsNumber<Int64> = true;
template <> inline constexpr bool IsNumber<Int128> = true;
template <> inline constexpr bool IsNumber<Int256> = true;
template <> inline constexpr bool IsNumber<Float32> = true;
template <> inline constexpr bool IsNumber<Float64> = true;
template <typename T> struct TypeName;
template <typename T> constexpr const char * TypeName = "";
template <> struct TypeName<UInt8> { static constexpr const char * get() { return "UInt8"; } };
template <> struct TypeName<UInt16> { static constexpr const char * get() { return "UInt16"; } };
template <> struct TypeName<UInt32> { static constexpr const char * get() { return "UInt32"; } };
template <> struct TypeName<UInt64> { static constexpr const char * get() { return "UInt64"; } };
template <> struct TypeName<UInt256> { static constexpr const char * get() { return "UInt256"; } };
template <> struct TypeName<Int8> { static constexpr const char * get() { return "Int8"; } };
template <> struct TypeName<Int16> { static constexpr const char * get() { return "Int16"; } };
template <> struct TypeName<Int32> { static constexpr const char * get() { return "Int32"; } };
template <> struct TypeName<Int64> { static constexpr const char * get() { return "Int64"; } };
template <> struct TypeName<Int128> { static constexpr const char * get() { return "Int128"; } };
template <> struct TypeName<Int256> { static constexpr const char * get() { return "Int256"; } };
template <> struct TypeName<Float32> { static constexpr const char * get() { return "Float32"; } };
template <> struct TypeName<Float64> { static constexpr const char * get() { return "Float64"; } };
template <> struct TypeName<String> { static constexpr const char * get() { return "String"; } };
template <> inline constexpr const char * TypeName<UInt8> = "UInt8";
template <> inline constexpr const char * TypeName<UInt16> = "UInt16";
template <> inline constexpr const char * TypeName<UInt32> = "UInt32";
template <> inline constexpr const char * TypeName<UInt64> = "UInt64";
template <> inline constexpr const char * TypeName<UInt128> = "UInt128";
template <> inline constexpr const char * TypeName<UInt256> = "UInt256";
template <> inline constexpr const char * TypeName<Int8> = "Int8";
template <> inline constexpr const char * TypeName<Int16> = "Int16";
template <> inline constexpr const char * TypeName<Int32> = "Int32";
template <> inline constexpr const char * TypeName<Int64> = "Int64";
template <> inline constexpr const char * TypeName<Int128> = "Int128";
template <> inline constexpr const char * TypeName<Int256> = "Int256";
template <> inline constexpr const char * TypeName<Float32> = "Float32";
template <> inline constexpr const char * TypeName<Float64> = "Float64";
template <> inline constexpr const char * TypeName<String> = "String";
template <> inline constexpr const char * TypeName<UUID> = "UUID";
/// TODO Try to remove it.
template <typename T> constexpr TypeIndex TypeId = TypeIndex::Nothing;
template <> inline constexpr TypeIndex TypeId<UInt8> = TypeIndex::UInt8;
template <> inline constexpr TypeIndex TypeId<UInt16> = TypeIndex::UInt16;
template <> inline constexpr TypeIndex TypeId<UInt32> = TypeIndex::UInt32;
template <> inline constexpr TypeIndex TypeId<UInt64> = TypeIndex::UInt64;
template <> inline constexpr TypeIndex TypeId<UInt128> = TypeIndex::UInt128;
template <> inline constexpr TypeIndex TypeId<UInt256> = TypeIndex::UInt256;
template <> inline constexpr TypeIndex TypeId<Int8> = TypeIndex::Int8;
template <> inline constexpr TypeIndex TypeId<Int16> = TypeIndex::Int16;
template <> inline constexpr TypeIndex TypeId<Int32> = TypeIndex::Int32;
template <> inline constexpr TypeIndex TypeId<Int64> = TypeIndex::Int64;
template <> inline constexpr TypeIndex TypeId<Int128> = TypeIndex::Int128;
template <> inline constexpr TypeIndex TypeId<Int256> = TypeIndex::Int256;
template <> inline constexpr TypeIndex TypeId<Float32> = TypeIndex::Float32;
template <> inline constexpr TypeIndex TypeId<Float64> = TypeIndex::Float64;
template <> inline constexpr TypeIndex TypeId<UUID> = TypeIndex::UUID;
template <typename T> struct TypeId;
template <> struct TypeId<UInt8> { static constexpr const TypeIndex value = TypeIndex::UInt8; };
template <> struct TypeId<UInt16> { static constexpr const TypeIndex value = TypeIndex::UInt16; };
template <> struct TypeId<UInt32> { static constexpr const TypeIndex value = TypeIndex::UInt32; };
template <> struct TypeId<UInt64> { static constexpr const TypeIndex value = TypeIndex::UInt64; };
template <> struct TypeId<UInt256> { static constexpr const TypeIndex value = TypeIndex::UInt256; };
template <> struct TypeId<Int8> { static constexpr const TypeIndex value = TypeIndex::Int8; };
template <> struct TypeId<Int16> { static constexpr const TypeIndex value = TypeIndex::Int16; };
template <> struct TypeId<Int32> { static constexpr const TypeIndex value = TypeIndex::Int32; };
template <> struct TypeId<Int64> { static constexpr const TypeIndex value = TypeIndex::Int64; };
template <> struct TypeId<Int128> { static constexpr const TypeIndex value = TypeIndex::Int128; };
template <> struct TypeId<Int256> { static constexpr const TypeIndex value = TypeIndex::Int256; };
template <> struct TypeId<Float32> { static constexpr const TypeIndex value = TypeIndex::Float32; };
template <> struct TypeId<Float64> { static constexpr const TypeIndex value = TypeIndex::Float64; };
/// Not a data type in database, defined just for convenience.
using Strings = std::vector<String>;
@ -167,6 +160,12 @@ struct Decimal
const Decimal<T> & operator /= (const T & x) { value /= x; return *this; }
const Decimal<T> & operator %= (const T & x) { value %= x; return *this; }
template <typename U> const Decimal<T> & operator += (const Decimal<U> & x) { value += x.value; return *this; }
template <typename U> const Decimal<T> & operator -= (const Decimal<U> & x) { value -= x.value; return *this; }
template <typename U> const Decimal<T> & operator *= (const Decimal<U> & x) { value *= x.value; return *this; }
template <typename U> const Decimal<T> & operator /= (const Decimal<U> & x) { value /= x.value; return *this; }
template <typename U> const Decimal<T> & operator %= (const Decimal<U> & x) { value %= x.value; return *this; }
/// This is to avoid UB for sumWithOverflow()
void NO_SANITIZE_UNDEFINED addOverflow(const T & x) { value += x; }
@ -175,6 +174,8 @@ struct Decimal
template <typename T> inline bool operator< (const Decimal<T> & x, const Decimal<T> & y) { return x.value < y.value; }
template <typename T> inline bool operator> (const Decimal<T> & x, const Decimal<T> & y) { return x.value > y.value; }
template <typename T> inline bool operator<= (const Decimal<T> & x, const Decimal<T> & y) { return x.value <= y.value; }
template <typename T> inline bool operator>= (const Decimal<T> & x, const Decimal<T> & y) { return x.value >= y.value; }
template <typename T> inline bool operator== (const Decimal<T> & x, const Decimal<T> & y) { return x.value == y.value; }
template <typename T> inline bool operator!= (const Decimal<T> & x, const Decimal<T> & y) { return x.value != y.value; }
@ -202,17 +203,17 @@ public:
{}
};
template <> struct TypeName<Decimal32> { static constexpr const char * get() { return "Decimal32"; } };
template <> struct TypeName<Decimal64> { static constexpr const char * get() { return "Decimal64"; } };
template <> struct TypeName<Decimal128> { static constexpr const char * get() { return "Decimal128"; } };
template <> struct TypeName<Decimal256> { static constexpr const char * get() { return "Decimal256"; } };
template <> struct TypeName<DateTime64> { static constexpr const char * get() { return "DateTime64"; } };
template <> inline constexpr const char * TypeName<Decimal32> = "Decimal32";
template <> inline constexpr const char * TypeName<Decimal64> = "Decimal64";
template <> inline constexpr const char * TypeName<Decimal128> = "Decimal128";
template <> inline constexpr const char * TypeName<Decimal256> = "Decimal256";
template <> inline constexpr const char * TypeName<DateTime64> = "DateTime64";
template <> struct TypeId<Decimal32> { static constexpr const TypeIndex value = TypeIndex::Decimal32; };
template <> struct TypeId<Decimal64> { static constexpr const TypeIndex value = TypeIndex::Decimal64; };
template <> struct TypeId<Decimal128> { static constexpr const TypeIndex value = TypeIndex::Decimal128; };
template <> struct TypeId<Decimal256> { static constexpr const TypeIndex value = TypeIndex::Decimal256; };
template <> struct TypeId<DateTime64> { static constexpr const TypeIndex value = TypeIndex::DateTime64; };
template <> inline constexpr TypeIndex TypeId<Decimal32> = TypeIndex::Decimal32;
template <> inline constexpr TypeIndex TypeId<Decimal64> = TypeIndex::Decimal64;
template <> inline constexpr TypeIndex TypeId<Decimal128> = TypeIndex::Decimal128;
template <> inline constexpr TypeIndex TypeId<Decimal256> = TypeIndex::Decimal256;
template <> inline constexpr TypeIndex TypeId<DateTime64> = TypeIndex::DateTime64;
template <typename T> constexpr bool IsDecimalNumber = false;
template <> inline constexpr bool IsDecimalNumber<Decimal32> = true;
@ -229,8 +230,11 @@ template <> struct NativeType<Decimal256> { using Type = Int256; };
template <> struct NativeType<DateTime64> { using Type = Int64; };
template <typename T> constexpr bool OverBigInt = false;
template <> inline constexpr bool OverBigInt<Int128> = true;
template <> inline constexpr bool OverBigInt<UInt128> = true;
template <> inline constexpr bool OverBigInt<Int256> = true;
template <> inline constexpr bool OverBigInt<UInt256> = true;
template <> inline constexpr bool OverBigInt<Decimal128> = true;
template <> inline constexpr bool OverBigInt<Decimal256> = true;
inline constexpr const char * getTypeName(TypeIndex idx)
@ -238,31 +242,31 @@ inline constexpr const char * getTypeName(TypeIndex idx)
switch (idx)
{
case TypeIndex::Nothing: return "Nothing";
case TypeIndex::UInt8: return TypeName<UInt8>::get();
case TypeIndex::UInt16: return TypeName<UInt16>::get();
case TypeIndex::UInt32: return TypeName<UInt32>::get();
case TypeIndex::UInt64: return TypeName<UInt64>::get();
case TypeIndex::UInt8: return "UInt8";
case TypeIndex::UInt16: return "UInt16";
case TypeIndex::UInt32: return "UInt32";
case TypeIndex::UInt64: return "UInt64";
case TypeIndex::UInt128: return "UInt128";
case TypeIndex::UInt256: return TypeName<UInt256>::get();
case TypeIndex::Int8: return TypeName<Int8>::get();
case TypeIndex::Int16: return TypeName<Int16>::get();
case TypeIndex::Int32: return TypeName<Int32>::get();
case TypeIndex::Int64: return TypeName<Int64>::get();
case TypeIndex::Int128: return TypeName<Int128>::get();
case TypeIndex::Int256: return TypeName<Int256>::get();
case TypeIndex::Float32: return TypeName<Float32>::get();
case TypeIndex::Float64: return TypeName<Float64>::get();
case TypeIndex::UInt256: return "UInt256";
case TypeIndex::Int8: return "Int8";
case TypeIndex::Int16: return "Int16";
case TypeIndex::Int32: return "Int32";
case TypeIndex::Int64: return "Int64";
case TypeIndex::Int128: return "Int128";
case TypeIndex::Int256: return "Int256";
case TypeIndex::Float32: return "Float32";
case TypeIndex::Float64: return "Float64";
case TypeIndex::Date: return "Date";
case TypeIndex::DateTime: return "DateTime";
case TypeIndex::DateTime64: return "DateTime64";
case TypeIndex::String: return TypeName<String>::get();
case TypeIndex::String: return "String";
case TypeIndex::FixedString: return "FixedString";
case TypeIndex::Enum8: return "Enum8";
case TypeIndex::Enum16: return "Enum16";
case TypeIndex::Decimal32: return TypeName<Decimal32>::get();
case TypeIndex::Decimal64: return TypeName<Decimal64>::get();
case TypeIndex::Decimal128: return TypeName<Decimal128>::get();
case TypeIndex::Decimal256: return TypeName<Decimal256>::get();
case TypeIndex::Decimal32: return "Decimal32";
case TypeIndex::Decimal64: return "Decimal64";
case TypeIndex::Decimal128: return "Decimal128";
case TypeIndex::Decimal256: return "Decimal256";
case TypeIndex::UUID: return "UUID";
case TypeIndex::Array: return "Array";
case TypeIndex::Tuple: return "Tuple";

19
src/Core/UUID.cpp Normal file
View File

@ -0,0 +1,19 @@
#include <Core/UUID.h>
#include <Common/thread_local_rng.h>
namespace DB
{
namespace UUIDHelpers
{
UUID generateV4()
{
UInt128 res{thread_local_rng(), thread_local_rng()};
res.items[0] = (res.items[0] & 0xffffffffffff0fffull) | 0x0000000000004000ull;
res.items[1] = (res.items[1] & 0x3fffffffffffffffull) | 0x8000000000000000ull;
return UUID{res};
}
}
}

View File

@ -1,25 +1,17 @@
#pragma once
#include <Common/UInt128.h>
#include <common/strong_typedef.h>
#include <Common/thread_local_rng.h>
#include <Core/Types.h>
namespace DB
{
STRONG_TYPEDEF(UInt128, UUID)
namespace UUIDHelpers
{
inline UUID generateV4()
{
UInt128 res{thread_local_rng(), thread_local_rng()};
res.low = (res.low & 0xffffffffffff0fffull) | 0x0000000000004000ull;
res.high = (res.high & 0x3fffffffffffffffull) | 0x8000000000000000ull;
return UUID{res};
}
/// Generate random UUID.
UUID generateV4();
const UUID Nil = UUID(UInt128(0, 0));
const UUID Nil{};
}
}

Some files were not shown because too many files have changed in this diff Show More