Merge branch 'master' of github.com:ClickHouse/ClickHouse into poco-file-to-std-fs

This commit is contained in:
kssenii 2021-05-11 16:49:23 +03:00
commit 393b026f2e
526 changed files with 14441 additions and 5113 deletions

View File

@ -169,8 +169,8 @@ endif ()
option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON)
option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF)
if (OS_LINUX AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0")
# Only for Linux, x86_64.
if (OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64) AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0")
# Only for Linux, x86_64 or aarch64.
option(GLIBC_COMPATIBILITY "Enable compatibility with older glibc libraries." ON)
elseif(GLIBC_COMPATIBILITY)
message (${RECONFIGURE_MESSAGE_LEVEL} "Glibc compatibility cannot be enabled in current configuration")

View File

@ -0,0 +1,216 @@
#pragma once
#include <cstdint>
#include <cstddef>
#include <cstring>
#include <common/extended_types.h>
/// Allows to check the internals of IEEE-754 floating point number.
template <typename T> struct FloatTraits;
template <>
struct FloatTraits<float>
{
using UInt = uint32_t;
static constexpr size_t bits = 32;
static constexpr size_t exponent_bits = 8;
static constexpr size_t mantissa_bits = bits - exponent_bits - 1;
};
template <>
struct FloatTraits<double>
{
using UInt = uint64_t;
static constexpr size_t bits = 64;
static constexpr size_t exponent_bits = 11;
static constexpr size_t mantissa_bits = bits - exponent_bits - 1;
};
/// x = sign * (2 ^ normalized_exponent) * (1 + mantissa * 2 ^ -mantissa_bits)
/// x = sign * (2 ^ normalized_exponent + mantissa * 2 ^ (normalized_exponent - mantissa_bits))
template <typename T>
struct DecomposedFloat
{
using Traits = FloatTraits<T>;
DecomposedFloat(T x)
{
memcpy(&x_uint, &x, sizeof(x));
}
typename Traits::UInt x_uint;
bool is_negative() const
{
return x_uint >> (Traits::bits - 1);
}
/// Returns 0 for both +0. and -0.
int sign() const
{
return (exponent() == 0 && mantissa() == 0)
? 0
: (is_negative()
? -1
: 1);
}
uint16_t exponent() const
{
return (x_uint >> (Traits::mantissa_bits)) & (((1ull << (Traits::exponent_bits + 1)) - 1) >> 1);
}
int16_t normalized_exponent() const
{
return int16_t(exponent()) - ((1ull << (Traits::exponent_bits - 1)) - 1);
}
uint64_t mantissa() const
{
return x_uint & ((1ull << Traits::mantissa_bits) - 1);
}
int64_t mantissa_with_sign() const
{
return is_negative() ? -mantissa() : mantissa();
}
/// NOTE Probably floating point instructions can be better.
bool is_integer_in_representable_range() const
{
return x_uint == 0
|| (normalized_exponent() >= 0 /// The number is not less than one
/// The number is inside the range where every integer has exact representation in float
&& normalized_exponent() <= static_cast<int16_t>(Traits::mantissa_bits)
/// After multiplying by 2^exp, the fractional part becomes zero, means the number is integer
&& ((mantissa() & ((1ULL << (Traits::mantissa_bits - normalized_exponent())) - 1)) == 0));
}
/// Compare float with integer of arbitrary width (both signed and unsigned are supported). Assuming two's complement arithmetic.
/// Infinities are compared correctly. NaNs are treat similarly to infinities, so they can be less than all numbers.
/// (note that we need total order)
template <typename Int>
int compare(Int rhs)
{
if (rhs == 0)
return sign();
/// Different signs
if (is_negative() && rhs > 0)
return -1;
if (!is_negative() && rhs < 0)
return 1;
/// Fractional number with magnitude less than one
if (normalized_exponent() < 0)
{
if (!is_negative())
return rhs > 0 ? -1 : 1;
else
return rhs >= 0 ? -1 : 1;
}
/// The case of the most negative integer
if constexpr (is_signed_v<Int>)
{
if (rhs == std::numeric_limits<Int>::lowest())
{
assert(is_negative());
if (normalized_exponent() < static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
return 1;
if (normalized_exponent() > static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
return -1;
if (mantissa() == 0)
return 0;
else
return -1;
}
}
/// Too large number: abs(float) > abs(rhs). Also the case with infinities and NaN.
if (normalized_exponent() >= static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
return is_negative() ? -1 : 1;
using UInt = make_unsigned_t<Int>;
UInt uint_rhs = rhs < 0 ? -rhs : rhs;
/// Smaller octave: abs(rhs) < abs(float)
if (uint_rhs < (static_cast<UInt>(1) << normalized_exponent()))
return is_negative() ? -1 : 1;
/// Larger octave: abs(rhs) > abs(float)
if (normalized_exponent() + 1 < static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>)
&& uint_rhs >= (static_cast<UInt>(1) << (normalized_exponent() + 1)))
return is_negative() ? 1 : -1;
/// The same octave
/// uint_rhs == 2 ^ normalized_exponent + mantissa * 2 ^ (normalized_exponent - mantissa_bits)
bool large_and_always_integer = normalized_exponent() >= static_cast<int16_t>(Traits::mantissa_bits);
typename Traits::UInt a = large_and_always_integer
? mantissa() << (normalized_exponent() - Traits::mantissa_bits)
: mantissa() >> (Traits::mantissa_bits - normalized_exponent());
typename Traits::UInt b = uint_rhs - (static_cast<UInt>(1) << normalized_exponent());
if (a < b)
return is_negative() ? 1 : -1;
if (a > b)
return is_negative() ? -1 : 1;
/// Float has no fractional part means that the numbers are equal.
if (large_and_always_integer || (mantissa() & ((1ULL << (Traits::mantissa_bits - normalized_exponent())) - 1)) == 0)
return 0;
else
/// Float has fractional part means its abs value is larger.
return is_negative() ? -1 : 1;
}
template <typename Int>
bool equals(Int rhs)
{
return compare(rhs) == 0;
}
template <typename Int>
bool notEquals(Int rhs)
{
return compare(rhs) != 0;
}
template <typename Int>
bool less(Int rhs)
{
return compare(rhs) < 0;
}
template <typename Int>
bool greater(Int rhs)
{
return compare(rhs) > 0;
}
template <typename Int>
bool lessOrEquals(Int rhs)
{
return compare(rhs) <= 0;
}
template <typename Int>
bool greaterOrEquals(Int rhs)
{
return compare(rhs) >= 0;
}
};
using DecomposedFloat64 = DecomposedFloat<double>;
using DecomposedFloat32 = DecomposedFloat<float>;

View File

@ -56,27 +56,33 @@ namespace common
}
template <>
inline bool addOverflow(__int128 x, __int128 y, __int128 & res)
inline bool addOverflow(Int128 x, Int128 y, Int128 & res)
{
static constexpr __int128 min_int128 = minInt128();
static constexpr __int128 max_int128 = maxInt128();
res = addIgnoreOverflow(x, y);
return (y > 0 && x > max_int128 - y) || (y < 0 && x < min_int128 - y);
return (y > 0 && x > std::numeric_limits<Int128>::max() - y) ||
(y < 0 && x < std::numeric_limits<Int128>::min() - y);
}
template <>
inline bool addOverflow(wInt256 x, wInt256 y, wInt256 & res)
inline bool addOverflow(UInt128 x, UInt128 y, UInt128 & res)
{
res = addIgnoreOverflow(x, y);
return (y > 0 && x > std::numeric_limits<wInt256>::max() - y) ||
(y < 0 && x < std::numeric_limits<wInt256>::min() - y);
return x > std::numeric_limits<UInt128>::max() - y;
}
template <>
inline bool addOverflow(wUInt256 x, wUInt256 y, wUInt256 & res)
inline bool addOverflow(Int256 x, Int256 y, Int256 & res)
{
res = addIgnoreOverflow(x, y);
return x > std::numeric_limits<wUInt256>::max() - y;
return (y > 0 && x > std::numeric_limits<Int256>::max() - y) ||
(y < 0 && x < std::numeric_limits<Int256>::min() - y);
}
template <>
inline bool addOverflow(UInt256 x, UInt256 y, UInt256 & res)
{
res = addIgnoreOverflow(x, y);
return x > std::numeric_limits<UInt256>::max() - y;
}
template <typename T>
@ -104,24 +110,30 @@ namespace common
}
template <>
inline bool subOverflow(__int128 x, __int128 y, __int128 & res)
inline bool subOverflow(Int128 x, Int128 y, Int128 & res)
{
static constexpr __int128 min_int128 = minInt128();
static constexpr __int128 max_int128 = maxInt128();
res = subIgnoreOverflow(x, y);
return (y < 0 && x > max_int128 + y) || (y > 0 && x < min_int128 + y);
return (y < 0 && x > std::numeric_limits<Int128>::max() + y) ||
(y > 0 && x < std::numeric_limits<Int128>::min() + y);
}
template <>
inline bool subOverflow(wInt256 x, wInt256 y, wInt256 & res)
inline bool subOverflow(UInt128 x, UInt128 y, UInt128 & res)
{
res = subIgnoreOverflow(x, y);
return (y < 0 && x > std::numeric_limits<wInt256>::max() + y) ||
(y > 0 && x < std::numeric_limits<wInt256>::min() + y);
return x < y;
}
template <>
inline bool subOverflow(wUInt256 x, wUInt256 y, wUInt256 & res)
inline bool subOverflow(Int256 x, Int256 y, Int256 & res)
{
res = subIgnoreOverflow(x, y);
return (y < 0 && x > std::numeric_limits<Int256>::max() + y) ||
(y > 0 && x < std::numeric_limits<Int256>::min() + y);
}
template <>
inline bool subOverflow(UInt256 x, UInt256 y, UInt256 & res)
{
res = subIgnoreOverflow(x, y);
return x < y;
@ -151,36 +163,33 @@ namespace common
return __builtin_smulll_overflow(x, y, &res);
}
/// Overflow check is not implemented for big integers.
template <>
inline bool mulOverflow(__int128 x, __int128 y, __int128 & res)
inline bool mulOverflow(Int128 x, Int128 y, Int128 & res)
{
res = mulIgnoreOverflow(x, y);
if (!x || !y)
return false;
unsigned __int128 a = (x > 0) ? x : -x;
unsigned __int128 b = (y > 0) ? y : -y;
return mulIgnoreOverflow(a, b) / b != a;
return false;
}
template <>
inline bool mulOverflow(wInt256 x, wInt256 y, wInt256 & res)
inline bool mulOverflow(Int256 x, Int256 y, Int256 & res)
{
res = mulIgnoreOverflow(x, y);
if (!x || !y)
return false;
wInt256 a = (x > 0) ? x : -x;
wInt256 b = (y > 0) ? y : -y;
return mulIgnoreOverflow(a, b) / b != a;
return false;
}
template <>
inline bool mulOverflow(wUInt256 x, wUInt256 y, wUInt256 & res)
inline bool mulOverflow(UInt128 x, UInt128 y, UInt128 & res)
{
res = mulIgnoreOverflow(x, y);
if (!x || !y)
return false;
return res / y != x;
return false;
}
template <>
inline bool mulOverflow(UInt256 x, UInt256 y, UInt256 & res)
{
res = mulIgnoreOverflow(x, y);
return false;
}
}

View File

@ -5,16 +5,14 @@
#include <common/types.h>
#include <common/wide_integer.h>
using Int128 = __int128;
using wInt256 = wide::integer<256, signed>;
using wUInt256 = wide::integer<256, unsigned>;
using Int128 = wide::integer<128, signed>;
using UInt128 = wide::integer<128, unsigned>;
using Int256 = wide::integer<256, signed>;
using UInt256 = wide::integer<256, unsigned>;
static_assert(sizeof(wInt256) == 32);
static_assert(sizeof(wUInt256) == 32);
static constexpr __int128 minInt128() { return static_cast<unsigned __int128>(1) << 127; }
static constexpr __int128 maxInt128() { return (static_cast<unsigned __int128>(1) << 127) - 1; }
static_assert(sizeof(Int256) == 32);
static_assert(sizeof(UInt256) == 32);
/// The standard library type traits, such as std::is_arithmetic, with one exception
/// (std::common_type), are "set in stone". Attempting to specialize them causes undefined behavior.
@ -26,7 +24,7 @@ struct is_signed
};
template <> struct is_signed<Int128> { static constexpr bool value = true; };
template <> struct is_signed<wInt256> { static constexpr bool value = true; };
template <> struct is_signed<Int256> { static constexpr bool value = true; };
template <typename T>
inline constexpr bool is_signed_v = is_signed<T>::value;
@ -37,7 +35,8 @@ struct is_unsigned
static constexpr bool value = std::is_unsigned_v<T>;
};
template <> struct is_unsigned<wUInt256> { static constexpr bool value = true; };
template <> struct is_unsigned<UInt128> { static constexpr bool value = true; };
template <> struct is_unsigned<UInt256> { static constexpr bool value = true; };
template <typename T>
inline constexpr bool is_unsigned_v = is_unsigned<T>::value;
@ -51,8 +50,9 @@ struct is_integer
};
template <> struct is_integer<Int128> { static constexpr bool value = true; };
template <> struct is_integer<wInt256> { static constexpr bool value = true; };
template <> struct is_integer<wUInt256> { static constexpr bool value = true; };
template <> struct is_integer<UInt128> { static constexpr bool value = true; };
template <> struct is_integer<Int256> { static constexpr bool value = true; };
template <> struct is_integer<UInt256> { static constexpr bool value = true; };
template <typename T>
inline constexpr bool is_integer_v = is_integer<T>::value;
@ -64,7 +64,11 @@ struct is_arithmetic
static constexpr bool value = std::is_arithmetic_v<T>;
};
template <> struct is_arithmetic<__int128> { static constexpr bool value = true; };
template <> struct is_arithmetic<Int128> { static constexpr bool value = true; };
template <> struct is_arithmetic<UInt128> { static constexpr bool value = true; };
template <> struct is_arithmetic<Int256> { static constexpr bool value = true; };
template <> struct is_arithmetic<UInt256> { static constexpr bool value = true; };
template <typename T>
inline constexpr bool is_arithmetic_v = is_arithmetic<T>::value;
@ -75,9 +79,10 @@ struct make_unsigned
typedef std::make_unsigned_t<T> type;
};
template <> struct make_unsigned<Int128> { using type = unsigned __int128; };
template <> struct make_unsigned<wInt256> { using type = wUInt256; };
template <> struct make_unsigned<wUInt256> { using type = wUInt256; };
template <> struct make_unsigned<Int128> { using type = UInt128; };
template <> struct make_unsigned<UInt128> { using type = UInt128; };
template <> struct make_unsigned<Int256> { using type = UInt256; };
template <> struct make_unsigned<UInt256> { using type = UInt256; };
template <typename T> using make_unsigned_t = typename make_unsigned<T>::type;
@ -87,8 +92,10 @@ struct make_signed
typedef std::make_signed_t<T> type;
};
template <> struct make_signed<wInt256> { using type = wInt256; };
template <> struct make_signed<wUInt256> { using type = wInt256; };
template <> struct make_signed<Int128> { using type = Int128; };
template <> struct make_signed<UInt128> { using type = Int128; };
template <> struct make_signed<Int256> { using type = Int256; };
template <> struct make_signed<UInt256> { using type = Int256; };
template <typename T> using make_signed_t = typename make_signed<T>::type;
@ -98,8 +105,10 @@ struct is_big_int
static constexpr bool value = false;
};
template <> struct is_big_int<wInt256> { static constexpr bool value = true; };
template <> struct is_big_int<wUInt256> { static constexpr bool value = true; };
template <> struct is_big_int<Int128> { static constexpr bool value = true; };
template <> struct is_big_int<UInt128> { static constexpr bool value = true; };
template <> struct is_big_int<Int256> { static constexpr bool value = true; };
template <> struct is_big_int<UInt256> { static constexpr bool value = true; };
template <typename T>
inline constexpr bool is_big_int_v = is_big_int<T>::value;

View File

@ -30,9 +30,8 @@
#include <cstddef>
#include <cstring>
#include <type_traits>
#include <common/extended_types.h>
using int128_t = __int128;
using uint128_t = unsigned __int128;
namespace impl
{
@ -106,7 +105,7 @@ using UnsignedOfSize = typename SelectType
uint16_t,
uint32_t,
uint64_t,
uint128_t
__uint128_t
>::Result;
/// Holds the result of dividing an unsigned N-byte variable by 10^N resulting in
@ -313,7 +312,8 @@ namespace convert
}
}
static inline int digits10(uint128_t x)
template <typename T>
static inline int digits10(T x)
{
if (x < 10ULL)
return 1;
@ -346,8 +346,11 @@ static inline int digits10(uint128_t x)
return 12 + digits10(x / 1000000000000ULL);
}
static inline char * writeUIntText(uint128_t x, char * p)
template <typename T>
static inline char * writeUIntText(T x, char * p)
{
static_assert(is_unsigned_v<T>);
int len = digits10(x);
auto pp = p + len;
while (x >= 100)
@ -370,14 +373,28 @@ static inline char * writeLeadingMinus(char * pos)
return pos + 1;
}
static inline char * writeSIntText(int128_t x, char * pos)
template <typename T>
static inline char * writeSIntText(T x, char * pos)
{
static constexpr int128_t min_int128 = uint128_t(1) << 127;
static_assert(std::is_same_v<T, Int128> || std::is_same_v<T, Int256>);
if (unlikely(x == min_int128))
using UnsignedT = make_unsigned_t<T>;
static constexpr T min_int = UnsignedT(1) << (sizeof(T) * 8 - 1);
if (unlikely(x == min_int))
{
memcpy(pos, "-170141183460469231731687303715884105728", 40);
return pos + 40;
if constexpr (std::is_same_v<T, Int128>)
{
const char * res = "-170141183460469231731687303715884105728";
memcpy(pos, res, strlen(res));
return pos + strlen(res);
}
else if constexpr (std::is_same_v<T, Int256>)
{
const char * res = "-57896044618658097711785492504343953926634992332820282019728792003956564819968";
memcpy(pos, res, strlen(res));
return pos + strlen(res);
}
}
if (x < 0)
@ -385,7 +402,7 @@ static inline char * writeSIntText(int128_t x, char * pos)
x = -x;
pos = writeLeadingMinus(pos);
}
return writeUIntText(static_cast<uint128_t>(x), pos);
return writeUIntText(UnsignedT(x), pos);
}
}
@ -403,13 +420,25 @@ inline char * itoa(char8_t i, char * p)
}
template <>
inline char * itoa<uint128_t>(uint128_t i, char * p)
inline char * itoa(UInt128 i, char * p)
{
return impl::writeUIntText(i, p);
}
template <>
inline char * itoa<int128_t>(int128_t i, char * p)
inline char * itoa(Int128 i, char * p)
{
return impl::writeSIntText(i, p);
}
template <>
inline char * itoa(UInt256 i, char * p)
{
return impl::writeUIntText(i, p);
}
template <>
inline char * itoa(Int256 i, char * p)
{
return impl::writeSIntText(i, p);
}

View File

@ -4,7 +4,8 @@
#include <type_traits>
#include <utility>
template <class T, class Tag>
template <typename T, typename Tag>
struct StrongTypedef
{
private:
@ -38,14 +39,16 @@ public:
bool operator==(const Self & rhs) const { return t == rhs.t; }
bool operator<(const Self & rhs) const { return t < rhs.t; }
bool operator>(const Self & rhs) const { return t > rhs.t; }
T & toUnderType() { return t; }
const T & toUnderType() const { return t; }
};
namespace std
{
template <class T, class Tag>
template <typename T, typename Tag>
struct hash<StrongTypedef<T, Tag>>
{
size_t operator()(const StrongTypedef<T, Tag> & x) const

View File

@ -1,13 +1,15 @@
#pragma once
#include <stdexcept>
/// Throw DB::Exception-like exception before its definition.
/// DB::Exception derived from Poco::Exception derived from std::exception.
/// DB::Exception generally cought as Poco::Exception. std::exception generally has other catch blocks and could lead to other outcomes.
/// DB::Exception generally caught as Poco::Exception. std::exception generally has other catch blocks and could lead to other outcomes.
/// DB::Exception is not defined yet. It'd better to throw Poco::Exception but we do not want to include any big header here, even <string>.
/// So we throw some std::exception instead in the hope its catch block is the same as DB::Exception one.
template <typename T>
inline void throwError(const T & err)
[[noreturn]] inline void throwError(const T & err)
{
throw std::runtime_error(err);
}

View File

@ -58,9 +58,11 @@ public:
using signed_base_type = int64_t;
// ctors
constexpr integer() noexcept;
constexpr integer() noexcept = default;
template <typename T>
constexpr integer(T rhs) noexcept;
template <typename T>
constexpr integer(std::initializer_list<T> il) noexcept;
@ -108,9 +110,9 @@ public:
constexpr explicit operator bool() const noexcept;
template <class T>
using __integral_not_wide_integer_class = typename std::enable_if<std::is_arithmetic<T>::value, T>::type;
using _integral_not_wide_integer_class = typename std::enable_if<std::is_arithmetic<T>::value, T>::type;
template <class T, class = __integral_not_wide_integer_class<T>>
template <class T, class = _integral_not_wide_integer_class<T>>
constexpr operator T() const noexcept;
constexpr operator long double() const noexcept;
@ -119,25 +121,27 @@ public:
struct _impl;
base_type items[_impl::item_count];
private:
template <size_t Bits2, typename Signed2>
friend class integer;
friend class std::numeric_limits<integer<Bits, signed>>;
friend class std::numeric_limits<integer<Bits, unsigned>>;
base_type items[_impl::item_count];
};
template <typename T>
static constexpr bool ArithmeticConcept() noexcept;
template <class T1, class T2>
using __only_arithmetic = typename std::enable_if<ArithmeticConcept<T1>() && ArithmeticConcept<T2>()>::type;
using _only_arithmetic = typename std::enable_if<ArithmeticConcept<T1>() && ArithmeticConcept<T2>()>::type;
template <typename T>
static constexpr bool IntegralConcept() noexcept;
template <class T, class T2>
using __only_integer = typename std::enable_if<IntegralConcept<T>() && IntegralConcept<T2>()>::type;
using _only_integer = typename std::enable_if<IntegralConcept<T>() && IntegralConcept<T2>()>::type;
// Unary operators
template <size_t Bits, typename Signed>
@ -153,54 +157,55 @@ constexpr integer<Bits, Signed> operator+(const integer<Bits, Signed> & lhs) noe
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator*(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
std::common_type_t<Arithmetic, Arithmetic2> constexpr operator*(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator/(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
std::common_type_t<Arithmetic, Arithmetic2> constexpr operator/(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator+(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
std::common_type_t<Arithmetic, Arithmetic2> constexpr operator+(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator-(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
std::common_type_t<Arithmetic, Arithmetic2> constexpr operator-(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator%(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
template <typename Integral, typename Integral2, class = _only_integer<Integral, Integral2>>
std::common_type_t<Integral, Integral2> constexpr operator%(const Integral & rhs, const Integral2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator&(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
template <typename Integral, typename Integral2, class = _only_integer<Integral, Integral2>>
std::common_type_t<Integral, Integral2> constexpr operator&(const Integral & rhs, const Integral2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator|(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
template <typename Integral, typename Integral2, class = _only_integer<Integral, Integral2>>
std::common_type_t<Integral, Integral2> constexpr operator|(const Integral & rhs, const Integral2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator^(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
template <typename Integral, typename Integral2, class = _only_integer<Integral, Integral2>>
std::common_type_t<Integral, Integral2> constexpr operator^(const Integral & rhs, const Integral2 & lhs);
// TODO: Integral
template <size_t Bits, typename Signed>
constexpr integer<Bits, Signed> operator<<(const integer<Bits, Signed> & lhs, int n) noexcept;
template <size_t Bits, typename Signed>
constexpr integer<Bits, Signed> operator>>(const integer<Bits, Signed> & lhs, int n) noexcept;
@ -217,32 +222,32 @@ constexpr integer<Bits, Signed> operator>>(const integer<Bits, Signed> & lhs, In
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator<(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
constexpr bool operator<(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator>(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
constexpr bool operator>(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator<=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
constexpr bool operator<=(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator>=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
constexpr bool operator>=(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator==(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
constexpr bool operator==(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator!=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
constexpr bool operator!=(const Arithmetic & rhs, const Arithmetic2 & lhs);
}

View File

@ -5,6 +5,7 @@
/// (See at http://www.boost.org/LICENSE_1_0.txt)
#include "throwError.h"
#include <cmath>
#include <cfloat>
#include <cassert>
@ -81,7 +82,7 @@ public:
res.items[T::_impl::big(0)] = std::numeric_limits<typename wide::integer<Bits, Signed>::signed_base_type>::min();
return res;
}
return 0;
return wide::integer<Bits, Signed>(0);
}
static constexpr wide::integer<Bits, Signed> max() noexcept
@ -176,7 +177,7 @@ struct integer<Bits, Signed>::_impl
constexpr static bool is_negative(const integer<B, T> & n) noexcept
{
if constexpr (std::is_same_v<T, signed>)
return static_cast<signed_base_type>(n.items[big(0)]) < 0;
return static_cast<signed_base_type>(n.items[integer<B, T>::_impl::big(0)]) < 0;
else
return false;
}
@ -193,40 +194,36 @@ struct integer<Bits, Signed>::_impl
template <size_t B, class S>
constexpr static integer<B, S> make_positive(const integer<B, S> & n) noexcept
{
return is_negative(n) ? operator_unary_minus(n) : n;
return is_negative(n) ? integer<B, S>(operator_unary_minus(n)) : n;
}
template <typename T>
__attribute__((no_sanitize("undefined"))) constexpr static auto to_Integral(T f) noexcept
{
if constexpr (std::is_same_v<T, __int128>)
return f;
else if constexpr (std::is_signed_v<T>)
if constexpr (std::is_signed_v<T>)
return static_cast<int64_t>(f);
else
return static_cast<uint64_t>(f);
}
template <typename Integral>
constexpr static void wide_integer_from_bultin(integer<Bits, Signed> & self, Integral rhs) noexcept
constexpr static void wide_integer_from_builtin(integer<Bits, Signed> & self, Integral rhs) noexcept
{
self.items[0] = _impl::to_Integral(rhs);
if constexpr (std::is_same_v<Integral, __int128>)
self.items[1] = rhs >> base_bits;
static_assert(sizeof(Integral) <= sizeof(base_type));
constexpr const unsigned start = (sizeof(Integral) == 16) ? 2 : 1;
self.items[0] = _impl::to_Integral(rhs);
if constexpr (std::is_signed_v<Integral>)
{
if (rhs < 0)
{
for (unsigned i = start; i < item_count; ++i)
for (size_t i = 1; i < item_count; ++i)
self.items[i] = -1;
return;
}
}
for (unsigned i = start; i < item_count; ++i)
for (size_t i = 1; i < item_count; ++i)
self.items[i] = 0;
}
@ -239,7 +236,8 @@ struct integer<Bits, Signed>::_impl
* a_(n - 1) = a_n * max_int + b2, a_n <= max_int <- base case.
*/
template <class T>
constexpr static void set_multiplier(integer<Bits, Signed> & self, T t) noexcept {
constexpr static void set_multiplier(integer<Bits, Signed> & self, T t) noexcept
{
constexpr uint64_t max_int = std::numeric_limits<uint64_t>::max();
/// Implementation specific behaviour on overflow (if we don't check here, stack overflow will triggered in bigint_cast).
@ -260,7 +258,8 @@ struct integer<Bits, Signed>::_impl
self += static_cast<uint64_t>(t - alpha * static_cast<T>(max_int)); // += b_i
}
constexpr static void wide_integer_from_bultin(integer<Bits, Signed>& self, double rhs) noexcept {
constexpr static void wide_integer_from_builtin(integer<Bits, Signed>& self, double rhs) noexcept
{
constexpr int64_t max_int = std::numeric_limits<int64_t>::max();
constexpr int64_t min_int = std::numeric_limits<int64_t>::min();
@ -383,13 +382,13 @@ struct integer<Bits, Signed>::_impl
if (bit_shift)
lhs.items[big(items_shift)] |= std::numeric_limits<base_type>::max() << (base_bits - bit_shift);
for (unsigned i = item_count - items_shift; i < items_shift; ++i)
lhs.items[little(i)] = std::numeric_limits<base_type>::max();
for (unsigned i = 0; i < items_shift; ++i)
lhs.items[big(i)] = std::numeric_limits<base_type>::max();
}
else
{
for (unsigned i = item_count - items_shift; i < items_shift; ++i)
lhs.items[little(i)] = 0;
for (unsigned i = 0; i < items_shift; ++i)
lhs.items[big(i)] = 0;
}
return lhs;
@ -397,23 +396,23 @@ struct integer<Bits, Signed>::_impl
private:
template <typename T>
constexpr static base_type get_item(const T & x, unsigned number)
constexpr static base_type get_item(const T & x, unsigned idx)
{
if constexpr (IsWideInteger<T>::value)
{
if (number < T::_impl::item_count)
return x.items[number];
if (idx < T::_impl::item_count)
return x.items[idx];
return 0;
}
else
{
if constexpr (sizeof(T) <= sizeof(base_type))
{
if (!number)
if (0 == idx)
return x;
}
else if (number * sizeof(base_type) < sizeof(T))
return x >> (number * base_bits); // & std::numeric_limits<base_type>::max()
else if (idx * sizeof(base_type) < sizeof(T))
return x >> (idx * base_bits); // & std::numeric_limits<base_type>::max()
return 0;
}
}
@ -439,7 +438,7 @@ private:
for (unsigned i = 1; i < item_count; ++i)
{
if (underflows[i-1])
if (underflows[i - 1])
{
base_type & res_item = res.items[little(i)];
if (res_item == 0)
@ -472,7 +471,7 @@ private:
for (unsigned i = 1; i < item_count; ++i)
{
if (overflows[i-1])
if (overflows[i - 1])
{
base_type & res_item = res.items[little(i)];
++res_item;
@ -532,6 +531,17 @@ private:
res.items[little(2)] = r12 >> 64;
return res;
}
else if constexpr (Bits == 128 && sizeof(base_type) == 8)
{
using CompilerUInt128 = unsigned __int128;
CompilerUInt128 a = (CompilerUInt128(lhs.items[1]) << 64) + lhs.items[0];
CompilerUInt128 b = (CompilerUInt128(rhs.items[1]) << 64) + rhs.items[0];
CompilerUInt128 c = a * b;
integer<Bits, Signed> res;
res.items[0] = c;
res.items[1] = c >> 64;
return res;
}
else
{
integer<Bits, Signed> res{};
@ -657,7 +667,7 @@ public:
}
template <typename T>
constexpr static bool operator_more(const integer<Bits, Signed> & lhs, const T & rhs) noexcept
constexpr static bool operator_greater(const integer<Bits, Signed> & lhs, const T & rhs) noexcept
{
if constexpr (should_keep_size<T>())
{
@ -677,7 +687,7 @@ public:
else
{
static_assert(IsWideInteger<T>::value);
return std::common_type_t<integer<Bits, Signed>, T>::_impl::operator_more(T(lhs), rhs);
return std::common_type_t<integer<Bits, Signed>, T>::_impl::operator_greater(T(lhs), rhs);
}
}
@ -764,7 +774,6 @@ public:
}
}
private:
template <typename T>
constexpr static bool is_zero(const T & x)
{
@ -781,46 +790,65 @@ private:
}
/// returns quotient as result and remainder in numerator.
template <typename T>
constexpr static T divide(T & numerator, T && denominator)
template <size_t Bits2>
constexpr static integer<Bits2, unsigned> divide(integer<Bits2, unsigned> & numerator, integer<Bits2, unsigned> denominator)
{
if (is_zero(denominator))
throwError("divide by zero");
static_assert(std::is_unsigned_v<Signed>);
T & n = numerator;
T & d = denominator;
T x = 1;
T quotient = 0;
while (!operator_more(d, n) && operator_eq(operator_amp(shift_right(d, base_bits * item_count - 1), 1), 0))
if constexpr (Bits == 128 && sizeof(base_type) == 8)
{
x = shift_left(x, 1);
d = shift_left(d, 1);
using CompilerUInt128 = unsigned __int128;
CompilerUInt128 a = (CompilerUInt128(numerator.items[1]) << 64) + numerator.items[0];
CompilerUInt128 b = (CompilerUInt128(denominator.items[1]) << 64) + denominator.items[0];
CompilerUInt128 c = a / b;
integer<Bits, Signed> res;
res.items[0] = c;
res.items[1] = c >> 64;
CompilerUInt128 remainder = a - b * c;
numerator.items[0] = remainder;
numerator.items[1] = remainder >> 64;
return res;
}
while (!operator_eq(x, 0))
if (is_zero(denominator))
throwError("Division by zero");
integer<Bits2, unsigned> x = 1;
integer<Bits2, unsigned> quotient = 0;
while (!operator_greater(denominator, numerator) && is_zero(operator_amp(shift_right(denominator, Bits2 - 1), 1)))
{
if (!operator_more(d, n))
x = shift_left(x, 1);
denominator = shift_left(denominator, 1);
}
while (!is_zero(x))
{
if (!operator_greater(denominator, numerator))
{
n = operator_minus(n, d);
numerator = operator_minus(numerator, denominator);
quotient = operator_pipe(quotient, x);
}
x = shift_right(x, 1);
d = shift_right(d, 1);
denominator = shift_right(denominator, 1);
}
return quotient;
}
public:
template <typename T>
constexpr static auto operator_slash(const integer<Bits, Signed> & lhs, const T & rhs)
{
if constexpr (should_keep_size<T>())
{
integer<Bits, Signed> numerator = make_positive(lhs);
integer<Bits, Signed> quotient = divide(numerator, make_positive(integer<Bits, Signed>(rhs)));
integer<Bits, unsigned> numerator = make_positive(lhs);
integer<Bits, unsigned> denominator = make_positive(integer<Bits, Signed>(rhs));
integer<Bits, unsigned> quotient = integer<Bits, unsigned>::_impl::divide(numerator, std::move(denominator));
if (std::is_same_v<Signed, signed> && is_negative(rhs) != is_negative(lhs))
quotient = operator_unary_minus(quotient);
@ -838,8 +866,9 @@ public:
{
if constexpr (should_keep_size<T>())
{
integer<Bits, Signed> remainder = make_positive(lhs);
divide(remainder, make_positive(integer<Bits, Signed>(rhs)));
integer<Bits, unsigned> remainder = make_positive(lhs);
integer<Bits, unsigned> denominator = make_positive(integer<Bits, Signed>(rhs));
integer<Bits, unsigned>::_impl::divide(remainder, std::move(denominator));
if (std::is_same_v<Signed, signed> && is_negative(lhs))
remainder = operator_unary_minus(remainder);
@ -905,7 +934,7 @@ public:
++c;
}
else
throwError("invalid char from");
throwError("Invalid char from");
}
}
else
@ -913,7 +942,7 @@ public:
while (*c)
{
if (*c < '0' || *c > '9')
throwError("invalid char from");
throwError("Invalid char from");
res = multiply(res, 10U);
res = plus(res, *c - '0');
@ -930,11 +959,6 @@ public:
// Members
template <size_t Bits, typename Signed>
constexpr integer<Bits, Signed>::integer() noexcept
: items{}
{}
template <size_t Bits, typename Signed>
template <typename T>
constexpr integer<Bits, Signed>::integer(T rhs) noexcept
@ -943,7 +967,7 @@ constexpr integer<Bits, Signed>::integer(T rhs) noexcept
if constexpr (IsWideInteger<T>::value)
_impl::wide_integer_from_wide_integer(*this, rhs);
else
_impl::wide_integer_from_bultin(*this, rhs);
_impl::wide_integer_from_builtin(*this, rhs);
}
template <size_t Bits, typename Signed>
@ -956,10 +980,19 @@ constexpr integer<Bits, Signed>::integer(std::initializer_list<T> il) noexcept
if constexpr (IsWideInteger<T>::value)
_impl::wide_integer_from_wide_integer(*this, *il.begin());
else
_impl::wide_integer_from_bultin(*this, *il.begin());
_impl::wide_integer_from_builtin(*this, *il.begin());
}
else if (il.size() == 0)
{
_impl::wide_integer_from_builtin(*this, 0);
}
else
_impl::wide_integer_from_bultin(*this, 0);
{
auto it = il.begin();
for (size_t i = 0; i < _impl::item_count; ++i)
if (it < il.end())
items[i] = *it;
}
}
template <size_t Bits, typename Signed>
@ -974,7 +1007,7 @@ template <size_t Bits, typename Signed>
template <typename T>
constexpr integer<Bits, Signed> & integer<Bits, Signed>::operator=(T rhs) noexcept
{
_impl::wide_integer_from_bultin(*this, rhs);
_impl::wide_integer_from_builtin(*this, rhs);
return *this;
}
@ -1057,7 +1090,7 @@ constexpr integer<Bits, Signed> & integer<Bits, Signed>::operator>>=(int n) noex
{
if (static_cast<size_t>(n) >= Bits)
{
if (is_negative(*this))
if (_impl::is_negative(*this))
*this = -1;
else
*this = 0;
@ -1107,16 +1140,17 @@ template <size_t Bits, typename Signed>
template <class T, class>
constexpr integer<Bits, Signed>::operator T() const noexcept
{
if constexpr (std::is_same_v<T, __int128>)
{
static_assert(Bits >= 128);
return (__int128(items[1]) << 64) | items[0];
}
else
{
static_assert(std::numeric_limits<T>::is_integer);
return items[0];
}
static_assert(std::numeric_limits<T>::is_integer);
/// NOTE: memcpy will suffice, but unfortunately, this function is constexpr.
using UnsignedT = std::make_unsigned_t<T>;
UnsignedT res{};
for (unsigned i = 0; i < _impl::item_count && i < (sizeof(T) + sizeof(base_type) - 1) / sizeof(base_type); ++i)
res += UnsignedT(items[i]) << (sizeof(base_type) * 8 * i);
return res;
}
template <size_t Bits, typename Signed>
@ -1280,7 +1314,7 @@ template <size_t Bits, typename Signed>
constexpr integer<Bits, Signed> operator<<(const integer<Bits, Signed> & lhs, int n) noexcept
{
if (static_cast<size_t>(n) >= Bits)
return 0;
return integer<Bits, Signed>(0);
if (n <= 0)
return lhs;
return integer<Bits, Signed>::_impl::shift_left(lhs, n);
@ -1289,7 +1323,7 @@ template <size_t Bits, typename Signed>
constexpr integer<Bits, Signed> operator>>(const integer<Bits, Signed> & lhs, int n) noexcept
{
if (static_cast<size_t>(n) >= Bits)
return 0;
return integer<Bits, Signed>(0);
if (n <= 0)
return lhs;
return integer<Bits, Signed>::_impl::shift_right(lhs, n);
@ -1309,7 +1343,7 @@ constexpr bool operator<(const Arithmetic & lhs, const Arithmetic2 & rhs)
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator>(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs)
{
return std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_more(lhs, rhs);
return std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_greater(lhs, rhs);
}
template <typename Arithmetic, typename Arithmetic2, class>
constexpr bool operator>(const Arithmetic & lhs, const Arithmetic2 & rhs)
@ -1332,7 +1366,7 @@ constexpr bool operator<=(const Arithmetic & lhs, const Arithmetic2 & rhs)
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator>=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs)
{
return std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_more(lhs, rhs)
return std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_greater(lhs, rhs)
|| std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_eq(lhs, rhs);
}
template <typename Arithmetic, typename Arithmetic2, class>

View File

@ -1,9 +1,12 @@
#pragma once
#include <string>
#include <ostream>
#include <fmt/format.h>
#include "wide_integer.h"
namespace wide
{
@ -33,3 +36,34 @@ inline std::string to_string(const integer<Bits, Signed> & n)
}
}
template <size_t Bits, typename Signed>
std::ostream & operator<<(std::ostream & out, const wide::integer<Bits, Signed> & value)
{
return out << to_string(value);
}
/// See https://fmt.dev/latest/api.html#formatting-user-defined-types
template <size_t Bits, typename Signed>
struct fmt::formatter<wide::integer<Bits, Signed>>
{
constexpr auto parse(format_parse_context & ctx)
{
auto it = ctx.begin();
auto end = ctx.end();
/// Only support {}.
if (it != end && *it != '}')
throw format_error("invalid format");
return it;
}
template <typename FormatContext>
auto format(const wide::integer<Bits, Signed> & value, FormatContext & ctx)
{
return format_to(ctx.out(), "{}", to_string(value));
}
};

View File

@ -102,7 +102,7 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config)
auto * logger = &Poco::Logger::get("SentryWriter");
if (config.getBool("send_crash_reports.enabled", false))
{
if (debug || (strlen(VERSION_OFFICIAL) > 0))
if (debug || (strlen(VERSION_OFFICIAL) > 0)) //-V560
{
enabled = true;
}

View File

@ -15,7 +15,7 @@ if (GLIBC_COMPATIBILITY)
add_headers_and_sources(glibc_compatibility .)
add_headers_and_sources(glibc_compatibility musl)
if (ARCH_ARM)
if (ARCH_AARCH64)
list (APPEND glibc_compatibility_sources musl/aarch64/syscall.s musl/aarch64/longjmp.s)
set (musl_arch_include_dir musl/aarch64)
elseif (ARCH_AMD64)

View File

@ -78,6 +78,9 @@
*
*/
// Disable warnings by PVS-Studio
//-V::GA
static const double
pi = 3.14159265358979311600e+00, /* 0x400921FB, 0x54442D18 */
a0 = 7.72156649015328655494e-02, /* 0x3FB3C467, 0xE37DB0C8 */

View File

@ -85,6 +85,9 @@
*
*/
// Disable warnings by PVS-Studio
//-V::GA
#include <stdint.h>
#include <math.h>
#include "libm.h"

View File

@ -155,7 +155,7 @@ static inline long double fp_barrierl(long double x)
static inline void fp_force_evalf(float x)
{
volatile float y;
y = x;
y = x; //-V1001
}
#endif
@ -164,7 +164,7 @@ static inline void fp_force_evalf(float x)
static inline void fp_force_eval(double x)
{
volatile double y;
y = x;
y = x; //-V1001
}
#endif
@ -173,7 +173,7 @@ static inline void fp_force_eval(double x)
static inline void fp_force_evall(long double x)
{
volatile long double y;
y = x;
y = x; //-V1001
}
#endif

View File

@ -3,6 +3,9 @@
* SPDX-License-Identifier: MIT
*/
// Disable warnings by PVS-Studio
//-V::GA
#include <math.h>
#include <stdint.h>
#include "libm.h"

View File

@ -40,7 +40,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
split->addTextLog(log, text_log_max_priority);
auto current_logger = config.getString("logger", "");
if (config_logger == current_logger)
if (config_logger == current_logger) //-V1051
return;
config_logger = current_logger;

View File

@ -447,69 +447,6 @@ inline SrcIter uneven_copy(SrcIter src_first,
std::integral_constant<bool, DEST_IS_SMALLER>{});
}
/* generate_to, fill in a fixed-size array of integral type using a SeedSeq
* (actually works for any random-access iterator)
*/
template <size_t size, typename SeedSeq, typename DestIter>
inline void generate_to_impl(SeedSeq&& generator, DestIter dest,
std::true_type)
{
generator.generate(dest, dest+size);
}
template <size_t size, typename SeedSeq, typename DestIter>
void generate_to_impl(SeedSeq&& generator, DestIter dest,
std::false_type)
{
typedef typename std::iterator_traits<DestIter>::value_type dest_t;
constexpr auto DEST_SIZE = sizeof(dest_t);
constexpr auto GEN_SIZE = sizeof(uint32_t);
constexpr bool GEN_IS_SMALLER = GEN_SIZE < DEST_SIZE;
constexpr size_t FROM_ELEMS =
GEN_IS_SMALLER
? size * ((DEST_SIZE+GEN_SIZE-1) / GEN_SIZE)
: (size + (GEN_SIZE / DEST_SIZE) - 1)
/ ((GEN_SIZE / DEST_SIZE) + GEN_IS_SMALLER);
// this odd code ^^^^^^^^^^^^^^^^^ is work-around for
// a bug: http://llvm.org/bugs/show_bug.cgi?id=21287
if (FROM_ELEMS <= 1024) {
uint32_t buffer[FROM_ELEMS];
generator.generate(buffer, buffer+FROM_ELEMS);
uneven_copy(buffer, dest, dest+size);
} else {
uint32_t* buffer = static_cast<uint32_t*>(malloc(GEN_SIZE * FROM_ELEMS));
generator.generate(buffer, buffer+FROM_ELEMS);
uneven_copy(buffer, dest, dest+size);
free(static_cast<void*>(buffer));
}
}
template <size_t size, typename SeedSeq, typename DestIter>
inline void generate_to(SeedSeq&& generator, DestIter dest)
{
typedef typename std::iterator_traits<DestIter>::value_type dest_t;
constexpr bool IS_32BIT = sizeof(dest_t) == sizeof(uint32_t);
generate_to_impl<size>(std::forward<SeedSeq>(generator), dest,
std::integral_constant<bool, IS_32BIT>{});
}
/* generate_one, produce a value of integral type using a SeedSeq
* (optionally, we can have it produce more than one and pick which one
* we want)
*/
template <typename UInt, size_t i = 0UL, size_t N = i+1UL, typename SeedSeq>
inline UInt generate_one(SeedSeq&& generator)
{
UInt result[N];
generate_to<N>(std::forward<SeedSeq>(generator), result);
return result[i];
}
template <typename RngType>
auto bounded_rand(RngType& rng, typename RngType::result_type upper_bound)
-> typename RngType::result_type
@ -517,7 +454,7 @@ auto bounded_rand(RngType& rng, typename RngType::result_type upper_bound)
typedef typename RngType::result_type rtype;
rtype threshold = (RngType::max() - RngType::min() + rtype(1) - upper_bound)
% upper_bound;
for (;;) {
for (;;) { //-V1044
rtype r = rng() - RngType::min();
if (r >= threshold)
return r % upper_bound;

View File

@ -928,7 +928,7 @@ struct rxs_m_xs_mixin {
constexpr bitcount_t shift = bits - xtypebits;
constexpr bitcount_t mask = (1 << opbits) - 1;
bitcount_t rshift =
opbits ? bitcount_t(internal >> (bits - opbits)) & mask : 0;
opbits ? bitcount_t(internal >> (bits - opbits)) & mask : 0; //-V547
internal ^= internal >> (opbits + rshift);
internal *= mcg_multiplier<itype>::multiplier();
xtype result = internal >> shift;
@ -950,7 +950,7 @@ struct rxs_m_xs_mixin {
internal *= mcg_unmultiplier<itype>::unmultiplier();
bitcount_t rshift = opbits ? (internal >> (bits - opbits)) & mask : 0;
bitcount_t rshift = opbits ? (internal >> (bits - opbits)) & mask : 0; //-V547
internal = unxorshift(internal, bits, opbits + rshift);
return internal;
@ -975,7 +975,7 @@ struct rxs_m_mixin {
: 2;
constexpr bitcount_t shift = bits - xtypebits;
constexpr bitcount_t mask = (1 << opbits) - 1;
bitcount_t rshift = opbits ? (internal >> (bits - opbits)) & mask : 0;
bitcount_t rshift = opbits ? (internal >> (bits - opbits)) & mask : 0; //-V547
internal ^= internal >> (opbits + rshift);
internal *= mcg_multiplier<itype>::multiplier();
xtype result = internal >> shift;
@ -1366,7 +1366,7 @@ void extended<table_pow2,advance_pow2,baseclass,extvalclass,kdd>::selfinit()
// - any strange correlations would only be apparent if we
// were to backstep the generator so that the base generator
// was generating the same values again
result_type xdiff = baseclass::operator()() - baseclass::operator()();
result_type xdiff = baseclass::operator()() - baseclass::operator()(); //-V501
for (size_t i = 0; i < table_size; ++i) {
data_[i] = baseclass::operator()() ^ xdiff;
}

View File

@ -2,7 +2,7 @@ if (APPLE OR SPLIT_SHARED_LIBRARIES OR NOT ARCH_AMD64)
set (ENABLE_EMBEDDED_COMPILER OFF CACHE INTERNAL "")
endif()
option (ENABLE_EMBEDDED_COMPILER "Set to TRUE to enable support for 'compile_expressions' option for query execution" ${ENABLE_LIBRARIES})
option (ENABLE_EMBEDDED_COMPILER "Enable support for 'compile_expressions' option for query execution" ON)
# Broken in macos. TODO: update clang, re-test, enable on Apple
if (ENABLE_EMBEDDED_COMPILER AND NOT SPLIT_SHARED_LIBRARIES AND ARCH_AMD64 AND NOT (SANITIZE STREQUAL "undefined"))
option (USE_INTERNAL_LLVM_LIBRARY "Use bundled or system LLVM library." ${NOT_UNBUNDLED})

@ -1 +1 @@
Subproject commit f915d35b2de676683493c86c585141a1e1c83334
Subproject commit 7d73d7610db31d4e1ecde0fb3a7ee90ef371207f

View File

@ -308,6 +308,8 @@ function run_tests
01354_order_by_tuple_collate_const
01355_ilike
01411_bayesian_ab_testing
01798_uniq_theta_sketch
01799_long_uniq_theta_sketch
collate
collation
_orc_
@ -370,6 +372,10 @@ function run_tests
# Depends on AWS
01801_s3_cluster
# Depends on LLVM JIT
01852_jit_if
01865_jit_comparison_constant_result
)
(time clickhouse-test --hung-check -j 8 --order=random --use-skip-list --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" -- "$FASTTEST_FOCUS" 2>&1 ||:) | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/test_log.txt"

View File

@ -14,11 +14,6 @@
<max_memory_usage>
<max>10G</max>
</max_memory_usage>
<!-- Not ready for production -->
<compile_expressions>
<readonly />
</compile_expressions>
</constraints>
</default>
</profiles>

View File

@ -35,7 +35,7 @@ RUN apt-get update \
ENV TZ=Europe/Moscow
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
RUN pip3 install urllib3 testflows==1.6.74 docker-compose docker dicttoxml kazoo tzlocal
RUN pip3 install urllib3 testflows==1.6.74 docker-compose docker dicttoxml kazoo tzlocal python-dateutil numpy
ENV DOCKER_CHANNEL stable
ENV DOCKER_VERSION 17.09.1-ce
@ -74,4 +74,3 @@ VOLUME /var/lib/docker
EXPOSE 2375
ENTRYPOINT ["dockerd-entrypoint.sh"]
CMD ["sh", "-c", "python3 regression.py --no-color -o classic --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json; /usr/local/bin/process_testflows_result.py || echo -e 'failure\tCannot parse results' > check_status.tsv"]

View File

@ -139,6 +139,7 @@ The following settings can be specified in configuration file for given endpoint
- `endpoint` — Specifies prefix of an endpoint. Mandatory.
- `access_key_id` and `secret_access_key` — Specifies credentials to use with given endpoint. Optional.
- `region` — Specifies S3 region name. Optional.
- `use_environment_credentials` — If set to `true`, S3 client will try to obtain credentials from environment variables and Amazon EC2 metadata for given endpoint. Optional, default value is `false`.
- `use_insecure_imds_request` — If set to `true`, S3 client will use insecure IMDS request while obtaining credentials from Amazon EC2 metadata. Optional, default value is `false`.
- `header` — Adds specified HTTP header to a request to given endpoint. Optional, can be speficied multiple times.
@ -152,6 +153,7 @@ The following settings can be specified in configuration file for given endpoint
<endpoint>https://storage.yandexcloud.net/my-test-bucket-768/</endpoint>
<!-- <access_key_id>ACCESS_KEY_ID</access_key_id> -->
<!-- <secret_access_key>SECRET_ACCESS_KEY</secret_access_key> -->
<!-- <region>us-west-1</region> -->
<!-- <use_environment_credentials>false</use_environment_credentials> -->
<!-- <use_insecure_imds_request>false</use_insecure_imds_request> -->
<!-- <header>Authorization: Bearer SOME-TOKEN</header> -->

View File

@ -739,6 +739,7 @@ Configuration markup:
<endpoint>https://storage.yandexcloud.net/my-bucket/root-path/</endpoint>
<access_key_id>your_access_key_id</access_key_id>
<secret_access_key>your_secret_access_key</secret_access_key>
<region></region>
<server_side_encryption_customer_key_base64>your_base64_encoded_customer_key</server_side_encryption_customer_key_base64>
<proxy>
<uri>http://proxy1</uri>
@ -764,6 +765,7 @@ Required parameters:
- `secret_access_key` — S3 secret access key.
Optional parameters:
- `region` — S3 region name.
- `use_environment_credentials` — Reads AWS credentials from the Environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY and AWS_SESSION_TOKEN if they exist. Default value is `false`.
- `use_insecure_imds_request` — If set to `true`, S3 client will use insecure IMDS request while obtaining credentials from Amazon EC2 metadata. Default value is `false`.
- `proxy` — Proxy configuration for S3 endpoint. Each `uri` element inside `proxy` block should contain a proxy URL.

View File

@ -21,6 +21,7 @@ Columns:
- `bytes_allocated` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Amount of RAM allocated for the dictionary.
- `query_count` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of queries since the dictionary was loaded or since the last successful reboot.
- `hit_rate` ([Float64](../../sql-reference/data-types/float.md)) — For cache dictionaries, the percentage of uses for which the value was in the cache.
- `found_rate` ([Float64](../../sql-reference/data-types/float.md)) — The percentage of uses for which the value was found.
- `element_count` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of items stored in the dictionary.
- `load_factor` ([Float64](../../sql-reference/data-types/float.md)) — Percentage filled in the dictionary (for a hashed dictionary, the percentage filled in the hash table).
- `source` ([String](../../sql-reference/data-types/string.md)) — Text describing the [data source](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md) for the dictionary.
@ -60,4 +61,4 @@ SELECT * FROM system.dictionaries
└──────────┴──────┴────────┴─────────────┴──────┴────────┴──────────────────────────────────────┴─────────────────────┴─────────────────┴─────────────┴──────────┴───────────────┴───────────────────────┴────────────────────────────┴──────────────┴──────────────┴─────────────────────┴──────────────────────────────┘───────────────────────┴────────────────┘
```
[Original article](https://clickhouse.tech/docs/en/operations/system_tables/dictionaries) <!--hide-->
[Original article](https://clickhouse.tech/docs/en/operations/system_tables/dictionaries) <!--hide-->

View File

@ -12,6 +12,9 @@ The result depends on the order of running the query, and is nondeterministic.
When using multiple `quantile*` functions with different levels in a query, the internal states are not combined (that is, the query works less efficiently than it could). In this case, use the [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) function.
!!! note "Note"
Using `quantileTDigestWeighted` [is not recommended for tiny data sets](https://github.com/tdunning/t-digest/issues/167#issuecomment-828650275) and can lead to significat error. In this case, consider possibility of using [`quantileTDigest`](../../../sql-reference/aggregate-functions/reference/quantiletdigest.md) instead.
**Syntax**
``` sql

View File

@ -38,3 +38,4 @@ We recommend using this function in almost all scenarios.
- [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64)
- [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniqhll12.md#agg_function-uniqhll12)
- [uniqExact](../../../sql-reference/aggregate-functions/reference/uniqexact.md#agg_function-uniqexact)
- [uniqThetaSketch](../../../sql-reference/aggregate-functions/reference/uniqthetasketch.md#agg_function-uniqthetasketch)

View File

@ -49,3 +49,4 @@ Compared to the [uniq](../../../sql-reference/aggregate-functions/reference/uniq
- [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64)
- [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniqhll12.md#agg_function-uniqhll12)
- [uniqExact](../../../sql-reference/aggregate-functions/reference/uniqexact.md#agg_function-uniqexact)
- [uniqThetaSketch](../../../sql-reference/aggregate-functions/reference/uniqthetasketch.md#agg_function-uniqthetasketch)

View File

@ -23,3 +23,4 @@ The function takes a variable number of parameters. Parameters can be `Tuple`, `
- [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq)
- [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniqcombined)
- [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniqhll12)
- [uniqThetaSketch](../../../sql-reference/aggregate-functions/reference/uniqthetasketch.md#agg_function-uniqthetasketch)

View File

@ -37,3 +37,4 @@ We dont recommend using this function. In most cases, use the [uniq](../../..
- [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq)
- [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md#agg_function-uniqcombined)
- [uniqExact](../../../sql-reference/aggregate-functions/reference/uniqexact.md#agg_function-uniqexact)
- [uniqThetaSketch](../../../sql-reference/aggregate-functions/reference/uniqthetasketch.md#agg_function-uniqthetasketch)

View File

@ -0,0 +1,39 @@
---
toc_priority: 195
---
# uniqThetaSketch {#agg_function-uniqthetasketch}
Calculates the approximate number of different argument values, using the [Theta Sketch Framework](https://datasketches.apache.org/docs/Theta/ThetaSketchFramework.html).
``` sql
uniqThetaSketch(x[, ...])
```
**Arguments**
The function takes a variable number of parameters. Parameters can be `Tuple`, `Array`, `Date`, `DateTime`, `String`, or numeric types.
**Returned value**
- A [UInt64](../../../sql-reference/data-types/int-uint.md)-type number.
**Implementation details**
Function:
- Calculates a hash for all parameters in the aggregate, then uses it in calculations.
- Uses the [KMV](https://datasketches.apache.org/docs/Theta/InverseEstimate.html) algorithm to approximate the number of different argument values.
4096(2^12) 64-bit sketch are used. The size of the state is about 41 KB.
- The relative error is 3.125% (95% confidence), see the [relative error table](https://datasketches.apache.org/docs/Theta/ThetaErrorTable.html) for detail.
**See Also**
- [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq)
- [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md#agg_function-uniqcombined)
- [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64)
- [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniqhll12.md#agg_function-uniqhll12)
- [uniqExact](../../../sql-reference/aggregate-functions/reference/uniqexact.md#agg_function-uniqexact)

View File

@ -47,9 +47,9 @@ Union
### EXPLAIN AST {#explain-ast}
Dump query AST.
Dump query AST. Supports all types of queries, not only `SELECT`.
Example:
Examples:
```sql
EXPLAIN AST SELECT 1;
@ -63,6 +63,22 @@ SelectWithUnionQuery (children 1)
Literal UInt64_1
```
```sql
EXPLAIN AST ALTER TABLE t1 DELETE WHERE date = today();
```
```sql
explain
AlterQuery t1 (children 1)
ExpressionList (children 1)
AlterCommand 27 (children 1)
Function equals (children 1)
ExpressionList (children 2)
Identifier date
Function today (children 1)
ExpressionList
```
### EXPLAIN SYNTAX {#explain-syntax}
Returns query after syntax optimizations.

View File

@ -82,6 +82,7 @@ SELECT * FROM s3_engine_table LIMIT 2;
Необязательные настройки:
- `access_key_id` и `secret_access_key` — указывают учетные данные для использования с данной точкой приема запроса.
- `region` — название региона S3.
- `use_environment_credentials` — если `true`, S3-клиент будет пытаться получить учетные данные из переменных среды и метаданных Amazon EC2 для данной точки приема запроса. Значение по умолчанию - `false`.
- `header` — добавляет указанный HTTP-заголовок к запросу на заданную точку приема запроса. Может быть определен несколько раз.
- `server_side_encryption_customer_key_base64` — устанавливает необходимые заголовки для доступа к объектам S3 с шифрованием SSE-C.
@ -94,6 +95,7 @@ SELECT * FROM s3_engine_table LIMIT 2;
<endpoint>https://storage.yandexcloud.net/my-test-bucket-768/</endpoint>
<!-- <access_key_id>ACCESS_KEY_ID</access_key_id> -->
<!-- <secret_access_key>SECRET_ACCESS_KEY</secret_access_key> -->
<!-- <region>us-west-1</region> -->
<!-- <use_environment_credentials>false</use_environment_credentials> -->
<!-- <header>Authorization: Bearer SOME-TOKEN</header> -->
<!-- <server_side_encryption_customer_key_base64>BASE64-ENCODED-KEY</server_side_encryption_customer_key_base64> -->

View File

@ -727,6 +727,7 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd'
<endpoint>https://storage.yandexcloud.net/my-bucket/root-path/</endpoint>
<access_key_id>your_access_key_id</access_key_id>
<secret_access_key>your_secret_access_key</secret_access_key>
<region></region>
<proxy>
<uri>http://proxy1</uri>
<uri>http://proxy2</uri>
@ -753,6 +754,7 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd'
Необязательные параметры:
- `region` — название региона S3.
- `use_environment_credentials` — признак, нужно ли считывать учетные данные AWS из сетевого окружения, а также из переменных окружения `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` и `AWS_SESSION_TOKEN`, если они есть. Значение по умолчанию: `false`.
- `use_insecure_imds_request` — признак, нужно ли использовать менее безопасное соединение при выполнении запроса к IMDS при получении учётных данных из метаданных Amazon EC2. Значение по умолчанию: `false`.
- `proxy` — конфигурация прокси-сервера для конечной точки S3. Каждый элемент `uri` внутри блока `proxy` должен содержать URL прокси-сервера.

View File

@ -1,248 +0,0 @@
---
toc_priority: 58
toc_title: "Советы по эксплуатации"
---
# Советы по эксплуатации {#sovety-po-ekspluatatsii}
## CPU Scaling Governor {#cpu-scaling-governor}
Всегда используйте `performance` scaling governor. `ondemand` scaling governor работает намного хуже при постоянно высоком спросе.
``` bash
$ echo 'performance' | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
```
## Ограничение CPU {#ogranichenie-cpu}
Процессоры могут перегреваться. С помощью `dmesg` можно увидеть, если тактовая частота процессора была ограничена из-за перегрева.
Также ограничение может устанавливаться снаружи на уровне дата-центра. С помощью `turbostat` можно за этим наблюдать под нагрузкой.
## Оперативная память {#operativnaia-pamiat}
Для небольших объёмов данных (до ~200 Гб в сжатом виде) лучше всего использовать столько памяти не меньше, чем объём данных.
Для больших объёмов данных, при выполнении интерактивных (онлайн) запросов, стоит использовать разумный объём оперативной памяти (128 Гб или более) для того, чтобы горячее подмножество данных поместилось в кеше страниц.
Даже для объёмов данных в ~50 Тб на сервер, использование 128 Гб оперативной памяти намного лучше для производительности выполнения запросов, чем 64 Гб.
Не выключайте overcommit. Значение `cat /proc/sys/vm/overcommit_memory` должно быть 0 or 1. Выполните:
``` bash
$ echo 0 | sudo tee /proc/sys/vm/overcommit_memory
```
## Huge Pages {#huge-pages}
Механизм прозрачных huge pages нужно отключить. Он мешает работе аллокаторов памяти, что приводит к значительной деградации производительности.
``` bash
$ echo 'madvise' | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
```
С помощью `perf top` можно наблюдать за временем, проведенном в ядре операционной системы для управления памятью.
Постоянные huge pages так же не нужно аллоцировать.
## Подсистема хранения {#podsistema-khraneniia}
Если ваш бюджет позволяет использовать SSD, используйте SSD.
В противном случае используйте HDD. SATA HDDs 7200 RPM подойдут.
Предпочитайте много серверов с локальными жесткими дисками вместо меньшего числа серверов с подключенными дисковыми полками.
Но для хранения архивов с редкими запросами полки всё же подходят.
## RAID {#raid}
При использовании HDD можно объединить их RAID-10, RAID-5, RAID-6 или RAID-50.
Лучше использовать программный RAID в Linux (`mdadm`). Лучше не использовать LVM.
При создании RAID-10, нужно выбрать `far` расположение.
Если бюджет позволяет, лучше выбрать RAID-10.
На более чем 4 дисках вместо RAID-5 нужно использовать RAID-6 (предпочтительнее) или RAID-50.
При использовании RAID-5, RAID-6 или RAID-50, нужно всегда увеличивать stripe_cache_size, так как значение по умолчанию выбрано не самым удачным образом.
``` bash
$ echo 4096 | sudo tee /sys/block/md2/md/stripe_cache_size
```
Точное число стоит вычислять из числа устройств и размер блока по формуле: `2 * num_devices * chunk_size_in_bytes / 4096`.
Размер блока в 1024 Кб подходит для всех конфигураций RAID.
Никогда не указывайте слишком маленький или слишком большой размер блока.
На SSD можно использовать RAID-0.
Вне зависимости от использования RAID, всегда используйте репликацию для безопасности данных.
Включите NCQ с длинной очередью. Для HDD стоит выбрать планировщик CFQ, а для SSD — noop. Не стоит уменьшать настройку readahead.
На HDD стоит включать кеш записи.
## Файловая система {#failovaia-sistema}
Ext4 самый проверенный вариант. Укажите опции монтирования `noatime,nobarrier`.
XFS также подходит, но не так тщательно протестирована в сочетании с ClickHouse.
Большинство других файловых систем также должны нормально работать. Файловые системы с отложенной аллокацией работают лучше.
## Ядро Linux {#iadro-linux}
Не используйте слишком старое ядро Linux.
## Сеть {#set}
При использовании IPv6, стоит увеличить размер кеша маршрутов.
Ядра Linux до 3.2 имели массу проблем в реализации IPv6.
Предпочитайте как минимум 10 Гбит сеть. 1 Гбит также будет работать, но намного хуже для починки реплик с десятками терабайт данных или для обработки распределенных запросов с большим объёмом промежуточных данных.
## ZooKeeper {#zookeeper}
Вероятно вы уже используете ZooKeeper для других целей. Можно использовать ту же инсталляцию ZooKeeper, если она не сильно перегружена.
Лучше использовать свежую версию ZooKeeper, как минимум 3.4.9. Версия в стабильных дистрибутивах Linux может быть устаревшей.
Никогда не используете написанные вручную скрипты для переноса данных между разными ZooKeeper кластерами, потому что результат будет некорректный для sequential нод. Никогда не используйте утилиту «zkcopy», по той же причине: https://github.com/ksprojects/zkcopy/issues/15
Если вы хотите разделить существующий ZooKeeper кластер на два, правильный способ - увеличить количество его реплик, а затем переконфигурировать его как два независимых кластера.
Не запускайте ZooKeeper на тех же серверах, что и ClickHouse. Потому что ZooKeeper очень чувствителен к задержкам, а ClickHouse может использовать все доступные системные ресурсы.
С настройками по умолчанию, ZooKeeper является бомбой замедленного действия:
> Сервер ZooKeeper не будет удалять файлы со старыми снепшоты и логами при использовании конфигурации по умолчанию (см. autopurge), это является ответственностью оператора.
Эту бомбу нужно обезвредить.
Далее описана конфигурация ZooKeeper (3.5.1), используемая в боевом окружении Яндекс.Метрики на момент 20 мая 2017 года:
zoo.cfg:
``` bash
# http://hadoop.apache.org/zookeeper/docs/current/zookeeperAdmin.html
# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial
# synchronization phase can take
initLimit=30000
# The number of ticks that can pass between
# sending a request and getting an acknowledgement
syncLimit=10
maxClientCnxns=2000
maxSessionTimeout=60000000
# the directory where the snapshot is stored.
dataDir=/opt/zookeeper/{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/data
# Place the dataLogDir to a separate physical disc for better performance
dataLogDir=/opt/zookeeper/{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/logs
autopurge.snapRetainCount=10
autopurge.purgeInterval=1
# To avoid seeks ZooKeeper allocates space in the transaction log file in
# blocks of preAllocSize kilobytes. The default block size is 64M. One reason
# for changing the size of the blocks is to reduce the block size if snapshots
# are taken more often. (Also, see snapCount).
preAllocSize=131072
# Clients can submit requests faster than ZooKeeper can process them,
# especially if there are a lot of clients. To prevent ZooKeeper from running
# out of memory due to queued requests, ZooKeeper will throttle clients so that
# there is no more than globalOutstandingLimit outstanding requests in the
# system. The default limit is 1,000.ZooKeeper logs transactions to a
# transaction log. After snapCount transactions are written to a log file a
# snapshot is started and a new transaction log file is started. The default
# snapCount is 10,000.
snapCount=3000000
# If this option is defined, requests will be will logged to a trace file named
# traceFile.year.month.day.
#traceFile=
# Leader accepts client connections. Default value is "yes". The leader machine
# coordinates updates. For higher update throughput at thes slight expense of
# read throughput the leader can be configured to not accept clients and focus
# on coordination.
leaderServes=yes
standaloneEnabled=false
dynamicConfigFile=/etc/zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/conf/zoo.cfg.dynamic
```
Версия Java:
``` text
Java(TM) SE Runtime Environment (build 1.8.0_25-b17)
Java HotSpot(TM) 64-Bit Server VM (build 25.25-b02, mixed mode)
```
Параметры JVM:
``` bash
NAME=zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}
ZOOCFGDIR=/etc/$NAME/conf
# TODO this is really ugly
# How to find out, which jars are needed?
# seems, that log4j requires the log4j.properties file to be in the classpath
CLASSPATH="$ZOOCFGDIR:/usr/build/classes:/usr/build/lib/*.jar:/usr/share/zookeeper/zookeeper-3.5.1-metrika.jar:/usr/share/zookeeper/slf4j-log4j12-1.7.5.jar:/usr/share/zookeeper/slf4j-api-1.7.5.jar:/usr/share/zookeeper/servlet-api-2.5-20081211.jar:/usr/share/zookeeper/netty-3.7.0.Final.jar:/usr/share/zookeeper/log4j-1.2.16.jar:/usr/share/zookeeper/jline-2.11.jar:/usr/share/zookeeper/jetty-util-6.1.26.jar:/usr/share/zookeeper/jetty-6.1.26.jar:/usr/share/zookeeper/javacc.jar:/usr/share/zookeeper/jackson-mapper-asl-1.9.11.jar:/usr/share/zookeeper/jackson-core-asl-1.9.11.jar:/usr/share/zookeeper/commons-cli-1.2.jar:/usr/src/java/lib/*.jar:/usr/etc/zookeeper"
ZOOCFG="$ZOOCFGDIR/zoo.cfg"
ZOO_LOG_DIR=/var/log/$NAME
USER=zookeeper
GROUP=zookeeper
PIDDIR=/var/run/$NAME
PIDFILE=$PIDDIR/$NAME.pid
SCRIPTNAME=/etc/init.d/$NAME
JAVA=/usr/bin/java
ZOOMAIN="org.apache.zookeeper.server.quorum.QuorumPeerMain"
ZOO_LOG4J_PROP="INFO,ROLLINGFILE"
JMXLOCALONLY=false
JAVA_OPTS="-Xms{{ '{{' }} cluster.get('xms','128M') {{ '{{' }} '}}' }} \
-Xmx{{ '{{' }} cluster.get('xmx','1G') {{ '{{' }} '}}' }} \
-Xloggc:/var/log/$NAME/zookeeper-gc.log \
-XX:+UseGCLogFileRotation \
-XX:NumberOfGCLogFiles=16 \
-XX:GCLogFileSize=16M \
-verbose:gc \
-XX:+PrintGCTimeStamps \
-XX:+PrintGCDateStamps \
-XX:+PrintGCDetails
-XX:+PrintTenuringDistribution \
-XX:+PrintGCApplicationStoppedTime \
-XX:+PrintGCApplicationConcurrentTime \
-XX:+PrintSafepointStatistics \
-XX:+UseParNewGC \
-XX:+UseConcMarkSweepGC \
-XX:+CMSParallelRemarkEnabled"
```
Salt init:
``` text
description "zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }} centralized coordination service"
start on runlevel [2345]
stop on runlevel [!2345]
respawn
limit nofile 8192 8192
pre-start script
[ -r "/etc/zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/conf/environment" ] || exit 0
. /etc/zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/conf/environment
[ -d $ZOO_LOG_DIR ] || mkdir -p $ZOO_LOG_DIR
chown $USER:$GROUP $ZOO_LOG_DIR
end script
script
. /etc/zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/conf/environment
[ -r /etc/default/zookeeper ] && . /etc/default/zookeeper
if [ -z "$JMXDISABLE" ]; then
JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY"
fi
exec start-stop-daemon --start -c $USER --exec $JAVA --name zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }} \
-- -cp $CLASSPATH $JAVA_OPTS -Dzookeeper.log.dir=${ZOO_LOG_DIR} \
-Dzookeeper.root.logger=${ZOO_LOG4J_PROP} $ZOOMAIN $ZOOCFG
end script
```

1
docs/ru/operations/tips.md Symbolic link
View File

@ -0,0 +1 @@
../../en/operations/tips.md

View File

@ -12,6 +12,9 @@ toc_priority: 208
Внутренние состояния функций `quantile*` не объединяются, если они используются в одном запросе. Если вам необходимо вычислить квантили нескольких уровней, используйте функцию [quantiles](#quantiles), это повысит эффективность запроса.
!!! note "Примечание"
Использование `quantileTDigestWeighted` [не рекомендуется для небольших наборов данных](https://github.com/tdunning/t-digest/issues/167#issuecomment-828650275) и может привести к значительной ошибке. Рассмотрите возможность использования [`quantileTDigest`](../../../sql-reference/aggregate-functions/reference/quantiletdigest.md) в таких случаях.
**Синтаксис**
``` sql

View File

@ -47,9 +47,9 @@ Union
### EXPLAIN AST {#explain-ast}
Дамп AST запроса.
Дамп AST запроса. Поддерживает все типы запросов, не только `SELECT`.
Пример:
Примеры:
```sql
EXPLAIN AST SELECT 1;
@ -63,6 +63,22 @@ SelectWithUnionQuery (children 1)
Literal UInt64_1
```
```sql
EXPLAIN AST ALTER TABLE t1 DELETE WHERE date = today();
```
```sql
explain
AlterQuery t1 (children 1)
ExpressionList (children 1)
AlterCommand 27 (children 1)
Function equals (children 1)
ExpressionList (children 2)
Identifier date
Function today (children 1)
ExpressionList
```
### EXPLAIN SYNTAX {#explain-syntax}
Возвращает текст запроса после применения синтаксических оптимизаций.

View File

@ -1461,18 +1461,25 @@ private:
// when `lambda()` function gets substituted into a wrong place.
// To avoid dealing with these cases, run the check only for the
// queries we were able to successfully execute.
// The final caveat is that sometimes WITH queries are not executed,
// Another caveat is that sometimes WITH queries are not executed,
// if they are not referenced by the main SELECT, so they can still
// have the aforementioned problems. Disable this check for such
// queries, for lack of a better solution.
if (!have_error && queryHasWithClause(parsed_query.get()))
// There is also a problem that fuzzer substitutes positive Int64
// literals or Decimal literals, which are then parsed back as
// UInt64, and suddenly duplicate alias substitition starts or stops
// working (ASTWithAlias::formatImpl) or something like that.
// So we compare not even the first and second formatting of the
// query, but second and third.
// If you have to add any more workarounds to this check, just remove
// it altogether, it's not so useful.
if (!have_error && !queryHasWithClause(parsed_query.get()))
{
ASTPtr parsed_formatted_query;
ASTPtr ast_2;
try
{
const auto * tmp_pos = query_to_send.c_str();
parsed_formatted_query = parseQuery(tmp_pos,
tmp_pos + query_to_send.size(),
ast_2 = parseQuery(tmp_pos, tmp_pos + query_to_send.size(),
false /* allow_multi_statements */);
}
catch (Exception & e)
@ -1483,25 +1490,31 @@ private:
}
}
if (parsed_formatted_query)
if (ast_2)
{
const auto formatted_twice
= parsed_formatted_query->formatForErrorMessage();
const auto text_2 = ast_2->formatForErrorMessage();
const auto * tmp_pos = text_2.c_str();
const auto ast_3 = parseQuery(tmp_pos, tmp_pos + text_2.size(),
false /* allow_multi_statements */);
const auto text_3 = ast_3->formatForErrorMessage();
if (formatted_twice != query_to_send)
if (text_3 != text_2)
{
fmt::print(stderr, "The query formatting is broken.\n");
printChangedSettings();
fmt::print(stderr, "Got the following (different) text after formatting the fuzzed query and parsing it back:\n'{}'\n, expected:\n'{}'\n",
formatted_twice, query_to_send);
fmt::print(stderr,
"Got the following (different) text after formatting the fuzzed query and parsing it back:\n'{}'\n, expected:\n'{}'\n",
text_3, text_2);
fmt::print(stderr, "In more detail:\n");
fmt::print(stderr, "AST-1:\n'{}'\n", parsed_query->dumpTree());
fmt::print(stderr, "AST-1 (generated by fuzzer):\n'{}'\n", parsed_query->dumpTree());
fmt::print(stderr, "Text-1 (AST-1 formatted):\n'{}'\n", query_to_send);
fmt::print(stderr, "AST-2 (Text-1 parsed):\n'{}'\n", parsed_formatted_query->dumpTree());
fmt::print(stderr, "Text-2 (AST-2 formatted):\n'{}'\n", formatted_twice);
fmt::print(stderr, "Text-1 must be equal to Text-2, but it is not.\n");
fmt::print(stderr, "AST-2 (Text-1 parsed):\n'{}'\n", ast_2->dumpTree());
fmt::print(stderr, "Text-2 (AST-2 formatted):\n'{}'\n", text_2);
fmt::print(stderr, "AST-3 (Text-2 parsed):\n'{}'\n", ast_3->dumpTree());
fmt::print(stderr, "Text-3 (AST-3 formatted):\n'{}'\n", text_3);
fmt::print(stderr, "Text-3 must be equal to Text-2, but it is not.\n");
exit(1);
}
@ -1518,6 +1531,11 @@ private:
server_exception.reset();
client_exception.reset();
have_error = false;
// We have to reinitialize connection after errors, because it
// might have gotten into a wrong state and we'll get false
// positives about "Unknown packet from server".
connection->forceConnected(connection_parameters.timeouts);
}
else if (ast_to_process->formatForErrorMessage().size() > 500)
{

View File

@ -4,11 +4,14 @@
#include <unordered_map>
#include <vector>
#include <pcg-random/pcg_random.hpp>
#include <Common/randomSeed.h>
#include <Common/Stopwatch.h>
#include <Core/Field.h>
#include <Parsers/IAST.h>
namespace DB
{

View File

@ -774,7 +774,7 @@ UInt128 diffHash(const CommitDiff & file_changes)
}
UInt128 hash_of_diff;
hasher.get128(hash_of_diff.low, hash_of_diff.high);
hasher.get128(hash_of_diff.items[0], hash_of_diff.items[1]);
return hash_of_diff;
}

View File

@ -23,6 +23,8 @@ public:
SharedLibraryHandler(const SharedLibraryHandler & other);
SharedLibraryHandler & operator=(const SharedLibraryHandler & other) = delete;
~SharedLibraryHandler();
BlockInputStreamPtr loadAll();

View File

@ -365,16 +365,20 @@ static void transformFixedString(const UInt8 * src, UInt8 * dst, size_t size, UI
}
}
static void transformUUID(const UInt128 & src, UInt128 & dst, UInt64 seed)
static void transformUUID(const UUID & src_uuid, UUID & dst_uuid, UInt64 seed)
{
const UInt128 & src = src_uuid.toUnderType();
UInt128 & dst = dst_uuid.toUnderType();
SipHash hash;
hash.update(seed);
hash.update(reinterpret_cast<const char *>(&src), sizeof(UInt128));
hash.update(reinterpret_cast<const char *>(&src), sizeof(UUID));
/// Saving version and variant from an old UUID
hash.get128(reinterpret_cast<char *>(&dst));
dst.high = (dst.high & 0x1fffffffffffffffull) | (src.high & 0xe000000000000000ull);
dst.low = (dst.low & 0xffffffffffff0fffull) | (src.low & 0x000000000000f000ull);
dst.items[1] = (dst.items[1] & 0x1fffffffffffffffull) | (src.items[1] & 0xe000000000000000ull);
dst.items[0] = (dst.items[0] & 0xffffffffffff0fffull) | (src.items[0] & 0x000000000000f000ull);
}
class FixedStringModel : public IModel
@ -426,10 +430,10 @@ public:
ColumnPtr generate(const IColumn & column) override
{
const ColumnUInt128 & src_column = assert_cast<const ColumnUInt128 &>(column);
const ColumnUUID & src_column = assert_cast<const ColumnUUID &>(column);
const auto & src_data = src_column.getData();
auto res_column = ColumnUInt128::create();
auto res_column = ColumnUUID::create();
auto & res_data = res_column->getData();
res_data.resize(src_data.size());

View File

@ -1,5 +1,6 @@
#include "ODBCBlockOutputStream.h"
#include <Common/hex.h>
#include <common/logger_useful.h>
#include <Core/Field.h>
#include <common/LocalDate.h>
@ -37,7 +38,6 @@ namespace
query.IAST::format(settings);
return buf.str();
}
}
ODBCBlockOutputStream::ODBCBlockOutputStream(nanodbc::ConnectionHolderPtr connection_,

View File

@ -13,6 +13,7 @@
#include <Poco/Net/HTTPServer.h>
#include <Poco/Net/NetException.h>
#include <Poco/Util/HelpFormatter.h>
#include <Poco/Environment.h>
#include <ext/scope_guard.h>
#include <common/defines.h>
#include <common/logger_useful.h>
@ -385,6 +386,11 @@ void Server::initialize(Poco::Util::Application & self)
{
BaseDaemon::initialize(self);
logger().information("starting up");
LOG_INFO(&logger(), "OS Name = {}, OS Version = {}, OS Architecture = {}",
Poco::Environment::osName(),
Poco::Environment::osVersion(),
Poco::Environment::osArchitecture());
}
std::string Server::getDefaultCorePath() const
@ -879,7 +885,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
global_context->setMMappedFileCache(mmap_cache_size);
#if USE_EMBEDDED_COMPILER
size_t compiled_expression_cache_size = config().getUInt64("compiled_expression_cache_size", 500);
constexpr size_t compiled_expression_cache_size_default = 1024 * 1024 * 1024;
size_t compiled_expression_cache_size = config().getUInt64("compiled_expression_cache_size", compiled_expression_cache_size_default);
CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_size);
#endif

View File

@ -329,6 +329,8 @@
-->
<mmap_cache_size>1000</mmap_cache_size>
<!-- Cache size for compiled expressions.-->
<compiled_expression_cache_size>1073741824</compiled_expression_cache_size>
<!-- Path to data directory, with trailing slash. -->
<path>/var/lib/clickhouse/</path>

View File

@ -355,8 +355,9 @@ String DiskAccessStorage::getStorageParamsJSON() const
std::lock_guard lock{mutex};
Poco::JSON::Object json;
json.set("path", directory_path);
if (readonly)
json.set("readonly", readonly.load());
bool readonly_loaded = readonly;
if (readonly_loaded)
json.set("readonly", Poco::Dynamic::Var{true});
std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
oss.exceptions(std::ios::failbit);
Poco::JSON::Stringifier::stringify(json, oss);

View File

@ -77,7 +77,7 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str
if (enable_tls_lc_str == "starttls")
params.enable_tls = LDAPClient::Params::TLSEnable::YES_STARTTLS;
else if (config.getBool(ldap_server_config + ".enable_tls"))
params.enable_tls = LDAPClient::Params::TLSEnable::YES;
params.enable_tls = LDAPClient::Params::TLSEnable::YES; //-V1048
else
params.enable_tls = LDAPClient::Params::TLSEnable::NO;
}
@ -96,7 +96,7 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str
else if (tls_minimum_protocol_version_lc_str == "tls1.1")
params.tls_minimum_protocol_version = LDAPClient::Params::TLSProtocolVersion::TLS1_1;
else if (tls_minimum_protocol_version_lc_str == "tls1.2")
params.tls_minimum_protocol_version = LDAPClient::Params::TLSProtocolVersion::TLS1_2;
params.tls_minimum_protocol_version = LDAPClient::Params::TLSProtocolVersion::TLS1_2; //-V1048
else
throw Exception("Bad value for 'tls_minimum_protocol_version' entry, allowed values are: 'ssl2', 'ssl3', 'tls1.0', 'tls1.1', 'tls1.2'", ErrorCodes::BAD_ARGUMENTS);
}
@ -113,7 +113,7 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str
else if (tls_require_cert_lc_str == "try")
params.tls_require_cert = LDAPClient::Params::TLSRequireCert::TRY;
else if (tls_require_cert_lc_str == "demand")
params.tls_require_cert = LDAPClient::Params::TLSRequireCert::DEMAND;
params.tls_require_cert = LDAPClient::Params::TLSRequireCert::DEMAND; //-V1048
else
throw Exception("Bad value for 'tls_require_cert' entry, allowed values are: 'never', 'allow', 'try', 'demand'", ErrorCodes::BAD_ARGUMENTS);
}

View File

@ -136,7 +136,7 @@ GrantedRoles::Elements GrantedRoles::getElements() const
boost::range::set_difference(roles, roles_with_admin_option, std::back_inserter(element.ids));
if (!element.empty())
{
element.admin_option = false;
element.admin_option = false; //-V1048
elements.emplace_back(std::move(element));
}

View File

@ -1,7 +1,7 @@
#pragma once
#include <Access/IAccessEntity.h>
#include <common/types.h>
#include <Core/Types.h>
#include <Core/UUID.h>
#include <ext/scope_guard.h>
#include <functional>

View File

@ -45,7 +45,7 @@ struct Quota : public IAccessEntity
struct ResourceTypeInfo
{
const char * const raw_name;
const char * const raw_name = "";
const String name; /// Lowercased with underscores, e.g. "result_rows".
const String keyword; /// Uppercased with spaces, e.g. "RESULT ROWS".
const bool output_as_float = false;

View File

@ -1,6 +1,7 @@
#pragma once
#include <Core/UUID.h>
#include <Core/Types.h>
#include <boost/container/flat_set.hpp>
#include <memory>
#include <optional>

View File

@ -2,6 +2,7 @@
#include <Access/IAccessEntity.h>
#include <Access/RolesOrUsersSet.h>
#include <Core/Types.h>
#include <array>

View File

@ -7,7 +7,7 @@
#include <DataTypes/DataTypesDecimal.h>
#include <DataTypes/DataTypesNumber.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include "Core/DecimalFunctions.h"
#include <Core/DecimalFunctions.h>
namespace DB

View File

@ -5,18 +5,18 @@
namespace DB
{
template <class T>
template <typename T>
using AvgWeightedFieldType = std::conditional_t<IsDecimalNumber<T>,
std::conditional_t<std::is_same_v<T, Decimal256>, Decimal256, Decimal128>,
std::conditional_t<DecimalOrExtendedInt<T>,
Float64, // no way to do UInt128 * UInt128, better cast to Float64
NearestFieldType<T>>>;
template <class T, class U>
template <typename T, typename U>
using MaxFieldType = std::conditional_t<(sizeof(AvgWeightedFieldType<T>) > sizeof(AvgWeightedFieldType<U>)),
AvgWeightedFieldType<T>, AvgWeightedFieldType<U>>;
template <class Value, class Weight>
template <typename Value, typename Weight>
class AggregateFunctionAvgWeighted final :
public AggregateFunctionAvgBase<
MaxFieldType<Value, Weight>, AvgWeightedFieldType<Weight>, AggregateFunctionAvgWeighted<Value, Weight>>

View File

@ -2,7 +2,6 @@
#include <DataTypes/DataTypesNumber.h>
#include <Columns/ColumnsNumber.h>
#include <Common/FieldVisitors.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <AggregateFunctions/Helpers.h>
@ -20,7 +19,7 @@ namespace ErrorCodes
/** Tracks the leftmost and rightmost (x, y) data points.
*/
struct AggregateFunctionBoundingRatioData
struct AggregateFunctionBoundingRatioData //-V730
{
struct Point
{

View File

@ -10,7 +10,6 @@
#include <Common/HashTable/HashMap.h>
#include <Common/SipHash.h>
#include <Common/FieldVisitors.h>
namespace DB
{

View File

@ -296,12 +296,7 @@ public:
if (size)
{
typename ColumnVector<T>::Container & data_to = assert_cast<ColumnVector<T> &>(arr_to.getData()).getData();
if constexpr (is_big_int_v<T>)
// is data_to empty? we should probably use std::vector::insert then
for (auto it = this->data(place).value.begin(); it != this->data(place).value.end(); it++)
data_to.push_back(*it);
else
data_to.insert(this->data(place).value.begin(), this->data(place).value.end());
data_to.insert(this->data(place).value.begin(), this->data(place).value.end());
}
}

View File

@ -21,6 +21,7 @@ namespace ErrorCodes
namespace
{
/// TODO Proper support for Decimal256.
template <typename T, typename LimitNumberOfElements>
struct MovingSum
{

View File

@ -38,7 +38,7 @@ struct MovingData
using Array = PODArray<T, 32, Allocator>;
Array value; /// Prefix sums.
T sum = 0;
T sum{};
void NO_SANITIZE_UNDEFINED add(T val, Arena * arena)
{
@ -69,9 +69,9 @@ struct MovingAvgData : public MovingData<T>
T NO_SANITIZE_UNDEFINED get(size_t idx, UInt64 window_size) const
{
if (idx < window_size)
return this->value[idx] / window_size;
return this->value[idx] / T(window_size);
else
return (this->value[idx] - this->value[idx - window_size]) / window_size;
return (this->value[idx] - this->value[idx - window_size]) / T(window_size);
}
};

View File

@ -220,7 +220,7 @@ private:
}
public:
AggregateFunctionHistogramData()
AggregateFunctionHistogramData() //-V730
: size(0)
, lower_bound(std::numeric_limits<Mean>::max())
, upper_bound(std::numeric_limits<Mean>::lowest())

View File

@ -6,7 +6,6 @@
#include <Columns/ColumnVector.h>
#include <Columns/ColumnTuple.h>
#include <Common/assert_cast.h>
#include <Common/FieldVisitors.h>
#include <Common/PODArray_fwd.h>
#include <common/types.h>
#include <DataTypes/DataTypesDecimal.h>
@ -21,7 +20,7 @@
#include <Common/ArenaAllocator.h>
#include <iostream>
namespace DB
{

View File

@ -181,7 +181,7 @@ public:
/** For strings. Short strings are stored in the object itself, and long strings are allocated separately.
* NOTE It could also be suitable for arrays of numbers.
*/
struct SingleValueDataString
struct SingleValueDataString //-V730
{
private:
using Self = SingleValueDataString;

View File

@ -100,13 +100,14 @@ AggregateFunctionPtr createAggregateFunctionQuantile(const std::string & name, c
if (which.idx == TypeIndex::Decimal32) return std::make_shared<Function<Decimal32, false>>(argument_types, params);
if (which.idx == TypeIndex::Decimal64) return std::make_shared<Function<Decimal64, false>>(argument_types, params);
if (which.idx == TypeIndex::Decimal128) return std::make_shared<Function<Decimal128, false>>(argument_types, params);
if (which.idx == TypeIndex::Decimal256) return std::make_shared<Function<Decimal256, false>>(argument_types, params);
if (which.idx == TypeIndex::DateTime64) return std::make_shared<Function<DateTime64, false>>(argument_types, params);
//if (which.idx == TypeIndex::Decimal256) return std::make_shared<Function<Decimal256, false>>(argument_types, params);
}
if constexpr (supportBigInt<Function>())
{
if (which.idx == TypeIndex::Int128) return std::make_shared<Function<Int128, true>>(argument_types, params);
if (which.idx == TypeIndex::UInt128) return std::make_shared<Function<Int128, true>>(argument_types, params);
if (which.idx == TypeIndex::Int256) return std::make_shared<Function<Int256, true>>(argument_types, params);
if (which.idx == TypeIndex::UInt256) return std::make_shared<Function<UInt256, true>>(argument_types, params);
}

View File

@ -36,7 +36,7 @@ AggregateFunctionPtr createAggregateFunctionStatisticsBinary(const std::string &
assertNoParameters(name, parameters);
assertBinary(name, argument_types);
AggregateFunctionPtr res(createWithTwoNumericTypes<FunctionTemplate>(*argument_types[0], *argument_types[1], argument_types));
AggregateFunctionPtr res(createWithTwoBasicNumericTypes<FunctionTemplate>(*argument_types[0], *argument_types[1], argument_types));
if (!res)
throw Exception("Illegal types " + argument_types[0]->getName() + " and " + argument_types[1]->getName()
+ " of arguments for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);

View File

@ -13,7 +13,7 @@
namespace DB
{
namespace
namespace detail
{
/// This function returns true if both values are large and comparable.
@ -72,7 +72,7 @@ public:
Float64 factor = static_cast<Float64>(count * source.count) / total_count;
Float64 delta = mean - source.mean;
if (areComparable(count, source.count))
if (detail::areComparable(count, source.count))
mean = (source.count * source.mean + count * mean) / total_count;
else
mean = source.mean + delta * (static_cast<Float64>(count) / total_count);
@ -302,7 +302,7 @@ public:
Float64 left_delta = left_mean - source.left_mean;
Float64 right_delta = right_mean - source.right_mean;
if (areComparable(count, source.count))
if (detail::areComparable(count, source.count))
{
left_mean = (source.count * source.left_mean + count * left_mean) / total_count;
right_mean = (source.count * source.right_mean + count * right_mean) / total_count;

View File

@ -41,7 +41,7 @@ AggregateFunctionPtr createAggregateFunctionStatisticsBinary(const std::string &
assertNoParameters(name, parameters);
assertBinary(name, argument_types);
AggregateFunctionPtr res(createWithTwoNumericTypes<FunctionTemplate>(*argument_types[0], *argument_types[1], argument_types));
AggregateFunctionPtr res(createWithTwoBasicNumericTypes<FunctionTemplate>(*argument_types[0], *argument_types[1], argument_types));
if (!res)
throw Exception("Illegal types " + argument_types[0]->getName() + " and " + argument_types[1]->getName()
+ " of arguments for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);

View File

@ -131,11 +131,10 @@ public:
static_cast<ResultType>(static_cast<const ColVecT2 &>(*columns[1]).getData()[row_num]));
else
{
if constexpr (std::is_same_v<T1, Decimal256>)
if constexpr (IsDecimalNumber<T1>)
{
this->data(place).add(static_cast<ResultType>(
static_cast<const ColVecT1 &>(*columns[0]).getData()[row_num].value
));
static_cast<const ColVecT1 &>(*columns[0]).getData()[row_num].value));
}
else
this->data(place).add(

View File

@ -2,9 +2,10 @@
#include <AggregateFunctions/AggregateFunctionTopK.h>
#include <AggregateFunctions/Helpers.h>
#include <AggregateFunctions/FactoryHelpers.h>
#include <Common/FieldVisitors.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include "registerAggregateFunctions.h"
#define TOP_K_MAX_SIZE 0xFFFFFF

View File

@ -10,7 +10,6 @@
#include <Columns/ColumnArray.h>
#include <Common/SpaceSaving.h>
#include <Common/FieldVisitors.h>
#include <Common/assert_cast.h>
#include <AggregateFunctions/IAggregateFunction.h>

View File

@ -132,6 +132,12 @@ void registerAggregateFunctionsUniq(AggregateFunctionFactory & factory)
factory.registerFunction("uniqExact",
{createAggregateFunctionUniq<true, AggregateFunctionUniqExactData, AggregateFunctionUniqExactData<String>>, properties});
#if USE_DATASKETCHES
factory.registerFunction("uniqThetaSketch",
{createAggregateFunctionUniq<AggregateFunctionUniqThetaSketchData, AggregateFunctionUniqThetaSketchData>, properties});
#endif
}
}

View File

@ -22,6 +22,7 @@
#include <AggregateFunctions/UniquesHashSet.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include <AggregateFunctions/ThetaSketchData.h>
#include <AggregateFunctions/UniqVariadicHash.h>
@ -69,7 +70,7 @@ struct AggregateFunctionUniqHLL12Data<String>
};
template <>
struct AggregateFunctionUniqHLL12Data<UInt128>
struct AggregateFunctionUniqHLL12Data<UUID>
{
using Set = HyperLogLogWithSmallSetOptimization<UInt64, 16, 12>;
Set set;
@ -124,6 +125,19 @@ struct AggregateFunctionUniqExactData<String>
};
/// uniqThetaSketch
#if USE_DATASKETCHES
struct AggregateFunctionUniqThetaSketchData
{
using Set = ThetaSketchData<UInt64>;
Set set;
static String getName() { return "uniqThetaSketch"; }
};
#endif
namespace detail
{
@ -133,16 +147,14 @@ template <typename T> struct AggregateFunctionUniqTraits
{
static UInt64 hash(T x)
{
if constexpr (std::is_same_v<T, UInt128>)
{
return sipHash64(x);
}
else if constexpr (std::is_same_v<T, Float32> || std::is_same_v<T, Float64>)
if constexpr (std::is_same_v<T, Float32> || std::is_same_v<T, Float64>)
{
return ext::bit_cast<UInt64>(x);
}
else if constexpr (sizeof(T) <= sizeof(UInt64))
{
return x;
}
else
return DefaultHash64<T>(x);
}
@ -184,11 +196,17 @@ struct OneAdder
UInt128 key;
SipHash hash;
hash.update(value.data, value.size);
hash.get128(key.low, key.high);
hash.get128(key);
data.set.insert(key);
}
}
#if USE_DATASKETCHES
else if constexpr (std::is_same_v<Data, AggregateFunctionUniqThetaSketchData>)
{
data.set.insertOriginal(column.getDataAt(row_num));
}
#endif
}
};

View File

@ -3,11 +3,13 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/Helpers.h>
#include <Common/FieldVisitors.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include <functional>
#include "registerAggregateFunctions.h"
namespace DB
{

View File

@ -1,7 +1,6 @@
#pragma once
#include <Common/CombinedCardinalityEstimator.h>
#include <Common/FieldVisitors.h>
#include <Common/SipHash.h>
#include <Common/typeid_cast.h>
#include <Common/assert_cast.h>

View File

@ -1,11 +1,11 @@
#include <AggregateFunctions/AggregateFunctionFactory.h>
#include <AggregateFunctions/Helpers.h>
#include <AggregateFunctions/AggregateFunctionUniqUpTo.h>
#include <Common/FieldVisitors.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypeFixedString.h>
#include "registerAggregateFunctions.h"
namespace DB

View File

@ -1,6 +1,7 @@
#pragma once
#include <Common/FieldVisitors.h>
#include <common/unaligned.h>
#include <Common/typeid_cast.h>
#include <Common/assert_cast.h>
@ -35,7 +36,7 @@ namespace DB
*/
template <typename T>
struct __attribute__((__packed__)) AggregateFunctionUniqUpToData
struct AggregateFunctionUniqUpToData
{
/** If count == threshold + 1 - this means that it is "overflowed" (values greater than threshold).
* In this case (for example, after calling the merge function), the `data` array does not necessarily contain the initialized values
@ -43,7 +44,17 @@ struct __attribute__((__packed__)) AggregateFunctionUniqUpToData
* then set count to `threshold + 1`, and values from another state are not copied.
*/
UInt8 count = 0;
T data[0];
char data_ptr[0];
T load(size_t i) const
{
return unalignedLoad<T>(data_ptr + i * sizeof(T));
}
void store(size_t i, const T & x)
{
unalignedStore<T>(data_ptr + i * sizeof(T), x);
}
size_t size() const
{
@ -60,12 +71,12 @@ struct __attribute__((__packed__)) AggregateFunctionUniqUpToData
/// Linear search for the matching element.
for (size_t i = 0; i < count; ++i)
if (data[i] == x)
if (load(i) == x)
return;
/// Did not find the matching element. If there is room for one more element, insert it.
if (count < threshold)
data[count] = x;
store(count, x);
/// After increasing count, the state may be overflowed.
++count;
@ -84,7 +95,7 @@ struct __attribute__((__packed__)) AggregateFunctionUniqUpToData
}
for (size_t i = 0; i < rhs.count; ++i)
insert(rhs.data[i], threshold);
insert(rhs.load(i), threshold);
}
void write(WriteBuffer & wb, UInt8 threshold) const
@ -93,7 +104,7 @@ struct __attribute__((__packed__)) AggregateFunctionUniqUpToData
/// Write values only if the state is not overflowed. Otherwise, they are not needed, and only the fact that the state is overflowed is important.
if (count <= threshold)
wb.write(reinterpret_cast<const char *>(data), count * sizeof(data[0]));
wb.write(data_ptr, count * sizeof(T));
}
void read(ReadBuffer & rb, UInt8 threshold)
@ -101,7 +112,7 @@ struct __attribute__((__packed__)) AggregateFunctionUniqUpToData
readBinary(count, rb);
if (count <= threshold)
rb.read(reinterpret_cast<char *>(data), count * sizeof(data[0]));
rb.read(data_ptr, count * sizeof(T));
}
/// ALWAYS_INLINE is required to have better code layout for uniqUpTo function

View File

@ -15,12 +15,12 @@
M(Float32) \
M(Float64)
// No UInt128 here because of the name conflict
#define FOR_NUMERIC_TYPES(M) \
M(UInt8) \
M(UInt16) \
M(UInt32) \
M(UInt64) \
M(UInt128) \
M(UInt256) \
M(Int8) \
M(Int16) \
@ -109,6 +109,8 @@ static IAggregateFunction * createWithUnsignedIntegerType(const IDataType & argu
if (which.idx == TypeIndex::UInt16) return new AggregateFunctionTemplate<UInt16, Data<UInt16>>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::UInt32) return new AggregateFunctionTemplate<UInt32, Data<UInt32>>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::UInt64) return new AggregateFunctionTemplate<UInt64, Data<UInt64>>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::UInt128) return new AggregateFunctionTemplate<UInt128, Data<UInt128>>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::UInt256) return new AggregateFunctionTemplate<UInt256, Data<UInt256>>(std::forward<TArgs>(args)...);
return nullptr;
}
@ -119,11 +121,11 @@ static IAggregateFunction * createWithNumericBasedType(const IDataType & argumen
if (f)
return f;
/// expects that DataTypeDate based on UInt16, DataTypeDateTime based on UInt32 and UUID based on UInt128
/// expects that DataTypeDate based on UInt16, DataTypeDateTime based on UInt32
WhichDataType which(argument_type);
if (which.idx == TypeIndex::Date) return new AggregateFunctionTemplate<UInt16>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::DateTime) return new AggregateFunctionTemplate<UInt32>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::UUID) return new AggregateFunctionTemplate<UInt128>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::UUID) return new AggregateFunctionTemplate<UUID>(std::forward<TArgs>(args)...);
return nullptr;
}
@ -184,6 +186,29 @@ static IAggregateFunction * createWithTwoNumericTypes(const IDataType & first_ty
return nullptr;
}
template <typename FirstType, template <typename, typename> class AggregateFunctionTemplate, typename... TArgs>
static IAggregateFunction * createWithTwoBasicNumericTypesSecond(const IDataType & second_type, TArgs && ... args)
{
WhichDataType which(second_type);
#define DISPATCH(TYPE) \
if (which.idx == TypeIndex::TYPE) return new AggregateFunctionTemplate<FirstType, TYPE>(std::forward<TArgs>(args)...);
FOR_BASIC_NUMERIC_TYPES(DISPATCH)
#undef DISPATCH
return nullptr;
}
template <template <typename, typename> class AggregateFunctionTemplate, typename... TArgs>
static IAggregateFunction * createWithTwoBasicNumericTypes(const IDataType & first_type, const IDataType & second_type, TArgs && ... args)
{
WhichDataType which(first_type);
#define DISPATCH(TYPE) \
if (which.idx == TypeIndex::TYPE) \
return createWithTwoBasicNumericTypesSecond<TYPE, AggregateFunctionTemplate>(second_type, std::forward<TArgs>(args)...);
FOR_BASIC_NUMERIC_TYPES(DISPATCH)
#undef DISPATCH
return nullptr;
}
template <typename FirstType, template <typename, typename> class AggregateFunctionTemplate, typename... TArgs>
static IAggregateFunction * createWithTwoNumericOrDateTypesSecond(const IDataType & second_type, TArgs && ... args)
{
@ -195,7 +220,7 @@ static IAggregateFunction * createWithTwoNumericOrDateTypesSecond(const IDataTyp
if (which.idx == TypeIndex::Enum8) return new AggregateFunctionTemplate<FirstType, Int8>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::Enum16) return new AggregateFunctionTemplate<FirstType, Int16>(std::forward<TArgs>(args)...);
/// expects that DataTypeDate based on UInt16, DataTypeDateTime based on UInt32 and UUID based on UInt128
/// expects that DataTypeDate based on UInt16, DataTypeDateTime based on UInt32
if (which.idx == TypeIndex::Date) return new AggregateFunctionTemplate<FirstType, UInt16>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::DateTime) return new AggregateFunctionTemplate<FirstType, UInt32>(std::forward<TArgs>(args)...);
@ -216,7 +241,7 @@ static IAggregateFunction * createWithTwoNumericOrDateTypes(const IDataType & fi
if (which.idx == TypeIndex::Enum16)
return createWithTwoNumericOrDateTypesSecond<Int16, AggregateFunctionTemplate>(second_type, std::forward<TArgs>(args)...);
/// expects that DataTypeDate based on UInt16, DataTypeDateTime based on UInt32 and UUID based on UInt128
/// expects that DataTypeDate based on UInt16, DataTypeDateTime based on UInt32
if (which.idx == TypeIndex::Date)
return createWithTwoNumericOrDateTypesSecond<UInt16, AggregateFunctionTemplate>(second_type, std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::DateTime)

View File

@ -263,7 +263,7 @@ struct QuantileExactLow : public QuantileExactBase<Value, QuantileExactLow<Value
{
// sort inputs in ascending order
std::sort(array.begin(), array.end());
size_t n = level < 1 ? level * array.size() : (array.size() - 1);
// if level is 0.5 then compute the "low" median of the sorted array
// by the method of rounding.
if (level == 0.5)
@ -278,10 +278,14 @@ struct QuantileExactLow : public QuantileExactBase<Value, QuantileExactLow<Value
return array[static_cast<size_t>((floor(s / 2)) - 1)];
}
}
// else quantile is the nth index of the sorted array obtained by multiplying
// level and size of array. Example if level = 0.1 and size of array is 10,
// then return array[1].
return array[n];
else
{
// else quantile is the nth index of the sorted array obtained by multiplying
// level and size of array. Example if level = 0.1 and size of array is 10,
// then return array[1].
size_t n = level < 1 ? level * array.size() : (array.size() - 1);
return array[n];
}
}
return std::numeric_limits<Value>::quiet_NaN();
}
@ -295,7 +299,7 @@ struct QuantileExactLow : public QuantileExactBase<Value, QuantileExactLow<Value
for (size_t i = 0; i < size; ++i)
{
auto level = levels[indices[i]];
size_t n = level < 1 ? level * array.size() : (array.size() - 1);
// if level is 0.5 then compute the "low" median of the sorted array
// by the method of rounding.
if (level == 0.5)
@ -310,9 +314,13 @@ struct QuantileExactLow : public QuantileExactBase<Value, QuantileExactLow<Value
result[indices[i]] = array[static_cast<size_t>(floor((s / 2) - 1))];
}
}
// else quantile is the nth index of the sorted array obtained by multiplying
// level and size of array. Example if level = 0.1 and size of array is 10.
result[indices[i]] = array[n];
else
{
// else quantile is the nth index of the sorted array obtained by multiplying
// level and size of array. Example if level = 0.1 and size of array is 10.
size_t n = level < 1 ? level * array.size() : (array.size() - 1);
result[indices[i]] = array[n];
}
}
}
else
@ -337,7 +345,7 @@ struct QuantileExactHigh : public QuantileExactBase<Value, QuantileExactHigh<Val
{
// sort inputs in ascending order
std::sort(array.begin(), array.end());
size_t n = level < 1 ? level * array.size() : (array.size() - 1);
// if level is 0.5 then compute the "high" median of the sorted array
// by the method of rounding.
if (level == 0.5)
@ -345,9 +353,13 @@ struct QuantileExactHigh : public QuantileExactBase<Value, QuantileExactHigh<Val
auto s = array.size();
return array[static_cast<size_t>(floor(s / 2))];
}
// else quantile is the nth index of the sorted array obtained by multiplying
// level and size of array. Example if level = 0.1 and size of array is 10.
return array[n];
else
{
// else quantile is the nth index of the sorted array obtained by multiplying
// level and size of array. Example if level = 0.1 and size of array is 10.
size_t n = level < 1 ? level * array.size() : (array.size() - 1);
return array[n];
}
}
return std::numeric_limits<Value>::quiet_NaN();
}
@ -361,7 +373,7 @@ struct QuantileExactHigh : public QuantileExactBase<Value, QuantileExactHigh<Val
for (size_t i = 0; i < size; ++i)
{
auto level = levels[indices[i]];
size_t n = level < 1 ? level * array.size() : (array.size() - 1);
// if level is 0.5 then compute the "high" median of the sorted array
// by the method of rounding.
if (level == 0.5)
@ -369,9 +381,13 @@ struct QuantileExactHigh : public QuantileExactBase<Value, QuantileExactHigh<Val
auto s = array.size();
result[indices[i]] = array[static_cast<size_t>(floor(s / 2))];
}
// else quantile is the nth index of the sorted array obtained by multiplying
// level and size of array. Example if level = 0.1 and size of array is 10.
result[indices[i]] = array[n];
else
{
// else quantile is the nth index of the sorted array obtained by multiplying
// level and size of array. Example if level = 0.1 and size of array is 10.
size_t n = level < 1 ? level * array.size() : (array.size() - 1);
result[indices[i]] = array[n];
}
}
}
else

View File

@ -54,7 +54,7 @@ struct QuantileReservoirSampler
/// Get the value of the `level` quantile. The level must be between 0 and 1.
Value get(Float64 level)
{
return data.quantileInterpolated(level);
return Value(data.quantileInterpolated(level));
}
/// Get the `size` values of `levels` quantiles. Write `size` results starting with `result` address.
@ -62,7 +62,7 @@ struct QuantileReservoirSampler
void getMany(const Float64 * levels, const size_t * indices, size_t size, Value * result)
{
for (size_t i = 0; i < size; ++i)
result[indices[i]] = data.quantileInterpolated(levels[indices[i]]);
result[indices[i]] = Value(data.quantileInterpolated(levels[indices[i]]));
}
/// The same, but in the case of an empty state, NaN is returned.

View File

@ -131,12 +131,20 @@ public:
size_t left_index = static_cast<size_t>(index);
size_t right_index = left_index + 1;
if (right_index == samples.size())
return static_cast<double>(samples[left_index]);
{
if constexpr (DB::IsDecimalNumber<T>)
return static_cast<double>(samples[left_index].value);
else
return static_cast<double>(samples[left_index]);
}
double left_coef = right_index - index;
double right_coef = index - left_index;
return static_cast<double>(samples[left_index]) * left_coef + static_cast<double>(samples[right_index]) * right_coef;
if constexpr (DB::IsDecimalNumber<T>)
return static_cast<double>(samples[left_index].value) * left_coef + static_cast<double>(samples[right_index].value) * right_coef;
else
return static_cast<double>(samples[left_index]) * left_coef + static_cast<double>(samples[right_index]) * right_coef;
}
void merge(const ReservoirSampler<T, OnEmpty> & b)

View File

@ -0,0 +1,119 @@
#pragma once
#if !defined(ARCADIA_BUILD)
# include <Common/config.h>
#endif
#if USE_DATASKETCHES
#include <boost/noncopyable.hpp>
#include <memory>
#include <theta_sketch.hpp> // Y_IGNORE
#include <theta_union.hpp> // Y_IGNORE
namespace DB
{
template <typename Key>
class ThetaSketchData : private boost::noncopyable
{
private:
std::unique_ptr<datasketches::update_theta_sketch> sk_update;
std::unique_ptr<datasketches::theta_union> sk_union;
inline datasketches::update_theta_sketch * getSkUpdate()
{
if (!sk_update)
sk_update = std::make_unique<datasketches::update_theta_sketch>(datasketches::update_theta_sketch::builder().build());
return sk_update.get();
}
inline datasketches::theta_union * getSkUnion()
{
if (!sk_union)
sk_union = std::make_unique<datasketches::theta_union>(datasketches::theta_union::builder().build());
return sk_union.get();
}
public:
using value_type = Key;
ThetaSketchData() = default;
~ThetaSketchData() = default;
/// Insert original value without hash, as `datasketches::update_theta_sketch.update` will do the hash internal.
void insertOriginal(const StringRef & value)
{
getSkUpdate()->update(value.data, value.size);
}
/// Note that `datasketches::update_theta_sketch.update` will do the hash again.
void insert(Key value)
{
getSkUpdate()->update(value);
}
UInt64 size() const
{
if (sk_union)
return static_cast<UInt64>(sk_union->get_result().get_estimate());
else if (sk_update)
return static_cast<UInt64>(sk_update->get_estimate());
else
return 0;
}
void merge(const ThetaSketchData & rhs)
{
datasketches::theta_union * u = getSkUnion();
if (sk_update)
{
u->update(*sk_update);
sk_update.reset(nullptr);
}
if (rhs.sk_update)
u->update(*rhs.sk_update);
else if (rhs.sk_union)
u->update(rhs.sk_union->get_result());
}
/// You can only call for an empty object.
void read(DB::ReadBuffer & in)
{
datasketches::compact_theta_sketch::vector_bytes bytes;
readVectorBinary(bytes, in);
if (!bytes.empty())
{
auto sk = datasketches::compact_theta_sketch::deserialize(bytes.data(), bytes.size());
getSkUnion()->update(sk);
}
}
void write(DB::WriteBuffer & out) const
{
if (sk_update)
{
auto bytes = sk_update->compact().serialize();
writeVectorBinary(bytes, out);
}
else if (sk_union)
{
auto bytes = sk_union->get_result().serialize();
writeVectorBinary(bytes, out);
}
else
{
datasketches::compact_theta_sketch::vector_bytes bytes;
writeVectorBinary(bytes, out);
}
}
};
}
#endif

View File

@ -3,7 +3,6 @@
#include <city.h>
#include <Core/Defines.h>
#include <Common/SipHash.h>
#include <Common/UInt128.h>
#include <Common/assert_cast.h>
#include <Columns/ColumnTuple.h>
@ -107,7 +106,7 @@ struct UniqVariadicHash<true, false>
}
UInt128 key;
hash.get128(key.low, key.high);
hash.get128(key);
return key;
}
};
@ -131,7 +130,7 @@ struct UniqVariadicHash<true, true>
}
UInt128 key;
hash.get128(key.low, key.high);
hash.get128(key);
return key;
}
};

View File

@ -184,6 +184,7 @@ add_object_library(clickhouse_disks Disks)
add_object_library(clickhouse_interpreters Interpreters)
add_object_library(clickhouse_interpreters_mysql Interpreters/MySQL)
add_object_library(clickhouse_interpreters_clusterproxy Interpreters/ClusterProxy)
add_object_library(clickhouse_interpreters_jit Interpreters/JIT)
add_object_library(clickhouse_columns Columns)
add_object_library(clickhouse_storages Storages)
add_object_library(clickhouse_storages_distributed Storages/Distributed)

View File

@ -85,8 +85,8 @@ private:
{}
public:
const char * getFamilyName() const override { return TypeName<T>::get(); }
TypeIndex getDataType() const override { return TypeId<T>::value; }
const char * getFamilyName() const override { return TypeName<T>; }
TypeIndex getDataType() const override { return TypeId<T>; }
bool isNumeric() const override { return false; }
bool canBeInsideNullable() const override { return true; }

View File

@ -665,7 +665,7 @@ UInt128 ColumnUnique<ColumnType>::IncrementalHash::getHash(const ColumnType & co
column.updateHashWithValue(i, sip_hash);
std::lock_guard lock(mutex);
sip_hash.get128(hash.low, hash.high);
sip_hash.get128(hash);
cur_hash = hash;
num_added_rows.store(column_size);
}

View File

@ -33,6 +33,7 @@ namespace ErrorCodes
extern const int PARAMETER_OUT_OF_BOUND;
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
extern const int LOGICAL_ERROR;
extern const int NOT_IMPLEMENTED;
}
template <typename T>
@ -154,7 +155,7 @@ void ColumnVector<T>::getPermutation(bool reverse, size_t limit, int nan_directi
else
{
/// A case for radix sort
if constexpr (is_arithmetic_v<T> && !std::is_same_v<T, UInt128>)
if constexpr (is_arithmetic_v<T> && !is_big_int_v<T>)
{
/// Thresholds on size. Lower threshold is arbitrary. Upper threshold is chosen by the type for histogram counters.
if (s >= 256 && s <= std::numeric_limits<UInt32>::max())
@ -292,28 +293,37 @@ MutableColumnPtr ColumnVector<T>::cloneResized(size_t size) const
memcpy(new_col.data.data(), data.data(), count * sizeof(data[0]));
if (size > count)
memset(static_cast<void *>(&new_col.data[count]), static_cast<int>(ValueType()), (size - count) * sizeof(ValueType));
memset(static_cast<void *>(&new_col.data[count]), 0, (size - count) * sizeof(ValueType));
}
return res;
}
template <typename T>
UInt64 ColumnVector<T>::get64(size_t n) const
UInt64 ColumnVector<T>::get64(size_t n [[maybe_unused]]) const
{
return ext::bit_cast<UInt64>(data[n]);
if constexpr (is_arithmetic_v<T>)
return ext::bit_cast<UInt64>(data[n]);
else
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get the value of {} as UInt64", TypeName<T>);
}
template <typename T>
inline Float64 ColumnVector<T>::getFloat64(size_t n) const
inline Float64 ColumnVector<T>::getFloat64(size_t n [[maybe_unused]]) const
{
return static_cast<Float64>(data[n]);
if constexpr (is_arithmetic_v<T>)
return static_cast<Float64>(data[n]);
else
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get the value of {} as Float64", TypeName<T>);
}
template <typename T>
Float32 ColumnVector<T>::getFloat32(size_t n) const
Float32 ColumnVector<T>::getFloat32(size_t n [[maybe_unused]]) const
{
return static_cast<Float32>(data[n]);
if constexpr (is_arithmetic_v<T>)
return static_cast<Float32>(data[n]);
else
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get the value of {} as Float32", TypeName<T>);
}
template <typename T>
@ -482,8 +492,6 @@ void ColumnVector<T>::gather(ColumnGathererStream & gatherer)
template <typename T>
void ColumnVector<T>::getExtremes(Field & min, Field & max) const
{
using FastRefT = std::conditional_t<is_big_int_v<T>, const T &, const T>;
size_t size = data.size();
if (size == 0)
@ -504,7 +512,7 @@ void ColumnVector<T>::getExtremes(Field & min, Field & max) const
T cur_min = NaNOrZero<T>();
T cur_max = NaNOrZero<T>();
for (FastRefT x : data)
for (const T & x : data)
{
if (isNaN(x))
continue;
@ -569,5 +577,6 @@ template class ColumnVector<Int128>;
template class ColumnVector<Int256>;
template class ColumnVector<Float32>;
template class ColumnVector<Float64>;
template class ColumnVector<UUID>;
}

View File

@ -12,6 +12,12 @@
namespace DB
{
namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
}
/** Stuff for comparing numbers.
* Integer values are compared as usual.
* Floating-point numbers are compared this way that NaNs always end up at the end
@ -90,6 +96,7 @@ struct FloatCompareHelper
template <class U> struct CompareHelper<Float32, U> : public FloatCompareHelper<Float32> {};
template <class U> struct CompareHelper<Float64, U> : public FloatCompareHelper<Float64> {};
/** A template for columns that use a simple array to store.
*/
template <typename T>
@ -118,7 +125,7 @@ private:
ColumnVector(std::initializer_list<T> il) : data{il} {}
public:
bool isNumeric() const override { return IsNumber<T>; }
bool isNumeric() const override { return is_arithmetic_v<T>; }
size_t size() const override
{
@ -221,8 +228,8 @@ public:
data.reserve(n);
}
const char * getFamilyName() const override { return TypeName<T>::get(); }
TypeIndex getDataType() const override { return TypeId<T>::value; }
const char * getFamilyName() const override { return TypeName<T>; }
TypeIndex getDataType() const override { return TypeId<T>; }
MutableColumnPtr cloneResized(size_t size) const override;
@ -245,18 +252,27 @@ public:
/// Out of range conversion is permitted.
UInt64 NO_SANITIZE_UNDEFINED getUInt(size_t n) const override
{
return UInt64(data[n]);
if constexpr (is_arithmetic_v<T>)
return UInt64(data[n]);
else
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get the value of {} as UInt", TypeName<T>);
}
/// Out of range conversion is permitted.
Int64 NO_SANITIZE_UNDEFINED getInt(size_t n) const override
{
return Int64(data[n]);
if constexpr (is_arithmetic_v<T>)
return Int64(data[n]);
else
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get the value of {} as Int", TypeName<T>);
}
bool getBool(size_t n) const override
{
return bool(data[n]);
if constexpr (is_arithmetic_v<T>)
return bool(data[n]);
else
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get the value of {} as bool", TypeName<T>);
}
void insert(const Field & x) override
@ -370,5 +386,6 @@ extern template class ColumnVector<Int128>;
extern template class ColumnVector<Int256>;
extern template class ColumnVector<Float32>;
extern template class ColumnVector<Float64>;
extern template class ColumnVector<UUID>;
}

View File

@ -26,6 +26,6 @@ using ColumnInt256 = ColumnVector<Int256>;
using ColumnFloat32 = ColumnVector<Float32>;
using ColumnFloat64 = ColumnVector<Float64>;
using ColumnUUID = ColumnVector<UInt128>;
using ColumnUUID = ColumnVector<UUID>;
}

View File

@ -1,7 +1,6 @@
#pragma once
#include <optional>
#include <Columns/IColumn.h>
#include <Common/UInt128.h>
namespace DB
{

View File

@ -14,11 +14,11 @@
#include <DataTypes/DataTypesNumber.h>
#include <Common/WeakHash.h>
#include <Common/hex.h>
#include <unordered_map>
#include <iostream>
#include <sstream>
#include <Common/hex.h>
using namespace DB;
@ -27,20 +27,11 @@ template <typename T>
void checkColumn(
const WeakHash32::Container & hash,
const PaddedPODArray<T> & eq_class,
const std::function<std::string(size_t)> & print_function,
size_t allowed_collisions = 0,
size_t max_collisions_to_print = 10)
{
ASSERT_EQ(hash.size(), eq_class.size());
auto print_for_row = [&](size_t row)
{
std::string res = "row: " + std::to_string(row);
res += " hash: " + std::to_string(hash[row]);
res += " value: " + print_function(row);
return res;
};
/// Check equal rows has equal hash.
{
std::unordered_map<T, UInt32> map;
@ -55,11 +46,7 @@ void checkColumn(
else
{
if (it->second != hash[i])
{
std::cout << "Different hashes for the same equivalent class (" << size_t(val) << "):\n";
std::cout << print_for_row(it->first) << '\n';
std::cout << print_for_row(i) << std::endl;
}
std::cout << "Different hashes for the same equivalent class (" << toString(val) << ")\n";
ASSERT_EQ(it->second, hash[i]);
}
@ -88,7 +75,6 @@ void checkColumn(
if (num_collisions <= max_collisions_to_print)
{
collisions_str << "Collision:\n";
collisions_str << print_for_row(i) << std::endl;
}
if (num_collisions > allowed_collisions)
@ -117,7 +103,7 @@ TEST(WeakHash32, ColumnVectorU8)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnVectorI8)
@ -134,7 +120,7 @@ TEST(WeakHash32, ColumnVectorI8)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnVectorU16)
@ -151,7 +137,7 @@ TEST(WeakHash32, ColumnVectorU16)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnVectorI16)
@ -168,7 +154,7 @@ TEST(WeakHash32, ColumnVectorI16)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnVectorU32)
@ -185,7 +171,7 @@ TEST(WeakHash32, ColumnVectorU32)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnVectorI32)
@ -202,7 +188,7 @@ TEST(WeakHash32, ColumnVectorI32)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnVectorU64)
@ -219,7 +205,7 @@ TEST(WeakHash32, ColumnVectorU64)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnVectorI64)
@ -236,7 +222,7 @@ TEST(WeakHash32, ColumnVectorI64)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnVectorU128)
@ -250,7 +236,9 @@ TEST(WeakHash32, ColumnVectorU128)
{
for (uint64_t i = 0; i < 65536; ++i)
{
UInt128 val(i << 32u, i << 32u);
UInt128 val;
val.items[0] = i << 32u;
val.items[1] = i << 32u;
data.push_back(val);
eq_data.push_back(i);
}
@ -259,7 +247,7 @@ TEST(WeakHash32, ColumnVectorU128)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), eq_data, [&](size_t row) { return col->getElement(row).toHexString(); });
checkColumn(hash.getData(), eq_data);
}
TEST(WeakHash32, ColumnVectorI128)
@ -276,7 +264,7 @@ TEST(WeakHash32, ColumnVectorI128)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(Int64(col->getElement(row))); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnDecimal32)
@ -293,7 +281,7 @@ TEST(WeakHash32, ColumnDecimal32)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnDecimal64)
@ -310,7 +298,7 @@ TEST(WeakHash32, ColumnDecimal64)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(col->getElement(row)); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnDecimal128)
@ -327,7 +315,7 @@ TEST(WeakHash32, ColumnDecimal128)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), col->getData(), [&](size_t row) { return std::to_string(Int64(col->getElement(row))); });
checkColumn(hash.getData(), col->getData());
}
TEST(WeakHash32, ColumnString1)
@ -349,7 +337,7 @@ TEST(WeakHash32, ColumnString1)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), data, [&](size_t row) { return col->getDataAt(row).toString(); });
checkColumn(hash.getData(), data);
}
TEST(WeakHash32, ColumnString2)
@ -389,7 +377,7 @@ TEST(WeakHash32, ColumnString2)
/// Now there is single collision between 'k' * 544 and 'q' * 2512 (which is calculated twice)
size_t allowed_collisions = 4;
checkColumn(hash.getData(), data, [&](size_t row) { return col->getDataAt(row).toString(); }, allowed_collisions);
checkColumn(hash.getData(), data, allowed_collisions);
}
TEST(WeakHash32, ColumnString3)
@ -427,7 +415,7 @@ TEST(WeakHash32, ColumnString3)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), data, [&](size_t row) { return col->getDataAt(row).toString(); });
checkColumn(hash.getData(), data);
}
TEST(WeakHash32, ColumnFixedString)
@ -455,7 +443,7 @@ TEST(WeakHash32, ColumnFixedString)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), data, [&](size_t row) { return col->getDataAt(row).toString(); });
checkColumn(hash.getData(), data);
}
TEST(WeakHash32, ColumnArray)
@ -502,15 +490,7 @@ TEST(WeakHash32, ColumnArray)
WeakHash32 hash(col_arr->size());
col_arr->updateWeakHash32(hash);
auto print_function = [&col_arr](size_t row)
{
auto & offsets = col_arr->getOffsets();
size_t s = offsets[row] - offsets[row - 1];
auto value = col_arr->getData().getUInt(offsets[row]);
return std::string("[array of size ") + std::to_string(s) + " with values " + std::to_string(value) + "]";
};
checkColumn(hash.getData(), eq_data, print_function);
checkColumn(hash.getData(), eq_data);
}
TEST(WeakHash32, ColumnArray2)
@ -545,15 +525,7 @@ TEST(WeakHash32, ColumnArray2)
WeakHash32 hash(col_arr->size());
col_arr->updateWeakHash32(hash);
auto print_function = [&col_arr](size_t row)
{
auto & offsets = col_arr->getOffsets();
auto value1 = col_arr->getData().getUInt(offsets[row]);
auto value2 = col_arr->getData().getUInt(offsets[row] + 1);
return std::string("[") + std::to_string(value1) + ", " + std::to_string(value2) + "]";
};
checkColumn(hash.getData(), eq_data, print_function);
checkColumn(hash.getData(), eq_data);
}
TEST(WeakHash32, ColumnArrayArray)
@ -610,20 +582,7 @@ TEST(WeakHash32, ColumnArrayArray)
WeakHash32 hash(col_arr_arr->size());
col_arr_arr->updateWeakHash32(hash);
auto print_function = [&col_arr_arr](size_t row2)
{
auto & offsets2 = col_arr_arr->getOffsets();
size_t s2 = offsets2[row2] - offsets2[row2 - 1];
const auto & arr2 = typeid_cast<const ColumnArray &>(col_arr_arr->getData());
const auto & offsets = arr2.getOffsets();
size_t row = offsets2[row2];
size_t s = offsets[row] - offsets[row - 1];
auto value = arr2.getData().getUInt(offsets[row]);
return std::string("[array of size ") + std::to_string(s2) + " with values ["
"[[array of size " + std::to_string(s) + " with values " + std::to_string(value) + "]]";
};
checkColumn(hash.getData(), eq_data, print_function);
checkColumn(hash.getData(), eq_data);
}
TEST(WeakHash32, ColumnConst)
@ -642,7 +601,7 @@ TEST(WeakHash32, ColumnConst)
WeakHash32 hash(col_const->size());
col_const->updateWeakHash32(hash);
checkColumn(hash.getData(), data, [&](size_t) { return std::to_string(0); });
checkColumn(hash.getData(), data);
}
TEST(WeakHash32, ColumnLowcardinality)
@ -663,7 +622,7 @@ TEST(WeakHash32, ColumnLowcardinality)
WeakHash32 hash(col->size());
col->updateWeakHash32(hash);
checkColumn(hash.getData(), data, [&](size_t row) { return std::to_string(col->getUInt(row)); });
checkColumn(hash.getData(), data);
}
TEST(WeakHash32, ColumnNullable)
@ -689,7 +648,7 @@ TEST(WeakHash32, ColumnNullable)
WeakHash32 hash(col_null->size());
col_null->updateWeakHash32(hash);
checkColumn(hash.getData(), eq, [&](size_t row) { return eq[row] == -1 ? "Null" : std::to_string(eq[row]); });
checkColumn(hash.getData(), eq);
}
TEST(WeakHash32, ColumnTupleUInt64UInt64)
@ -720,14 +679,7 @@ TEST(WeakHash32, ColumnTupleUInt64UInt64)
WeakHash32 hash(col_tuple->size());
col_tuple->updateWeakHash32(hash);
auto print_func = [&](size_t row)
{
std::string l = std::to_string(col_tuple->getColumn(0).getUInt(row));
std::string r = std::to_string(col_tuple->getColumn(1).getUInt(row));
return "(" + l + ", " + r + ")";
};
checkColumn(hash.getData(), eq, print_func);
checkColumn(hash.getData(), eq);
}
TEST(WeakHash32, ColumnTupleUInt64String)
@ -765,15 +717,8 @@ TEST(WeakHash32, ColumnTupleUInt64String)
WeakHash32 hash(col_tuple->size());
col_tuple->updateWeakHash32(hash);
auto print_func = [&](size_t row)
{
std::string l = std::to_string(col_tuple->getColumn(0).getUInt(row));
std::string r = col_tuple->getColumn(1).getDataAt(row).toString();
return "(" + l + ", " + r + ")";
};
size_t allowed_collisions = 8;
checkColumn(hash.getData(), eq, print_func, allowed_collisions);
checkColumn(hash.getData(), eq, allowed_collisions);
}
TEST(WeakHash32, ColumnTupleUInt64FixedString)
@ -811,14 +756,7 @@ TEST(WeakHash32, ColumnTupleUInt64FixedString)
WeakHash32 hash(col_tuple->size());
col_tuple->updateWeakHash32(hash);
auto print_func = [&](size_t row)
{
std::string l = std::to_string(col_tuple->getColumn(0).getUInt(row));
std::string r = col_tuple->getColumn(1).getDataAt(row).toString();
return "(" + l + ", " + r + ")";
};
checkColumn(hash.getData(), eq, print_func);
checkColumn(hash.getData(), eq);
}
TEST(WeakHash32, ColumnTupleUInt64Array)
@ -865,23 +803,10 @@ TEST(WeakHash32, ColumnTupleUInt64Array)
WeakHash32 hash(col_tuple->size());
col_tuple->updateWeakHash32(hash);
auto print_func = [&](size_t row)
{
std::string l = std::to_string(col_tuple->getColumn(0).getUInt(row));
const auto * col_arr = typeid_cast<const ColumnArray *>(col_tuple->getColumnPtr(1).get());
const auto & offsets = col_arr->getOffsets();
size_t s = offsets[row] - offsets[row - 1];
auto value = col_arr->getData().getUInt(offsets[row]);
auto r = std::string("[array of size ") + std::to_string(s) + " with values " + std::to_string(value) + "]";
return "(" + l + ", " + r + ")";
};
/// There are 2 collisions right now (repeated 2 times each):
/// (0, [array of size 1212 with values 7]) vs (0, [array of size 2265 with values 17])
/// (0, [array of size 558 with values 5]) vs (1, [array of size 879 with values 21])
size_t allowed_collisions = 8;
checkColumn(hash.getData(), eq_data, print_func, allowed_collisions);
checkColumn(hash.getData(), eq_data, allowed_collisions);
}

View File

@ -166,8 +166,7 @@ public:
size_t operator()(const DictionaryKey & key) const
{
SipHash hash;
hash.update(key.hash.low);
hash.update(key.hash.high);
hash.update(key.hash);
hash.update(key.size);
return hash.get64();
}

View File

@ -159,12 +159,12 @@ private:
/// The number of bytes read.
size_t read_count = 0;
/// The content in the current position.
UInt8 value_l;
UInt8 value_r;
UInt8 value_l = 0;
UInt8 value_r = 0;
///
bool is_eof = false;
/// Does the cell fully fit into one byte?
bool fits_in_byte;
bool fits_in_byte = false;
};
/** TODO This code looks very suboptimal.

View File

@ -804,7 +804,7 @@ bool Dwarf::findLocation(
findSubProgramDieForAddress(cu, die, address, base_addr_cu, subprogram);
// Subprogram is the DIE of caller function.
if (check_inline && subprogram.abbr.has_children)
if (/*check_inline &&*/ subprogram.abbr.has_children)
{
// Use an extra location and get its call file and call line, so that
// they can be used for the second last location when we don't have
@ -832,7 +832,7 @@ bool Dwarf::findLocation(
// file+line of the non-inlined outer function making the call.
// locationInfo.name is already set by the caller by looking up the
// non-inlined function @address belongs to.
info.has_file_and_line = true;
info.has_file_and_line = true; //-V1048
info.file = call_locations[0].file;
info.line = call_locations[0].line;

View File

@ -121,7 +121,7 @@ public:
struct CallLocation
{
Path file = {};
uint64_t line;
uint64_t line = 0;
std::string_view name;
};
@ -202,8 +202,8 @@ private:
// Abbreviation for a Debugging Information Entry.
struct DIEAbbreviation
{
uint64_t code;
uint64_t tag;
uint64_t code = 0;
uint64_t tag = 0;
bool has_children = false;
std::string_view attributes;

View File

@ -29,7 +29,7 @@ namespace ErrorCodes
struct Error
{
/// Number of times Exception with this ErrorCode had been throw.
Value count;
Value count = 0;
/// Time of the last error.
UInt64 error_time_ms = 0;
/// Message for the last error.

View File

@ -1,4 +1,3 @@
#include <Core/UUID.h>
#include <IO/ReadBuffer.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteHelpers.h>
@ -27,7 +26,7 @@ template <typename T>
static inline String formatQuotedWithPrefix(T x, const char * prefix)
{
WriteBufferFromOwnString wb;
wb.write(prefix, strlen(prefix));
writeCString(prefix, wb);
writeQuoted(x, wb);
return wb.str();
}
@ -48,10 +47,11 @@ String FieldVisitorDump::operator() (const DecimalField<Decimal32> & x) const {
String FieldVisitorDump::operator() (const DecimalField<Decimal64> & x) const { return formatQuotedWithPrefix(x, "Decimal64_"); }
String FieldVisitorDump::operator() (const DecimalField<Decimal128> & x) const { return formatQuotedWithPrefix(x, "Decimal128_"); }
String FieldVisitorDump::operator() (const DecimalField<Decimal256> & x) const { return formatQuotedWithPrefix(x, "Decimal256_"); }
String FieldVisitorDump::operator() (const UInt128 & x) const { return formatQuotedWithPrefix(x, "UInt128_"); }
String FieldVisitorDump::operator() (const UInt256 & x) const { return formatQuotedWithPrefix(x, "UInt256_"); }
String FieldVisitorDump::operator() (const Int256 & x) const { return formatQuotedWithPrefix(x, "Int256_"); }
String FieldVisitorDump::operator() (const Int128 & x) const { return formatQuotedWithPrefix(x, "Int128_"); }
String FieldVisitorDump::operator() (const UInt128 & x) const { return formatQuotedWithPrefix(UUID(x), "UUID_"); }
String FieldVisitorDump::operator() (const Int256 & x) const { return formatQuotedWithPrefix(x, "Int256_"); }
String FieldVisitorDump::operator() (const UUID & x) const { return formatQuotedWithPrefix(x, "UUID_"); }
String FieldVisitorDump::operator() (const String & x) const
@ -152,13 +152,11 @@ String FieldVisitorToString::operator() (const DecimalField<Decimal64> & x) cons
String FieldVisitorToString::operator() (const DecimalField<Decimal128> & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const DecimalField<Decimal256> & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const Int128 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const UInt128 & x) const { return formatQuoted(UUID(x)); }
String FieldVisitorToString::operator() (const AggregateFunctionStateData & x) const
{
return formatQuoted(x.data);
}
String FieldVisitorToString::operator() (const UInt128 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const UInt256 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const Int256 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const UUID & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const AggregateFunctionStateData & x) const { return formatQuoted(x.data); }
String FieldVisitorToString::operator() (const Array & x) const
{
@ -228,6 +226,7 @@ void FieldVisitorWriteBinary::operator() (const UInt128 & x, WriteBuffer & buf)
void FieldVisitorWriteBinary::operator() (const Int128 & x, WriteBuffer & buf) const { DB::writeVarInt(x, buf); }
void FieldVisitorWriteBinary::operator() (const UInt256 & x, WriteBuffer & buf) const { DB::writeBinary(x, buf); }
void FieldVisitorWriteBinary::operator() (const Int256 & x, WriteBuffer & buf) const { DB::writeBinary(x, buf); }
void FieldVisitorWriteBinary::operator() (const UUID & x, WriteBuffer & buf) const { DB::writeBinary(x, buf); }
void FieldVisitorWriteBinary::operator() (const DecimalField<Decimal32> & x, WriteBuffer & buf) const { DB::writeBinary(x.getValue(), buf); }
void FieldVisitorWriteBinary::operator() (const DecimalField<Decimal64> & x, WriteBuffer & buf) const { DB::writeBinary(x.getValue(), buf); }
void FieldVisitorWriteBinary::operator() (const DecimalField<Decimal128> & x, WriteBuffer & buf) const { DB::writeBinary(x.getValue(), buf); }
@ -315,6 +314,13 @@ void FieldVisitorHash::operator() (const Int128 & x) const
hash.update(x);
}
void FieldVisitorHash::operator() (const UUID & x) const
{
UInt8 type = Field::Types::UUID;
hash.update(type);
hash.update(x);
}
void FieldVisitorHash::operator() (const Float64 & x) const
{
UInt8 type = Field::Types::Float64;

View File

@ -19,14 +19,6 @@ namespace ErrorCodes
extern const int NOT_IMPLEMENTED;
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wredundant-decls"
// Just dont mess with it. If the redundant redeclaration is removed then ReaderHelpers.h should be included.
// This leads to Arena.h inclusion which has a problem with ASAN stuff included properly and messing macro definition
// which intefrers with... You dont want to know, really.
UInt128 stringToUUID(const String & str);
#pragma GCC diagnostic pop
/** StaticVisitor (and its descendants) - class with overloaded operator() for all types of fields.
* You could call visitor for field using function 'applyVisitor'.
@ -71,8 +63,11 @@ public:
String operator() (const Null & x) const;
String operator() (const UInt64 & x) const;
String operator() (const UInt128 & x) const;
String operator() (const UInt256 & x) const;
String operator() (const Int64 & x) const;
String operator() (const Int128 & x) const;
String operator() (const Int256 & x) const;
String operator() (const UUID & x) const;
String operator() (const Float64 & x) const;
String operator() (const String & x) const;
String operator() (const Array & x) const;
@ -83,9 +78,6 @@ public:
String operator() (const DecimalField<Decimal128> & x) const;
String operator() (const DecimalField<Decimal256> & x) const;
String operator() (const AggregateFunctionStateData & x) const;
String operator() (const UInt256 & x) const;
String operator() (const Int256 & x) const;
};
@ -95,8 +87,11 @@ public:
void operator() (const Null & x, WriteBuffer & buf) const;
void operator() (const UInt64 & x, WriteBuffer & buf) const;
void operator() (const UInt128 & x, WriteBuffer & buf) const;
void operator() (const UInt256 & x, WriteBuffer & buf) const;
void operator() (const Int64 & x, WriteBuffer & buf) const;
void operator() (const Int128 & x, WriteBuffer & buf) const;
void operator() (const Int256 & x, WriteBuffer & buf) const;
void operator() (const UUID & x, WriteBuffer & buf) const;
void operator() (const Float64 & x, WriteBuffer & buf) const;
void operator() (const String & x, WriteBuffer & buf) const;
void operator() (const Array & x, WriteBuffer & buf) const;
@ -107,9 +102,6 @@ public:
void operator() (const DecimalField<Decimal128> & x, WriteBuffer & buf) const;
void operator() (const DecimalField<Decimal256> & x, WriteBuffer & buf) const;
void operator() (const AggregateFunctionStateData & x, WriteBuffer & buf) const;
void operator() (const UInt256 & x, WriteBuffer & buf) const;
void operator() (const Int256 & x, WriteBuffer & buf) const;
};
@ -120,8 +112,11 @@ public:
String operator() (const Null & x) const;
String operator() (const UInt64 & x) const;
String operator() (const UInt128 & x) const;
String operator() (const UInt256 & x) const;
String operator() (const Int64 & x) const;
String operator() (const Int128 & x) const;
String operator() (const Int256 & x) const;
String operator() (const UUID & x) const;
String operator() (const Float64 & x) const;
String operator() (const String & x) const;
String operator() (const Array & x) const;
@ -132,9 +127,6 @@ public:
String operator() (const DecimalField<Decimal128> & x) const;
String operator() (const DecimalField<Decimal256> & x) const;
String operator() (const AggregateFunctionStateData & x) const;
String operator() (const UInt256 & x) const;
String operator() (const Int256 & x) const;
};
@ -171,6 +163,7 @@ public:
T operator() (const UInt64 & x) const { return T(x); }
T operator() (const Int64 & x) const { return T(x); }
T operator() (const Int128 & x) const { return T(x); }
T operator() (const UUID & x) const { return T(x.toUnderType()); }
T operator() (const Float64 & x) const
{
@ -259,8 +252,11 @@ public:
void operator() (const Null & x) const;
void operator() (const UInt64 & x) const;
void operator() (const UInt128 & x) const;
void operator() (const UInt256 & x) const;
void operator() (const Int64 & x) const;
void operator() (const Int128 & x) const;
void operator() (const Int256 & x) const;
void operator() (const UUID & x) const;
void operator() (const Float64 & x) const;
void operator() (const String & x) const;
void operator() (const Array & x) const;
@ -271,9 +267,6 @@ public:
void operator() (const DecimalField<Decimal128> & x) const;
void operator() (const DecimalField<Decimal256> & x) const;
void operator() (const AggregateFunctionStateData & x) const;
void operator() (const UInt256 & x) const;
void operator() (const Int256 & x) const;
};
@ -309,7 +302,7 @@ public:
bool operator() (Array &) const { throw Exception("Cannot sum Arrays", ErrorCodes::LOGICAL_ERROR); }
bool operator() (Tuple &) const { throw Exception("Cannot sum Tuples", ErrorCodes::LOGICAL_ERROR); }
bool operator() (Map &) const { throw Exception("Cannot sum Maps", ErrorCodes::LOGICAL_ERROR); }
bool operator() (UInt128 &) const { throw Exception("Cannot sum UUIDs", ErrorCodes::LOGICAL_ERROR); }
bool operator() (UUID &) const { throw Exception("Cannot sum UUIDs", ErrorCodes::LOGICAL_ERROR); }
bool operator() (AggregateFunctionStateData &) const { throw Exception("Cannot sum AggregateFunctionStates", ErrorCodes::LOGICAL_ERROR); }
bool operator() (Int128 & x) const

View File

@ -36,41 +36,30 @@ public:
if constexpr (std::is_arithmetic_v<T> && std::is_arithmetic_v<U>)
return accurate::equalsOp(l, r);
/// TODO This is wrong (does not respect scale).
if constexpr (isDecimalField<T>() && isDecimalField<U>())
return l == r;
if constexpr (isDecimalField<T>() && std::is_arithmetic_v<U>)
return l == DecimalField<Decimal128>(r, 0);
return l == DecimalField<Decimal256>(Decimal256(r), 0);
if constexpr (std::is_arithmetic_v<T> && isDecimalField<U>())
return DecimalField<Decimal128>(l, 0) == r;
return DecimalField<Decimal256>(Decimal256(l), 0) == r;
if constexpr (std::is_same_v<T, String>)
if constexpr (std::is_same_v<T, String> && std::is_arithmetic_v<U>)
{
if constexpr (std::is_same_v<U, UInt128>)
return stringToUUID(l) == r;
if constexpr (std::is_arithmetic_v<U>)
{
ReadBufferFromString in(l);
U parsed;
readText(parsed, in);
return operator()(parsed, r);
}
ReadBufferFromString in(l);
U parsed;
readText(parsed, in);
return operator()(parsed, r);
}
if constexpr (std::is_same_v<U, String>)
if constexpr (std::is_same_v<U, String> && std::is_arithmetic_v<T>)
{
if constexpr (std::is_same_v<T, UInt128>)
return l == stringToUUID(r);
if constexpr (std::is_arithmetic_v<T>)
{
ReadBufferFromString in(r);
T parsed;
readText(parsed, in);
return operator()(l, parsed);
}
ReadBufferFromString in(r);
T parsed;
readText(parsed, in);
return operator()(l, parsed);
}
}
@ -96,41 +85,30 @@ public:
if constexpr (std::is_arithmetic_v<T> && std::is_arithmetic_v<U>)
return accurate::lessOp(l, r);
/// TODO This is wrong (does not respect scale).
if constexpr (isDecimalField<T>() && isDecimalField<U>())
return l < r;
if constexpr (isDecimalField<T>() && std::is_arithmetic_v<U>)
return l < DecimalField<Decimal128>(r, 0);
return l < DecimalField<Decimal256>(Decimal256(r), 0);
if constexpr (std::is_arithmetic_v<T> && isDecimalField<U>())
return DecimalField<Decimal128>(l, 0) < r;
return DecimalField<Decimal256>(Decimal256(l), 0) < r;
if constexpr (std::is_same_v<T, String>)
if constexpr (std::is_same_v<T, String> && std::is_arithmetic_v<U>)
{
if constexpr (std::is_same_v<U, UInt128>)
return stringToUUID(l) < r;
if constexpr (std::is_arithmetic_v<U>)
{
ReadBufferFromString in(l);
U parsed;
readText(parsed, in);
return operator()(parsed, r);
}
ReadBufferFromString in(l);
U parsed;
readText(parsed, in);
return operator()(parsed, r);
}
if constexpr (std::is_same_v<U, String>)
if constexpr (std::is_same_v<U, String> && std::is_arithmetic_v<T>)
{
if constexpr (std::is_same_v<T, UInt128>)
return l < stringToUUID(r);
if constexpr (std::is_arithmetic_v<T>)
{
ReadBufferFromString in(r);
T parsed;
readText(parsed, in);
return operator()(l, parsed);
}
ReadBufferFromString in(r);
T parsed;
readText(parsed, in);
return operator()(l, parsed);
}
}

Some files were not shown because too many files have changed in this diff Show More