mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-22 07:31:57 +00:00
Merge remote-tracking branch 'origin/progress_bar_for_LocalServer' into progress_bar_for_LocalServer
This commit is contained in:
commit
d12243b76c
@ -169,8 +169,8 @@ endif ()
|
||||
option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON)
|
||||
option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF)
|
||||
|
||||
if (OS_LINUX AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0")
|
||||
# Only for Linux, x86_64.
|
||||
if (OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64) AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0")
|
||||
# Only for Linux, x86_64 or aarch64.
|
||||
option(GLIBC_COMPATIBILITY "Enable compatibility with older glibc libraries." ON)
|
||||
elseif(GLIBC_COMPATIBILITY)
|
||||
message (${RECONFIGURE_MESSAGE_LEVEL} "Glibc compatibility cannot be enabled in current configuration")
|
||||
|
216
base/common/DecomposedFloat.h
Normal file
216
base/common/DecomposedFloat.h
Normal file
@ -0,0 +1,216 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstddef>
|
||||
#include <cstring>
|
||||
#include <common/extended_types.h>
|
||||
|
||||
|
||||
/// Allows to check the internals of IEEE-754 floating point number.
|
||||
|
||||
template <typename T> struct FloatTraits;
|
||||
|
||||
template <>
|
||||
struct FloatTraits<float>
|
||||
{
|
||||
using UInt = uint32_t;
|
||||
static constexpr size_t bits = 32;
|
||||
static constexpr size_t exponent_bits = 8;
|
||||
static constexpr size_t mantissa_bits = bits - exponent_bits - 1;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct FloatTraits<double>
|
||||
{
|
||||
using UInt = uint64_t;
|
||||
static constexpr size_t bits = 64;
|
||||
static constexpr size_t exponent_bits = 11;
|
||||
static constexpr size_t mantissa_bits = bits - exponent_bits - 1;
|
||||
};
|
||||
|
||||
|
||||
/// x = sign * (2 ^ normalized_exponent) * (1 + mantissa * 2 ^ -mantissa_bits)
|
||||
/// x = sign * (2 ^ normalized_exponent + mantissa * 2 ^ (normalized_exponent - mantissa_bits))
|
||||
template <typename T>
|
||||
struct DecomposedFloat
|
||||
{
|
||||
using Traits = FloatTraits<T>;
|
||||
|
||||
DecomposedFloat(T x)
|
||||
{
|
||||
memcpy(&x_uint, &x, sizeof(x));
|
||||
}
|
||||
|
||||
typename Traits::UInt x_uint;
|
||||
|
||||
bool is_negative() const
|
||||
{
|
||||
return x_uint >> (Traits::bits - 1);
|
||||
}
|
||||
|
||||
/// Returns 0 for both +0. and -0.
|
||||
int sign() const
|
||||
{
|
||||
return (exponent() == 0 && mantissa() == 0)
|
||||
? 0
|
||||
: (is_negative()
|
||||
? -1
|
||||
: 1);
|
||||
}
|
||||
|
||||
uint16_t exponent() const
|
||||
{
|
||||
return (x_uint >> (Traits::mantissa_bits)) & (((1ull << (Traits::exponent_bits + 1)) - 1) >> 1);
|
||||
}
|
||||
|
||||
int16_t normalized_exponent() const
|
||||
{
|
||||
return int16_t(exponent()) - ((1ull << (Traits::exponent_bits - 1)) - 1);
|
||||
}
|
||||
|
||||
uint64_t mantissa() const
|
||||
{
|
||||
return x_uint & ((1ull << Traits::mantissa_bits) - 1);
|
||||
}
|
||||
|
||||
int64_t mantissa_with_sign() const
|
||||
{
|
||||
return is_negative() ? -mantissa() : mantissa();
|
||||
}
|
||||
|
||||
/// NOTE Probably floating point instructions can be better.
|
||||
bool is_integer_in_representable_range() const
|
||||
{
|
||||
return x_uint == 0
|
||||
|| (normalized_exponent() >= 0 /// The number is not less than one
|
||||
/// The number is inside the range where every integer has exact representation in float
|
||||
&& normalized_exponent() <= static_cast<int16_t>(Traits::mantissa_bits)
|
||||
/// After multiplying by 2^exp, the fractional part becomes zero, means the number is integer
|
||||
&& ((mantissa() & ((1ULL << (Traits::mantissa_bits - normalized_exponent())) - 1)) == 0));
|
||||
}
|
||||
|
||||
|
||||
/// Compare float with integer of arbitrary width (both signed and unsigned are supported). Assuming two's complement arithmetic.
|
||||
/// Infinities are compared correctly. NaNs are treat similarly to infinities, so they can be less than all numbers.
|
||||
/// (note that we need total order)
|
||||
template <typename Int>
|
||||
int compare(Int rhs)
|
||||
{
|
||||
if (rhs == 0)
|
||||
return sign();
|
||||
|
||||
/// Different signs
|
||||
if (is_negative() && rhs > 0)
|
||||
return -1;
|
||||
if (!is_negative() && rhs < 0)
|
||||
return 1;
|
||||
|
||||
/// Fractional number with magnitude less than one
|
||||
if (normalized_exponent() < 0)
|
||||
{
|
||||
if (!is_negative())
|
||||
return rhs > 0 ? -1 : 1;
|
||||
else
|
||||
return rhs >= 0 ? -1 : 1;
|
||||
}
|
||||
|
||||
/// The case of the most negative integer
|
||||
if constexpr (is_signed_v<Int>)
|
||||
{
|
||||
if (rhs == std::numeric_limits<Int>::lowest())
|
||||
{
|
||||
assert(is_negative());
|
||||
|
||||
if (normalized_exponent() < static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
|
||||
return 1;
|
||||
if (normalized_exponent() > static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
|
||||
return -1;
|
||||
|
||||
if (mantissa() == 0)
|
||||
return 0;
|
||||
else
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
/// Too large number: abs(float) > abs(rhs). Also the case with infinities and NaN.
|
||||
if (normalized_exponent() >= static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>))
|
||||
return is_negative() ? -1 : 1;
|
||||
|
||||
using UInt = make_unsigned_t<Int>;
|
||||
UInt uint_rhs = rhs < 0 ? -rhs : rhs;
|
||||
|
||||
/// Smaller octave: abs(rhs) < abs(float)
|
||||
if (uint_rhs < (static_cast<UInt>(1) << normalized_exponent()))
|
||||
return is_negative() ? -1 : 1;
|
||||
|
||||
/// Larger octave: abs(rhs) > abs(float)
|
||||
if (normalized_exponent() + 1 < static_cast<int16_t>(8 * sizeof(Int) - is_signed_v<Int>)
|
||||
&& uint_rhs >= (static_cast<UInt>(1) << (normalized_exponent() + 1)))
|
||||
return is_negative() ? 1 : -1;
|
||||
|
||||
/// The same octave
|
||||
/// uint_rhs == 2 ^ normalized_exponent + mantissa * 2 ^ (normalized_exponent - mantissa_bits)
|
||||
|
||||
bool large_and_always_integer = normalized_exponent() >= static_cast<int16_t>(Traits::mantissa_bits);
|
||||
|
||||
typename Traits::UInt a = large_and_always_integer
|
||||
? mantissa() << (normalized_exponent() - Traits::mantissa_bits)
|
||||
: mantissa() >> (Traits::mantissa_bits - normalized_exponent());
|
||||
|
||||
typename Traits::UInt b = uint_rhs - (static_cast<UInt>(1) << normalized_exponent());
|
||||
|
||||
if (a < b)
|
||||
return is_negative() ? 1 : -1;
|
||||
if (a > b)
|
||||
return is_negative() ? -1 : 1;
|
||||
|
||||
/// Float has no fractional part means that the numbers are equal.
|
||||
if (large_and_always_integer || (mantissa() & ((1ULL << (Traits::mantissa_bits - normalized_exponent())) - 1)) == 0)
|
||||
return 0;
|
||||
else
|
||||
/// Float has fractional part means its abs value is larger.
|
||||
return is_negative() ? -1 : 1;
|
||||
}
|
||||
|
||||
|
||||
template <typename Int>
|
||||
bool equals(Int rhs)
|
||||
{
|
||||
return compare(rhs) == 0;
|
||||
}
|
||||
|
||||
template <typename Int>
|
||||
bool notEquals(Int rhs)
|
||||
{
|
||||
return compare(rhs) != 0;
|
||||
}
|
||||
|
||||
template <typename Int>
|
||||
bool less(Int rhs)
|
||||
{
|
||||
return compare(rhs) < 0;
|
||||
}
|
||||
|
||||
template <typename Int>
|
||||
bool greater(Int rhs)
|
||||
{
|
||||
return compare(rhs) > 0;
|
||||
}
|
||||
|
||||
template <typename Int>
|
||||
bool lessOrEquals(Int rhs)
|
||||
{
|
||||
return compare(rhs) <= 0;
|
||||
}
|
||||
|
||||
template <typename Int>
|
||||
bool greaterOrEquals(Int rhs)
|
||||
{
|
||||
return compare(rhs) >= 0;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
using DecomposedFloat64 = DecomposedFloat<double>;
|
||||
using DecomposedFloat32 = DecomposedFloat<float>;
|
@ -56,27 +56,33 @@ namespace common
|
||||
}
|
||||
|
||||
template <>
|
||||
inline bool addOverflow(__int128 x, __int128 y, __int128 & res)
|
||||
inline bool addOverflow(Int128 x, Int128 y, Int128 & res)
|
||||
{
|
||||
static constexpr __int128 min_int128 = minInt128();
|
||||
static constexpr __int128 max_int128 = maxInt128();
|
||||
res = addIgnoreOverflow(x, y);
|
||||
return (y > 0 && x > max_int128 - y) || (y < 0 && x < min_int128 - y);
|
||||
return (y > 0 && x > std::numeric_limits<Int128>::max() - y) ||
|
||||
(y < 0 && x < std::numeric_limits<Int128>::min() - y);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline bool addOverflow(wInt256 x, wInt256 y, wInt256 & res)
|
||||
inline bool addOverflow(UInt128 x, UInt128 y, UInt128 & res)
|
||||
{
|
||||
res = addIgnoreOverflow(x, y);
|
||||
return (y > 0 && x > std::numeric_limits<wInt256>::max() - y) ||
|
||||
(y < 0 && x < std::numeric_limits<wInt256>::min() - y);
|
||||
return x > std::numeric_limits<UInt128>::max() - y;
|
||||
}
|
||||
|
||||
template <>
|
||||
inline bool addOverflow(wUInt256 x, wUInt256 y, wUInt256 & res)
|
||||
inline bool addOverflow(Int256 x, Int256 y, Int256 & res)
|
||||
{
|
||||
res = addIgnoreOverflow(x, y);
|
||||
return x > std::numeric_limits<wUInt256>::max() - y;
|
||||
return (y > 0 && x > std::numeric_limits<Int256>::max() - y) ||
|
||||
(y < 0 && x < std::numeric_limits<Int256>::min() - y);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline bool addOverflow(UInt256 x, UInt256 y, UInt256 & res)
|
||||
{
|
||||
res = addIgnoreOverflow(x, y);
|
||||
return x > std::numeric_limits<UInt256>::max() - y;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
@ -104,24 +110,30 @@ namespace common
|
||||
}
|
||||
|
||||
template <>
|
||||
inline bool subOverflow(__int128 x, __int128 y, __int128 & res)
|
||||
inline bool subOverflow(Int128 x, Int128 y, Int128 & res)
|
||||
{
|
||||
static constexpr __int128 min_int128 = minInt128();
|
||||
static constexpr __int128 max_int128 = maxInt128();
|
||||
res = subIgnoreOverflow(x, y);
|
||||
return (y < 0 && x > max_int128 + y) || (y > 0 && x < min_int128 + y);
|
||||
return (y < 0 && x > std::numeric_limits<Int128>::max() + y) ||
|
||||
(y > 0 && x < std::numeric_limits<Int128>::min() + y);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline bool subOverflow(wInt256 x, wInt256 y, wInt256 & res)
|
||||
inline bool subOverflow(UInt128 x, UInt128 y, UInt128 & res)
|
||||
{
|
||||
res = subIgnoreOverflow(x, y);
|
||||
return (y < 0 && x > std::numeric_limits<wInt256>::max() + y) ||
|
||||
(y > 0 && x < std::numeric_limits<wInt256>::min() + y);
|
||||
return x < y;
|
||||
}
|
||||
|
||||
template <>
|
||||
inline bool subOverflow(wUInt256 x, wUInt256 y, wUInt256 & res)
|
||||
inline bool subOverflow(Int256 x, Int256 y, Int256 & res)
|
||||
{
|
||||
res = subIgnoreOverflow(x, y);
|
||||
return (y < 0 && x > std::numeric_limits<Int256>::max() + y) ||
|
||||
(y > 0 && x < std::numeric_limits<Int256>::min() + y);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline bool subOverflow(UInt256 x, UInt256 y, UInt256 & res)
|
||||
{
|
||||
res = subIgnoreOverflow(x, y);
|
||||
return x < y;
|
||||
@ -151,36 +163,33 @@ namespace common
|
||||
return __builtin_smulll_overflow(x, y, &res);
|
||||
}
|
||||
|
||||
/// Overflow check is not implemented for big integers.
|
||||
|
||||
template <>
|
||||
inline bool mulOverflow(__int128 x, __int128 y, __int128 & res)
|
||||
inline bool mulOverflow(Int128 x, Int128 y, Int128 & res)
|
||||
{
|
||||
res = mulIgnoreOverflow(x, y);
|
||||
if (!x || !y)
|
||||
return false;
|
||||
|
||||
unsigned __int128 a = (x > 0) ? x : -x;
|
||||
unsigned __int128 b = (y > 0) ? y : -y;
|
||||
return mulIgnoreOverflow(a, b) / b != a;
|
||||
return false;
|
||||
}
|
||||
|
||||
template <>
|
||||
inline bool mulOverflow(wInt256 x, wInt256 y, wInt256 & res)
|
||||
inline bool mulOverflow(Int256 x, Int256 y, Int256 & res)
|
||||
{
|
||||
res = mulIgnoreOverflow(x, y);
|
||||
if (!x || !y)
|
||||
return false;
|
||||
|
||||
wInt256 a = (x > 0) ? x : -x;
|
||||
wInt256 b = (y > 0) ? y : -y;
|
||||
return mulIgnoreOverflow(a, b) / b != a;
|
||||
return false;
|
||||
}
|
||||
|
||||
template <>
|
||||
inline bool mulOverflow(wUInt256 x, wUInt256 y, wUInt256 & res)
|
||||
inline bool mulOverflow(UInt128 x, UInt128 y, UInt128 & res)
|
||||
{
|
||||
res = mulIgnoreOverflow(x, y);
|
||||
if (!x || !y)
|
||||
return false;
|
||||
return res / y != x;
|
||||
return false;
|
||||
}
|
||||
|
||||
template <>
|
||||
inline bool mulOverflow(UInt256 x, UInt256 y, UInt256 & res)
|
||||
{
|
||||
res = mulIgnoreOverflow(x, y);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
@ -5,16 +5,14 @@
|
||||
#include <common/types.h>
|
||||
#include <common/wide_integer.h>
|
||||
|
||||
using Int128 = __int128;
|
||||
|
||||
using wInt256 = wide::integer<256, signed>;
|
||||
using wUInt256 = wide::integer<256, unsigned>;
|
||||
using Int128 = wide::integer<128, signed>;
|
||||
using UInt128 = wide::integer<128, unsigned>;
|
||||
using Int256 = wide::integer<256, signed>;
|
||||
using UInt256 = wide::integer<256, unsigned>;
|
||||
|
||||
static_assert(sizeof(wInt256) == 32);
|
||||
static_assert(sizeof(wUInt256) == 32);
|
||||
|
||||
static constexpr __int128 minInt128() { return static_cast<unsigned __int128>(1) << 127; }
|
||||
static constexpr __int128 maxInt128() { return (static_cast<unsigned __int128>(1) << 127) - 1; }
|
||||
static_assert(sizeof(Int256) == 32);
|
||||
static_assert(sizeof(UInt256) == 32);
|
||||
|
||||
/// The standard library type traits, such as std::is_arithmetic, with one exception
|
||||
/// (std::common_type), are "set in stone". Attempting to specialize them causes undefined behavior.
|
||||
@ -26,7 +24,7 @@ struct is_signed
|
||||
};
|
||||
|
||||
template <> struct is_signed<Int128> { static constexpr bool value = true; };
|
||||
template <> struct is_signed<wInt256> { static constexpr bool value = true; };
|
||||
template <> struct is_signed<Int256> { static constexpr bool value = true; };
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_signed_v = is_signed<T>::value;
|
||||
@ -37,7 +35,8 @@ struct is_unsigned
|
||||
static constexpr bool value = std::is_unsigned_v<T>;
|
||||
};
|
||||
|
||||
template <> struct is_unsigned<wUInt256> { static constexpr bool value = true; };
|
||||
template <> struct is_unsigned<UInt128> { static constexpr bool value = true; };
|
||||
template <> struct is_unsigned<UInt256> { static constexpr bool value = true; };
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_unsigned_v = is_unsigned<T>::value;
|
||||
@ -51,8 +50,9 @@ struct is_integer
|
||||
};
|
||||
|
||||
template <> struct is_integer<Int128> { static constexpr bool value = true; };
|
||||
template <> struct is_integer<wInt256> { static constexpr bool value = true; };
|
||||
template <> struct is_integer<wUInt256> { static constexpr bool value = true; };
|
||||
template <> struct is_integer<UInt128> { static constexpr bool value = true; };
|
||||
template <> struct is_integer<Int256> { static constexpr bool value = true; };
|
||||
template <> struct is_integer<UInt256> { static constexpr bool value = true; };
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_integer_v = is_integer<T>::value;
|
||||
@ -64,7 +64,11 @@ struct is_arithmetic
|
||||
static constexpr bool value = std::is_arithmetic_v<T>;
|
||||
};
|
||||
|
||||
template <> struct is_arithmetic<__int128> { static constexpr bool value = true; };
|
||||
template <> struct is_arithmetic<Int128> { static constexpr bool value = true; };
|
||||
template <> struct is_arithmetic<UInt128> { static constexpr bool value = true; };
|
||||
template <> struct is_arithmetic<Int256> { static constexpr bool value = true; };
|
||||
template <> struct is_arithmetic<UInt256> { static constexpr bool value = true; };
|
||||
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_arithmetic_v = is_arithmetic<T>::value;
|
||||
@ -75,9 +79,10 @@ struct make_unsigned
|
||||
typedef std::make_unsigned_t<T> type;
|
||||
};
|
||||
|
||||
template <> struct make_unsigned<Int128> { using type = unsigned __int128; };
|
||||
template <> struct make_unsigned<wInt256> { using type = wUInt256; };
|
||||
template <> struct make_unsigned<wUInt256> { using type = wUInt256; };
|
||||
template <> struct make_unsigned<Int128> { using type = UInt128; };
|
||||
template <> struct make_unsigned<UInt128> { using type = UInt128; };
|
||||
template <> struct make_unsigned<Int256> { using type = UInt256; };
|
||||
template <> struct make_unsigned<UInt256> { using type = UInt256; };
|
||||
|
||||
template <typename T> using make_unsigned_t = typename make_unsigned<T>::type;
|
||||
|
||||
@ -87,8 +92,10 @@ struct make_signed
|
||||
typedef std::make_signed_t<T> type;
|
||||
};
|
||||
|
||||
template <> struct make_signed<wInt256> { using type = wInt256; };
|
||||
template <> struct make_signed<wUInt256> { using type = wInt256; };
|
||||
template <> struct make_signed<Int128> { using type = Int128; };
|
||||
template <> struct make_signed<UInt128> { using type = Int128; };
|
||||
template <> struct make_signed<Int256> { using type = Int256; };
|
||||
template <> struct make_signed<UInt256> { using type = Int256; };
|
||||
|
||||
template <typename T> using make_signed_t = typename make_signed<T>::type;
|
||||
|
||||
@ -98,8 +105,10 @@ struct is_big_int
|
||||
static constexpr bool value = false;
|
||||
};
|
||||
|
||||
template <> struct is_big_int<wInt256> { static constexpr bool value = true; };
|
||||
template <> struct is_big_int<wUInt256> { static constexpr bool value = true; };
|
||||
template <> struct is_big_int<Int128> { static constexpr bool value = true; };
|
||||
template <> struct is_big_int<UInt128> { static constexpr bool value = true; };
|
||||
template <> struct is_big_int<Int256> { static constexpr bool value = true; };
|
||||
template <> struct is_big_int<UInt256> { static constexpr bool value = true; };
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_big_int_v = is_big_int<T>::value;
|
||||
|
@ -30,9 +30,8 @@
|
||||
#include <cstddef>
|
||||
#include <cstring>
|
||||
#include <type_traits>
|
||||
#include <common/extended_types.h>
|
||||
|
||||
using int128_t = __int128;
|
||||
using uint128_t = unsigned __int128;
|
||||
|
||||
namespace impl
|
||||
{
|
||||
@ -106,7 +105,7 @@ using UnsignedOfSize = typename SelectType
|
||||
uint16_t,
|
||||
uint32_t,
|
||||
uint64_t,
|
||||
uint128_t
|
||||
__uint128_t
|
||||
>::Result;
|
||||
|
||||
/// Holds the result of dividing an unsigned N-byte variable by 10^N resulting in
|
||||
@ -313,7 +312,8 @@ namespace convert
|
||||
}
|
||||
}
|
||||
|
||||
static inline int digits10(uint128_t x)
|
||||
template <typename T>
|
||||
static inline int digits10(T x)
|
||||
{
|
||||
if (x < 10ULL)
|
||||
return 1;
|
||||
@ -346,8 +346,11 @@ static inline int digits10(uint128_t x)
|
||||
return 12 + digits10(x / 1000000000000ULL);
|
||||
}
|
||||
|
||||
static inline char * writeUIntText(uint128_t x, char * p)
|
||||
template <typename T>
|
||||
static inline char * writeUIntText(T x, char * p)
|
||||
{
|
||||
static_assert(is_unsigned_v<T>);
|
||||
|
||||
int len = digits10(x);
|
||||
auto pp = p + len;
|
||||
while (x >= 100)
|
||||
@ -370,14 +373,28 @@ static inline char * writeLeadingMinus(char * pos)
|
||||
return pos + 1;
|
||||
}
|
||||
|
||||
static inline char * writeSIntText(int128_t x, char * pos)
|
||||
template <typename T>
|
||||
static inline char * writeSIntText(T x, char * pos)
|
||||
{
|
||||
static constexpr int128_t min_int128 = uint128_t(1) << 127;
|
||||
static_assert(std::is_same_v<T, Int128> || std::is_same_v<T, Int256>);
|
||||
|
||||
if (unlikely(x == min_int128))
|
||||
using UnsignedT = make_unsigned_t<T>;
|
||||
static constexpr T min_int = UnsignedT(1) << (sizeof(T) * 8 - 1);
|
||||
|
||||
if (unlikely(x == min_int))
|
||||
{
|
||||
memcpy(pos, "-170141183460469231731687303715884105728", 40);
|
||||
return pos + 40;
|
||||
if constexpr (std::is_same_v<T, Int128>)
|
||||
{
|
||||
const char * res = "-170141183460469231731687303715884105728";
|
||||
memcpy(pos, res, strlen(res));
|
||||
return pos + strlen(res);
|
||||
}
|
||||
else if constexpr (std::is_same_v<T, Int256>)
|
||||
{
|
||||
const char * res = "-57896044618658097711785492504343953926634992332820282019728792003956564819968";
|
||||
memcpy(pos, res, strlen(res));
|
||||
return pos + strlen(res);
|
||||
}
|
||||
}
|
||||
|
||||
if (x < 0)
|
||||
@ -385,7 +402,7 @@ static inline char * writeSIntText(int128_t x, char * pos)
|
||||
x = -x;
|
||||
pos = writeLeadingMinus(pos);
|
||||
}
|
||||
return writeUIntText(static_cast<uint128_t>(x), pos);
|
||||
return writeUIntText(UnsignedT(x), pos);
|
||||
}
|
||||
|
||||
}
|
||||
@ -403,13 +420,25 @@ inline char * itoa(char8_t i, char * p)
|
||||
}
|
||||
|
||||
template <>
|
||||
inline char * itoa<uint128_t>(uint128_t i, char * p)
|
||||
inline char * itoa(UInt128 i, char * p)
|
||||
{
|
||||
return impl::writeUIntText(i, p);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline char * itoa<int128_t>(int128_t i, char * p)
|
||||
inline char * itoa(Int128 i, char * p)
|
||||
{
|
||||
return impl::writeSIntText(i, p);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline char * itoa(UInt256 i, char * p)
|
||||
{
|
||||
return impl::writeUIntText(i, p);
|
||||
}
|
||||
|
||||
template <>
|
||||
inline char * itoa(Int256 i, char * p)
|
||||
{
|
||||
return impl::writeSIntText(i, p);
|
||||
}
|
||||
|
@ -4,7 +4,8 @@
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
|
||||
template <class T, class Tag>
|
||||
|
||||
template <typename T, typename Tag>
|
||||
struct StrongTypedef
|
||||
{
|
||||
private:
|
||||
@ -38,14 +39,16 @@ public:
|
||||
|
||||
bool operator==(const Self & rhs) const { return t == rhs.t; }
|
||||
bool operator<(const Self & rhs) const { return t < rhs.t; }
|
||||
bool operator>(const Self & rhs) const { return t > rhs.t; }
|
||||
|
||||
T & toUnderType() { return t; }
|
||||
const T & toUnderType() const { return t; }
|
||||
};
|
||||
|
||||
|
||||
namespace std
|
||||
{
|
||||
template <class T, class Tag>
|
||||
template <typename T, typename Tag>
|
||||
struct hash<StrongTypedef<T, Tag>>
|
||||
{
|
||||
size_t operator()(const StrongTypedef<T, Tag> & x) const
|
||||
|
@ -1,13 +1,15 @@
|
||||
#pragma once
|
||||
|
||||
#include <stdexcept>
|
||||
|
||||
|
||||
/// Throw DB::Exception-like exception before its definition.
|
||||
/// DB::Exception derived from Poco::Exception derived from std::exception.
|
||||
/// DB::Exception generally cought as Poco::Exception. std::exception generally has other catch blocks and could lead to other outcomes.
|
||||
/// DB::Exception generally caught as Poco::Exception. std::exception generally has other catch blocks and could lead to other outcomes.
|
||||
/// DB::Exception is not defined yet. It'd better to throw Poco::Exception but we do not want to include any big header here, even <string>.
|
||||
/// So we throw some std::exception instead in the hope its catch block is the same as DB::Exception one.
|
||||
template <typename T>
|
||||
inline void throwError(const T & err)
|
||||
[[noreturn]] inline void throwError(const T & err)
|
||||
{
|
||||
throw std::runtime_error(err);
|
||||
}
|
||||
|
@ -58,9 +58,11 @@ public:
|
||||
using signed_base_type = int64_t;
|
||||
|
||||
// ctors
|
||||
constexpr integer() noexcept;
|
||||
constexpr integer() noexcept = default;
|
||||
|
||||
template <typename T>
|
||||
constexpr integer(T rhs) noexcept;
|
||||
|
||||
template <typename T>
|
||||
constexpr integer(std::initializer_list<T> il) noexcept;
|
||||
|
||||
@ -108,9 +110,9 @@ public:
|
||||
constexpr explicit operator bool() const noexcept;
|
||||
|
||||
template <class T>
|
||||
using __integral_not_wide_integer_class = typename std::enable_if<std::is_arithmetic<T>::value, T>::type;
|
||||
using _integral_not_wide_integer_class = typename std::enable_if<std::is_arithmetic<T>::value, T>::type;
|
||||
|
||||
template <class T, class = __integral_not_wide_integer_class<T>>
|
||||
template <class T, class = _integral_not_wide_integer_class<T>>
|
||||
constexpr operator T() const noexcept;
|
||||
|
||||
constexpr operator long double() const noexcept;
|
||||
@ -119,25 +121,27 @@ public:
|
||||
|
||||
struct _impl;
|
||||
|
||||
base_type items[_impl::item_count];
|
||||
|
||||
private:
|
||||
template <size_t Bits2, typename Signed2>
|
||||
friend class integer;
|
||||
|
||||
friend class std::numeric_limits<integer<Bits, signed>>;
|
||||
friend class std::numeric_limits<integer<Bits, unsigned>>;
|
||||
|
||||
base_type items[_impl::item_count];
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
static constexpr bool ArithmeticConcept() noexcept;
|
||||
|
||||
template <class T1, class T2>
|
||||
using __only_arithmetic = typename std::enable_if<ArithmeticConcept<T1>() && ArithmeticConcept<T2>()>::type;
|
||||
using _only_arithmetic = typename std::enable_if<ArithmeticConcept<T1>() && ArithmeticConcept<T2>()>::type;
|
||||
|
||||
template <typename T>
|
||||
static constexpr bool IntegralConcept() noexcept;
|
||||
|
||||
template <class T, class T2>
|
||||
using __only_integer = typename std::enable_if<IntegralConcept<T>() && IntegralConcept<T2>()>::type;
|
||||
using _only_integer = typename std::enable_if<IntegralConcept<T>() && IntegralConcept<T2>()>::type;
|
||||
|
||||
// Unary operators
|
||||
template <size_t Bits, typename Signed>
|
||||
@ -153,54 +157,55 @@ constexpr integer<Bits, Signed> operator+(const integer<Bits, Signed> & lhs) noe
|
||||
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
|
||||
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
|
||||
operator*(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
|
||||
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
std::common_type_t<Arithmetic, Arithmetic2> constexpr operator*(const Arithmetic & rhs, const Arithmetic2 & lhs);
|
||||
|
||||
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
|
||||
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
|
||||
operator/(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
|
||||
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
std::common_type_t<Arithmetic, Arithmetic2> constexpr operator/(const Arithmetic & rhs, const Arithmetic2 & lhs);
|
||||
|
||||
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
|
||||
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
|
||||
operator+(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
|
||||
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
std::common_type_t<Arithmetic, Arithmetic2> constexpr operator+(const Arithmetic & rhs, const Arithmetic2 & lhs);
|
||||
|
||||
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
|
||||
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
|
||||
operator-(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
|
||||
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
std::common_type_t<Arithmetic, Arithmetic2> constexpr operator-(const Arithmetic & rhs, const Arithmetic2 & lhs);
|
||||
|
||||
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
|
||||
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
|
||||
operator%(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
|
||||
template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
|
||||
template <typename Integral, typename Integral2, class = _only_integer<Integral, Integral2>>
|
||||
std::common_type_t<Integral, Integral2> constexpr operator%(const Integral & rhs, const Integral2 & lhs);
|
||||
|
||||
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
|
||||
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
|
||||
operator&(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
|
||||
template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
|
||||
template <typename Integral, typename Integral2, class = _only_integer<Integral, Integral2>>
|
||||
std::common_type_t<Integral, Integral2> constexpr operator&(const Integral & rhs, const Integral2 & lhs);
|
||||
|
||||
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
|
||||
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
|
||||
operator|(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
|
||||
template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
|
||||
template <typename Integral, typename Integral2, class = _only_integer<Integral, Integral2>>
|
||||
std::common_type_t<Integral, Integral2> constexpr operator|(const Integral & rhs, const Integral2 & lhs);
|
||||
|
||||
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
|
||||
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
|
||||
operator^(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
|
||||
template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
|
||||
template <typename Integral, typename Integral2, class = _only_integer<Integral, Integral2>>
|
||||
std::common_type_t<Integral, Integral2> constexpr operator^(const Integral & rhs, const Integral2 & lhs);
|
||||
|
||||
// TODO: Integral
|
||||
template <size_t Bits, typename Signed>
|
||||
constexpr integer<Bits, Signed> operator<<(const integer<Bits, Signed> & lhs, int n) noexcept;
|
||||
|
||||
template <size_t Bits, typename Signed>
|
||||
constexpr integer<Bits, Signed> operator>>(const integer<Bits, Signed> & lhs, int n) noexcept;
|
||||
|
||||
@ -217,32 +222,32 @@ constexpr integer<Bits, Signed> operator>>(const integer<Bits, Signed> & lhs, In
|
||||
|
||||
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
|
||||
constexpr bool operator<(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
|
||||
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
constexpr bool operator<(const Arithmetic & rhs, const Arithmetic2 & lhs);
|
||||
|
||||
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
|
||||
constexpr bool operator>(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
|
||||
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
constexpr bool operator>(const Arithmetic & rhs, const Arithmetic2 & lhs);
|
||||
|
||||
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
|
||||
constexpr bool operator<=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
|
||||
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
constexpr bool operator<=(const Arithmetic & rhs, const Arithmetic2 & lhs);
|
||||
|
||||
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
|
||||
constexpr bool operator>=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
|
||||
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
constexpr bool operator>=(const Arithmetic & rhs, const Arithmetic2 & lhs);
|
||||
|
||||
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
|
||||
constexpr bool operator==(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
|
||||
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
constexpr bool operator==(const Arithmetic & rhs, const Arithmetic2 & lhs);
|
||||
|
||||
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
|
||||
constexpr bool operator!=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
|
||||
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
template <typename Arithmetic, typename Arithmetic2, class = _only_arithmetic<Arithmetic, Arithmetic2>>
|
||||
constexpr bool operator!=(const Arithmetic & rhs, const Arithmetic2 & lhs);
|
||||
|
||||
}
|
||||
|
@ -5,6 +5,7 @@
|
||||
/// (See at http://www.boost.org/LICENSE_1_0.txt)
|
||||
|
||||
#include "throwError.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cfloat>
|
||||
#include <cassert>
|
||||
@ -81,7 +82,7 @@ public:
|
||||
res.items[T::_impl::big(0)] = std::numeric_limits<typename wide::integer<Bits, Signed>::signed_base_type>::min();
|
||||
return res;
|
||||
}
|
||||
return 0;
|
||||
return wide::integer<Bits, Signed>(0);
|
||||
}
|
||||
|
||||
static constexpr wide::integer<Bits, Signed> max() noexcept
|
||||
@ -176,7 +177,7 @@ struct integer<Bits, Signed>::_impl
|
||||
constexpr static bool is_negative(const integer<B, T> & n) noexcept
|
||||
{
|
||||
if constexpr (std::is_same_v<T, signed>)
|
||||
return static_cast<signed_base_type>(n.items[big(0)]) < 0;
|
||||
return static_cast<signed_base_type>(n.items[integer<B, T>::_impl::big(0)]) < 0;
|
||||
else
|
||||
return false;
|
||||
}
|
||||
@ -193,40 +194,36 @@ struct integer<Bits, Signed>::_impl
|
||||
template <size_t B, class S>
|
||||
constexpr static integer<B, S> make_positive(const integer<B, S> & n) noexcept
|
||||
{
|
||||
return is_negative(n) ? operator_unary_minus(n) : n;
|
||||
return is_negative(n) ? integer<B, S>(operator_unary_minus(n)) : n;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
__attribute__((no_sanitize("undefined"))) constexpr static auto to_Integral(T f) noexcept
|
||||
{
|
||||
if constexpr (std::is_same_v<T, __int128>)
|
||||
return f;
|
||||
else if constexpr (std::is_signed_v<T>)
|
||||
if constexpr (std::is_signed_v<T>)
|
||||
return static_cast<int64_t>(f);
|
||||
else
|
||||
return static_cast<uint64_t>(f);
|
||||
}
|
||||
|
||||
template <typename Integral>
|
||||
constexpr static void wide_integer_from_bultin(integer<Bits, Signed> & self, Integral rhs) noexcept
|
||||
constexpr static void wide_integer_from_builtin(integer<Bits, Signed> & self, Integral rhs) noexcept
|
||||
{
|
||||
self.items[0] = _impl::to_Integral(rhs);
|
||||
if constexpr (std::is_same_v<Integral, __int128>)
|
||||
self.items[1] = rhs >> base_bits;
|
||||
static_assert(sizeof(Integral) <= sizeof(base_type));
|
||||
|
||||
constexpr const unsigned start = (sizeof(Integral) == 16) ? 2 : 1;
|
||||
self.items[0] = _impl::to_Integral(rhs);
|
||||
|
||||
if constexpr (std::is_signed_v<Integral>)
|
||||
{
|
||||
if (rhs < 0)
|
||||
{
|
||||
for (unsigned i = start; i < item_count; ++i)
|
||||
for (size_t i = 1; i < item_count; ++i)
|
||||
self.items[i] = -1;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
for (unsigned i = start; i < item_count; ++i)
|
||||
for (size_t i = 1; i < item_count; ++i)
|
||||
self.items[i] = 0;
|
||||
}
|
||||
|
||||
@ -239,7 +236,8 @@ struct integer<Bits, Signed>::_impl
|
||||
* a_(n - 1) = a_n * max_int + b2, a_n <= max_int <- base case.
|
||||
*/
|
||||
template <class T>
|
||||
constexpr static void set_multiplier(integer<Bits, Signed> & self, T t) noexcept {
|
||||
constexpr static void set_multiplier(integer<Bits, Signed> & self, T t) noexcept
|
||||
{
|
||||
constexpr uint64_t max_int = std::numeric_limits<uint64_t>::max();
|
||||
|
||||
/// Implementation specific behaviour on overflow (if we don't check here, stack overflow will triggered in bigint_cast).
|
||||
@ -260,7 +258,8 @@ struct integer<Bits, Signed>::_impl
|
||||
self += static_cast<uint64_t>(t - alpha * static_cast<T>(max_int)); // += b_i
|
||||
}
|
||||
|
||||
constexpr static void wide_integer_from_bultin(integer<Bits, Signed>& self, double rhs) noexcept {
|
||||
constexpr static void wide_integer_from_builtin(integer<Bits, Signed>& self, double rhs) noexcept
|
||||
{
|
||||
constexpr int64_t max_int = std::numeric_limits<int64_t>::max();
|
||||
constexpr int64_t min_int = std::numeric_limits<int64_t>::min();
|
||||
|
||||
@ -383,13 +382,13 @@ struct integer<Bits, Signed>::_impl
|
||||
if (bit_shift)
|
||||
lhs.items[big(items_shift)] |= std::numeric_limits<base_type>::max() << (base_bits - bit_shift);
|
||||
|
||||
for (unsigned i = item_count - items_shift; i < items_shift; ++i)
|
||||
lhs.items[little(i)] = std::numeric_limits<base_type>::max();
|
||||
for (unsigned i = 0; i < items_shift; ++i)
|
||||
lhs.items[big(i)] = std::numeric_limits<base_type>::max();
|
||||
}
|
||||
else
|
||||
{
|
||||
for (unsigned i = item_count - items_shift; i < items_shift; ++i)
|
||||
lhs.items[little(i)] = 0;
|
||||
for (unsigned i = 0; i < items_shift; ++i)
|
||||
lhs.items[big(i)] = 0;
|
||||
}
|
||||
|
||||
return lhs;
|
||||
@ -397,23 +396,23 @@ struct integer<Bits, Signed>::_impl
|
||||
|
||||
private:
|
||||
template <typename T>
|
||||
constexpr static base_type get_item(const T & x, unsigned number)
|
||||
constexpr static base_type get_item(const T & x, unsigned idx)
|
||||
{
|
||||
if constexpr (IsWideInteger<T>::value)
|
||||
{
|
||||
if (number < T::_impl::item_count)
|
||||
return x.items[number];
|
||||
if (idx < T::_impl::item_count)
|
||||
return x.items[idx];
|
||||
return 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
if constexpr (sizeof(T) <= sizeof(base_type))
|
||||
{
|
||||
if (!number)
|
||||
if (0 == idx)
|
||||
return x;
|
||||
}
|
||||
else if (number * sizeof(base_type) < sizeof(T))
|
||||
return x >> (number * base_bits); // & std::numeric_limits<base_type>::max()
|
||||
else if (idx * sizeof(base_type) < sizeof(T))
|
||||
return x >> (idx * base_bits); // & std::numeric_limits<base_type>::max()
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
@ -439,7 +438,7 @@ private:
|
||||
|
||||
for (unsigned i = 1; i < item_count; ++i)
|
||||
{
|
||||
if (underflows[i-1])
|
||||
if (underflows[i - 1])
|
||||
{
|
||||
base_type & res_item = res.items[little(i)];
|
||||
if (res_item == 0)
|
||||
@ -472,7 +471,7 @@ private:
|
||||
|
||||
for (unsigned i = 1; i < item_count; ++i)
|
||||
{
|
||||
if (overflows[i-1])
|
||||
if (overflows[i - 1])
|
||||
{
|
||||
base_type & res_item = res.items[little(i)];
|
||||
++res_item;
|
||||
@ -532,6 +531,17 @@ private:
|
||||
res.items[little(2)] = r12 >> 64;
|
||||
return res;
|
||||
}
|
||||
else if constexpr (Bits == 128 && sizeof(base_type) == 8)
|
||||
{
|
||||
using CompilerUInt128 = unsigned __int128;
|
||||
CompilerUInt128 a = (CompilerUInt128(lhs.items[1]) << 64) + lhs.items[0];
|
||||
CompilerUInt128 b = (CompilerUInt128(rhs.items[1]) << 64) + rhs.items[0];
|
||||
CompilerUInt128 c = a * b;
|
||||
integer<Bits, Signed> res;
|
||||
res.items[0] = c;
|
||||
res.items[1] = c >> 64;
|
||||
return res;
|
||||
}
|
||||
else
|
||||
{
|
||||
integer<Bits, Signed> res{};
|
||||
@ -657,7 +667,7 @@ public:
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
constexpr static bool operator_more(const integer<Bits, Signed> & lhs, const T & rhs) noexcept
|
||||
constexpr static bool operator_greater(const integer<Bits, Signed> & lhs, const T & rhs) noexcept
|
||||
{
|
||||
if constexpr (should_keep_size<T>())
|
||||
{
|
||||
@ -677,7 +687,7 @@ public:
|
||||
else
|
||||
{
|
||||
static_assert(IsWideInteger<T>::value);
|
||||
return std::common_type_t<integer<Bits, Signed>, T>::_impl::operator_more(T(lhs), rhs);
|
||||
return std::common_type_t<integer<Bits, Signed>, T>::_impl::operator_greater(T(lhs), rhs);
|
||||
}
|
||||
}
|
||||
|
||||
@ -764,7 +774,6 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
template <typename T>
|
||||
constexpr static bool is_zero(const T & x)
|
||||
{
|
||||
@ -781,46 +790,65 @@ private:
|
||||
}
|
||||
|
||||
/// returns quotient as result and remainder in numerator.
|
||||
template <typename T>
|
||||
constexpr static T divide(T & numerator, T && denominator)
|
||||
template <size_t Bits2>
|
||||
constexpr static integer<Bits2, unsigned> divide(integer<Bits2, unsigned> & numerator, integer<Bits2, unsigned> denominator)
|
||||
{
|
||||
if (is_zero(denominator))
|
||||
throwError("divide by zero");
|
||||
static_assert(std::is_unsigned_v<Signed>);
|
||||
|
||||
T & n = numerator;
|
||||
T & d = denominator;
|
||||
T x = 1;
|
||||
T quotient = 0;
|
||||
|
||||
while (!operator_more(d, n) && operator_eq(operator_amp(shift_right(d, base_bits * item_count - 1), 1), 0))
|
||||
if constexpr (Bits == 128 && sizeof(base_type) == 8)
|
||||
{
|
||||
x = shift_left(x, 1);
|
||||
d = shift_left(d, 1);
|
||||
using CompilerUInt128 = unsigned __int128;
|
||||
|
||||
CompilerUInt128 a = (CompilerUInt128(numerator.items[1]) << 64) + numerator.items[0];
|
||||
CompilerUInt128 b = (CompilerUInt128(denominator.items[1]) << 64) + denominator.items[0];
|
||||
CompilerUInt128 c = a / b;
|
||||
|
||||
integer<Bits, Signed> res;
|
||||
res.items[0] = c;
|
||||
res.items[1] = c >> 64;
|
||||
|
||||
CompilerUInt128 remainder = a - b * c;
|
||||
numerator.items[0] = remainder;
|
||||
numerator.items[1] = remainder >> 64;
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
while (!operator_eq(x, 0))
|
||||
if (is_zero(denominator))
|
||||
throwError("Division by zero");
|
||||
|
||||
integer<Bits2, unsigned> x = 1;
|
||||
integer<Bits2, unsigned> quotient = 0;
|
||||
|
||||
while (!operator_greater(denominator, numerator) && is_zero(operator_amp(shift_right(denominator, Bits2 - 1), 1)))
|
||||
{
|
||||
if (!operator_more(d, n))
|
||||
x = shift_left(x, 1);
|
||||
denominator = shift_left(denominator, 1);
|
||||
}
|
||||
|
||||
while (!is_zero(x))
|
||||
{
|
||||
if (!operator_greater(denominator, numerator))
|
||||
{
|
||||
n = operator_minus(n, d);
|
||||
numerator = operator_minus(numerator, denominator);
|
||||
quotient = operator_pipe(quotient, x);
|
||||
}
|
||||
|
||||
x = shift_right(x, 1);
|
||||
d = shift_right(d, 1);
|
||||
denominator = shift_right(denominator, 1);
|
||||
}
|
||||
|
||||
return quotient;
|
||||
}
|
||||
|
||||
public:
|
||||
template <typename T>
|
||||
constexpr static auto operator_slash(const integer<Bits, Signed> & lhs, const T & rhs)
|
||||
{
|
||||
if constexpr (should_keep_size<T>())
|
||||
{
|
||||
integer<Bits, Signed> numerator = make_positive(lhs);
|
||||
integer<Bits, Signed> quotient = divide(numerator, make_positive(integer<Bits, Signed>(rhs)));
|
||||
integer<Bits, unsigned> numerator = make_positive(lhs);
|
||||
integer<Bits, unsigned> denominator = make_positive(integer<Bits, Signed>(rhs));
|
||||
integer<Bits, unsigned> quotient = integer<Bits, unsigned>::_impl::divide(numerator, std::move(denominator));
|
||||
|
||||
if (std::is_same_v<Signed, signed> && is_negative(rhs) != is_negative(lhs))
|
||||
quotient = operator_unary_minus(quotient);
|
||||
@ -838,8 +866,9 @@ public:
|
||||
{
|
||||
if constexpr (should_keep_size<T>())
|
||||
{
|
||||
integer<Bits, Signed> remainder = make_positive(lhs);
|
||||
divide(remainder, make_positive(integer<Bits, Signed>(rhs)));
|
||||
integer<Bits, unsigned> remainder = make_positive(lhs);
|
||||
integer<Bits, unsigned> denominator = make_positive(integer<Bits, Signed>(rhs));
|
||||
integer<Bits, unsigned>::_impl::divide(remainder, std::move(denominator));
|
||||
|
||||
if (std::is_same_v<Signed, signed> && is_negative(lhs))
|
||||
remainder = operator_unary_minus(remainder);
|
||||
@ -905,7 +934,7 @@ public:
|
||||
++c;
|
||||
}
|
||||
else
|
||||
throwError("invalid char from");
|
||||
throwError("Invalid char from");
|
||||
}
|
||||
}
|
||||
else
|
||||
@ -913,7 +942,7 @@ public:
|
||||
while (*c)
|
||||
{
|
||||
if (*c < '0' || *c > '9')
|
||||
throwError("invalid char from");
|
||||
throwError("Invalid char from");
|
||||
|
||||
res = multiply(res, 10U);
|
||||
res = plus(res, *c - '0');
|
||||
@ -930,11 +959,6 @@ public:
|
||||
|
||||
// Members
|
||||
|
||||
template <size_t Bits, typename Signed>
|
||||
constexpr integer<Bits, Signed>::integer() noexcept
|
||||
: items{}
|
||||
{}
|
||||
|
||||
template <size_t Bits, typename Signed>
|
||||
template <typename T>
|
||||
constexpr integer<Bits, Signed>::integer(T rhs) noexcept
|
||||
@ -943,7 +967,7 @@ constexpr integer<Bits, Signed>::integer(T rhs) noexcept
|
||||
if constexpr (IsWideInteger<T>::value)
|
||||
_impl::wide_integer_from_wide_integer(*this, rhs);
|
||||
else
|
||||
_impl::wide_integer_from_bultin(*this, rhs);
|
||||
_impl::wide_integer_from_builtin(*this, rhs);
|
||||
}
|
||||
|
||||
template <size_t Bits, typename Signed>
|
||||
@ -956,10 +980,19 @@ constexpr integer<Bits, Signed>::integer(std::initializer_list<T> il) noexcept
|
||||
if constexpr (IsWideInteger<T>::value)
|
||||
_impl::wide_integer_from_wide_integer(*this, *il.begin());
|
||||
else
|
||||
_impl::wide_integer_from_bultin(*this, *il.begin());
|
||||
_impl::wide_integer_from_builtin(*this, *il.begin());
|
||||
}
|
||||
else if (il.size() == 0)
|
||||
{
|
||||
_impl::wide_integer_from_builtin(*this, 0);
|
||||
}
|
||||
else
|
||||
_impl::wide_integer_from_bultin(*this, 0);
|
||||
{
|
||||
auto it = il.begin();
|
||||
for (size_t i = 0; i < _impl::item_count; ++i)
|
||||
if (it < il.end())
|
||||
items[i] = *it;
|
||||
}
|
||||
}
|
||||
|
||||
template <size_t Bits, typename Signed>
|
||||
@ -974,7 +1007,7 @@ template <size_t Bits, typename Signed>
|
||||
template <typename T>
|
||||
constexpr integer<Bits, Signed> & integer<Bits, Signed>::operator=(T rhs) noexcept
|
||||
{
|
||||
_impl::wide_integer_from_bultin(*this, rhs);
|
||||
_impl::wide_integer_from_builtin(*this, rhs);
|
||||
return *this;
|
||||
}
|
||||
|
||||
@ -1057,7 +1090,7 @@ constexpr integer<Bits, Signed> & integer<Bits, Signed>::operator>>=(int n) noex
|
||||
{
|
||||
if (static_cast<size_t>(n) >= Bits)
|
||||
{
|
||||
if (is_negative(*this))
|
||||
if (_impl::is_negative(*this))
|
||||
*this = -1;
|
||||
else
|
||||
*this = 0;
|
||||
@ -1107,16 +1140,17 @@ template <size_t Bits, typename Signed>
|
||||
template <class T, class>
|
||||
constexpr integer<Bits, Signed>::operator T() const noexcept
|
||||
{
|
||||
if constexpr (std::is_same_v<T, __int128>)
|
||||
{
|
||||
static_assert(Bits >= 128);
|
||||
return (__int128(items[1]) << 64) | items[0];
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(std::numeric_limits<T>::is_integer);
|
||||
return items[0];
|
||||
}
|
||||
static_assert(std::numeric_limits<T>::is_integer);
|
||||
|
||||
/// NOTE: memcpy will suffice, but unfortunately, this function is constexpr.
|
||||
|
||||
using UnsignedT = std::make_unsigned_t<T>;
|
||||
|
||||
UnsignedT res{};
|
||||
for (unsigned i = 0; i < _impl::item_count && i < (sizeof(T) + sizeof(base_type) - 1) / sizeof(base_type); ++i)
|
||||
res += UnsignedT(items[i]) << (sizeof(base_type) * 8 * i);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
template <size_t Bits, typename Signed>
|
||||
@ -1280,7 +1314,7 @@ template <size_t Bits, typename Signed>
|
||||
constexpr integer<Bits, Signed> operator<<(const integer<Bits, Signed> & lhs, int n) noexcept
|
||||
{
|
||||
if (static_cast<size_t>(n) >= Bits)
|
||||
return 0;
|
||||
return integer<Bits, Signed>(0);
|
||||
if (n <= 0)
|
||||
return lhs;
|
||||
return integer<Bits, Signed>::_impl::shift_left(lhs, n);
|
||||
@ -1289,7 +1323,7 @@ template <size_t Bits, typename Signed>
|
||||
constexpr integer<Bits, Signed> operator>>(const integer<Bits, Signed> & lhs, int n) noexcept
|
||||
{
|
||||
if (static_cast<size_t>(n) >= Bits)
|
||||
return 0;
|
||||
return integer<Bits, Signed>(0);
|
||||
if (n <= 0)
|
||||
return lhs;
|
||||
return integer<Bits, Signed>::_impl::shift_right(lhs, n);
|
||||
@ -1309,7 +1343,7 @@ constexpr bool operator<(const Arithmetic & lhs, const Arithmetic2 & rhs)
|
||||
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
|
||||
constexpr bool operator>(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs)
|
||||
{
|
||||
return std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_more(lhs, rhs);
|
||||
return std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_greater(lhs, rhs);
|
||||
}
|
||||
template <typename Arithmetic, typename Arithmetic2, class>
|
||||
constexpr bool operator>(const Arithmetic & lhs, const Arithmetic2 & rhs)
|
||||
@ -1332,7 +1366,7 @@ constexpr bool operator<=(const Arithmetic & lhs, const Arithmetic2 & rhs)
|
||||
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
|
||||
constexpr bool operator>=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs)
|
||||
{
|
||||
return std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_more(lhs, rhs)
|
||||
return std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_greater(lhs, rhs)
|
||||
|| std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>>::_impl::operator_eq(lhs, rhs);
|
||||
}
|
||||
template <typename Arithmetic, typename Arithmetic2, class>
|
||||
|
@ -1,9 +1,12 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <ostream>
|
||||
#include <fmt/format.h>
|
||||
|
||||
#include "wide_integer.h"
|
||||
|
||||
|
||||
namespace wide
|
||||
{
|
||||
|
||||
@ -33,3 +36,34 @@ inline std::string to_string(const integer<Bits, Signed> & n)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
template <size_t Bits, typename Signed>
|
||||
std::ostream & operator<<(std::ostream & out, const wide::integer<Bits, Signed> & value)
|
||||
{
|
||||
return out << to_string(value);
|
||||
}
|
||||
|
||||
|
||||
/// See https://fmt.dev/latest/api.html#formatting-user-defined-types
|
||||
template <size_t Bits, typename Signed>
|
||||
struct fmt::formatter<wide::integer<Bits, Signed>>
|
||||
{
|
||||
constexpr auto parse(format_parse_context & ctx)
|
||||
{
|
||||
auto it = ctx.begin();
|
||||
auto end = ctx.end();
|
||||
|
||||
/// Only support {}.
|
||||
if (it != end && *it != '}')
|
||||
throw format_error("invalid format");
|
||||
|
||||
return it;
|
||||
}
|
||||
|
||||
template <typename FormatContext>
|
||||
auto format(const wide::integer<Bits, Signed> & value, FormatContext & ctx)
|
||||
{
|
||||
return format_to(ctx.out(), "{}", to_string(value));
|
||||
}
|
||||
};
|
||||
|
@ -102,7 +102,7 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config)
|
||||
auto * logger = &Poco::Logger::get("SentryWriter");
|
||||
if (config.getBool("send_crash_reports.enabled", false))
|
||||
{
|
||||
if (debug || (strlen(VERSION_OFFICIAL) > 0))
|
||||
if (debug || (strlen(VERSION_OFFICIAL) > 0)) //-V560
|
||||
{
|
||||
enabled = true;
|
||||
}
|
||||
|
@ -15,7 +15,7 @@ if (GLIBC_COMPATIBILITY)
|
||||
|
||||
add_headers_and_sources(glibc_compatibility .)
|
||||
add_headers_and_sources(glibc_compatibility musl)
|
||||
if (ARCH_ARM)
|
||||
if (ARCH_AARCH64)
|
||||
list (APPEND glibc_compatibility_sources musl/aarch64/syscall.s musl/aarch64/longjmp.s)
|
||||
set (musl_arch_include_dir musl/aarch64)
|
||||
elseif (ARCH_AMD64)
|
||||
|
@ -78,6 +78,9 @@
|
||||
*
|
||||
*/
|
||||
|
||||
// Disable warnings by PVS-Studio
|
||||
//-V::GA
|
||||
|
||||
static const double
|
||||
pi = 3.14159265358979311600e+00, /* 0x400921FB, 0x54442D18 */
|
||||
a0 = 7.72156649015328655494e-02, /* 0x3FB3C467, 0xE37DB0C8 */
|
||||
|
@ -85,6 +85,9 @@
|
||||
*
|
||||
*/
|
||||
|
||||
// Disable warnings by PVS-Studio
|
||||
//-V::GA
|
||||
|
||||
#include <stdint.h>
|
||||
#include <math.h>
|
||||
#include "libm.h"
|
||||
|
@ -155,7 +155,7 @@ static inline long double fp_barrierl(long double x)
|
||||
static inline void fp_force_evalf(float x)
|
||||
{
|
||||
volatile float y;
|
||||
y = x;
|
||||
y = x; //-V1001
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -164,7 +164,7 @@ static inline void fp_force_evalf(float x)
|
||||
static inline void fp_force_eval(double x)
|
||||
{
|
||||
volatile double y;
|
||||
y = x;
|
||||
y = x; //-V1001
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -173,7 +173,7 @@ static inline void fp_force_eval(double x)
|
||||
static inline void fp_force_evall(long double x)
|
||||
{
|
||||
volatile long double y;
|
||||
y = x;
|
||||
y = x; //-V1001
|
||||
}
|
||||
#endif
|
||||
|
||||
|
@ -3,6 +3,9 @@
|
||||
* SPDX-License-Identifier: MIT
|
||||
*/
|
||||
|
||||
// Disable warnings by PVS-Studio
|
||||
//-V::GA
|
||||
|
||||
#include <math.h>
|
||||
#include <stdint.h>
|
||||
#include "libm.h"
|
||||
|
@ -40,7 +40,7 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log
|
||||
split->addTextLog(log, text_log_max_priority);
|
||||
|
||||
auto current_logger = config.getString("logger", "");
|
||||
if (config_logger == current_logger)
|
||||
if (config_logger == current_logger) //-V1051
|
||||
return;
|
||||
|
||||
config_logger = current_logger;
|
||||
|
@ -447,69 +447,6 @@ inline SrcIter uneven_copy(SrcIter src_first,
|
||||
std::integral_constant<bool, DEST_IS_SMALLER>{});
|
||||
}
|
||||
|
||||
/* generate_to, fill in a fixed-size array of integral type using a SeedSeq
|
||||
* (actually works for any random-access iterator)
|
||||
*/
|
||||
|
||||
template <size_t size, typename SeedSeq, typename DestIter>
|
||||
inline void generate_to_impl(SeedSeq&& generator, DestIter dest,
|
||||
std::true_type)
|
||||
{
|
||||
generator.generate(dest, dest+size);
|
||||
}
|
||||
|
||||
template <size_t size, typename SeedSeq, typename DestIter>
|
||||
void generate_to_impl(SeedSeq&& generator, DestIter dest,
|
||||
std::false_type)
|
||||
{
|
||||
typedef typename std::iterator_traits<DestIter>::value_type dest_t;
|
||||
constexpr auto DEST_SIZE = sizeof(dest_t);
|
||||
constexpr auto GEN_SIZE = sizeof(uint32_t);
|
||||
|
||||
constexpr bool GEN_IS_SMALLER = GEN_SIZE < DEST_SIZE;
|
||||
constexpr size_t FROM_ELEMS =
|
||||
GEN_IS_SMALLER
|
||||
? size * ((DEST_SIZE+GEN_SIZE-1) / GEN_SIZE)
|
||||
: (size + (GEN_SIZE / DEST_SIZE) - 1)
|
||||
/ ((GEN_SIZE / DEST_SIZE) + GEN_IS_SMALLER);
|
||||
// this odd code ^^^^^^^^^^^^^^^^^ is work-around for
|
||||
// a bug: http://llvm.org/bugs/show_bug.cgi?id=21287
|
||||
|
||||
if (FROM_ELEMS <= 1024) {
|
||||
uint32_t buffer[FROM_ELEMS];
|
||||
generator.generate(buffer, buffer+FROM_ELEMS);
|
||||
uneven_copy(buffer, dest, dest+size);
|
||||
} else {
|
||||
uint32_t* buffer = static_cast<uint32_t*>(malloc(GEN_SIZE * FROM_ELEMS));
|
||||
generator.generate(buffer, buffer+FROM_ELEMS);
|
||||
uneven_copy(buffer, dest, dest+size);
|
||||
free(static_cast<void*>(buffer));
|
||||
}
|
||||
}
|
||||
|
||||
template <size_t size, typename SeedSeq, typename DestIter>
|
||||
inline void generate_to(SeedSeq&& generator, DestIter dest)
|
||||
{
|
||||
typedef typename std::iterator_traits<DestIter>::value_type dest_t;
|
||||
constexpr bool IS_32BIT = sizeof(dest_t) == sizeof(uint32_t);
|
||||
|
||||
generate_to_impl<size>(std::forward<SeedSeq>(generator), dest,
|
||||
std::integral_constant<bool, IS_32BIT>{});
|
||||
}
|
||||
|
||||
/* generate_one, produce a value of integral type using a SeedSeq
|
||||
* (optionally, we can have it produce more than one and pick which one
|
||||
* we want)
|
||||
*/
|
||||
|
||||
template <typename UInt, size_t i = 0UL, size_t N = i+1UL, typename SeedSeq>
|
||||
inline UInt generate_one(SeedSeq&& generator)
|
||||
{
|
||||
UInt result[N];
|
||||
generate_to<N>(std::forward<SeedSeq>(generator), result);
|
||||
return result[i];
|
||||
}
|
||||
|
||||
template <typename RngType>
|
||||
auto bounded_rand(RngType& rng, typename RngType::result_type upper_bound)
|
||||
-> typename RngType::result_type
|
||||
@ -517,7 +454,7 @@ auto bounded_rand(RngType& rng, typename RngType::result_type upper_bound)
|
||||
typedef typename RngType::result_type rtype;
|
||||
rtype threshold = (RngType::max() - RngType::min() + rtype(1) - upper_bound)
|
||||
% upper_bound;
|
||||
for (;;) {
|
||||
for (;;) { //-V1044
|
||||
rtype r = rng() - RngType::min();
|
||||
if (r >= threshold)
|
||||
return r % upper_bound;
|
||||
|
@ -928,7 +928,7 @@ struct rxs_m_xs_mixin {
|
||||
constexpr bitcount_t shift = bits - xtypebits;
|
||||
constexpr bitcount_t mask = (1 << opbits) - 1;
|
||||
bitcount_t rshift =
|
||||
opbits ? bitcount_t(internal >> (bits - opbits)) & mask : 0;
|
||||
opbits ? bitcount_t(internal >> (bits - opbits)) & mask : 0; //-V547
|
||||
internal ^= internal >> (opbits + rshift);
|
||||
internal *= mcg_multiplier<itype>::multiplier();
|
||||
xtype result = internal >> shift;
|
||||
@ -950,7 +950,7 @@ struct rxs_m_xs_mixin {
|
||||
|
||||
internal *= mcg_unmultiplier<itype>::unmultiplier();
|
||||
|
||||
bitcount_t rshift = opbits ? (internal >> (bits - opbits)) & mask : 0;
|
||||
bitcount_t rshift = opbits ? (internal >> (bits - opbits)) & mask : 0; //-V547
|
||||
internal = unxorshift(internal, bits, opbits + rshift);
|
||||
|
||||
return internal;
|
||||
@ -975,7 +975,7 @@ struct rxs_m_mixin {
|
||||
: 2;
|
||||
constexpr bitcount_t shift = bits - xtypebits;
|
||||
constexpr bitcount_t mask = (1 << opbits) - 1;
|
||||
bitcount_t rshift = opbits ? (internal >> (bits - opbits)) & mask : 0;
|
||||
bitcount_t rshift = opbits ? (internal >> (bits - opbits)) & mask : 0; //-V547
|
||||
internal ^= internal >> (opbits + rshift);
|
||||
internal *= mcg_multiplier<itype>::multiplier();
|
||||
xtype result = internal >> shift;
|
||||
@ -1366,7 +1366,7 @@ void extended<table_pow2,advance_pow2,baseclass,extvalclass,kdd>::selfinit()
|
||||
// - any strange correlations would only be apparent if we
|
||||
// were to backstep the generator so that the base generator
|
||||
// was generating the same values again
|
||||
result_type xdiff = baseclass::operator()() - baseclass::operator()();
|
||||
result_type xdiff = baseclass::operator()() - baseclass::operator()(); //-V501
|
||||
for (size_t i = 0; i < table_size; ++i) {
|
||||
data_[i] = baseclass::operator()() ^ xdiff;
|
||||
}
|
||||
|
@ -2,7 +2,7 @@ if (APPLE OR SPLIT_SHARED_LIBRARIES OR NOT ARCH_AMD64)
|
||||
set (ENABLE_EMBEDDED_COMPILER OFF CACHE INTERNAL "")
|
||||
endif()
|
||||
|
||||
option (ENABLE_EMBEDDED_COMPILER "Set to TRUE to enable support for 'compile_expressions' option for query execution" ${ENABLE_LIBRARIES})
|
||||
option (ENABLE_EMBEDDED_COMPILER "Enable support for 'compile_expressions' option for query execution" ON)
|
||||
# Broken in macos. TODO: update clang, re-test, enable on Apple
|
||||
if (ENABLE_EMBEDDED_COMPILER AND NOT SPLIT_SHARED_LIBRARIES AND ARCH_AMD64 AND NOT (SANITIZE STREQUAL "undefined"))
|
||||
option (USE_INTERNAL_LLVM_LIBRARY "Use bundled or system LLVM library." ${NOT_UNBUNDLED})
|
||||
|
2
contrib/datasketches-cpp
vendored
2
contrib/datasketches-cpp
vendored
@ -1 +1 @@
|
||||
Subproject commit f915d35b2de676683493c86c585141a1e1c83334
|
||||
Subproject commit 7d73d7610db31d4e1ecde0fb3a7ee90ef371207f
|
@ -308,6 +308,8 @@ function run_tests
|
||||
01354_order_by_tuple_collate_const
|
||||
01355_ilike
|
||||
01411_bayesian_ab_testing
|
||||
01798_uniq_theta_sketch
|
||||
01799_long_uniq_theta_sketch
|
||||
collate
|
||||
collation
|
||||
_orc_
|
||||
@ -370,6 +372,10 @@ function run_tests
|
||||
|
||||
# Depends on AWS
|
||||
01801_s3_cluster
|
||||
|
||||
# Depends on LLVM JIT
|
||||
01852_jit_if
|
||||
01865_jit_comparison_constant_result
|
||||
)
|
||||
|
||||
(time clickhouse-test --hung-check -j 8 --order=random --use-skip-list --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" -- "$FASTTEST_FOCUS" 2>&1 ||:) | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/test_log.txt"
|
||||
|
@ -14,11 +14,6 @@
|
||||
<max_memory_usage>
|
||||
<max>10G</max>
|
||||
</max_memory_usage>
|
||||
|
||||
<!-- Not ready for production -->
|
||||
<compile_expressions>
|
||||
<readonly />
|
||||
</compile_expressions>
|
||||
</constraints>
|
||||
</default>
|
||||
</profiles>
|
||||
|
@ -35,7 +35,7 @@ RUN apt-get update \
|
||||
ENV TZ=Europe/Moscow
|
||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||
|
||||
RUN pip3 install urllib3 testflows==1.6.74 docker-compose docker dicttoxml kazoo tzlocal
|
||||
RUN pip3 install urllib3 testflows==1.6.74 docker-compose docker dicttoxml kazoo tzlocal python-dateutil numpy
|
||||
|
||||
ENV DOCKER_CHANNEL stable
|
||||
ENV DOCKER_VERSION 17.09.1-ce
|
||||
@ -74,4 +74,3 @@ VOLUME /var/lib/docker
|
||||
EXPOSE 2375
|
||||
ENTRYPOINT ["dockerd-entrypoint.sh"]
|
||||
CMD ["sh", "-c", "python3 regression.py --no-color -o classic --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json; /usr/local/bin/process_testflows_result.py || echo -e 'failure\tCannot parse results' > check_status.tsv"]
|
||||
|
||||
|
@ -31,10 +31,10 @@ toc_title: Cloud
|
||||
|
||||
## Alibaba Cloud {#alibaba-cloud}
|
||||
|
||||
Alibaba Cloud Managed Service for ClickHouse. [China Site](https://www.aliyun.com/product/clickhouse) (will be available at the international site in May 2021). Provides the following key features:
|
||||
[Alibaba Cloud Managed Service for ClickHouse](https://www.alibabacloud.com/product/clickhouse) provides the following key features:
|
||||
|
||||
- Highly reliable cloud disk storage engine based on [Alibaba Cloud Apsara](https://www.alibabacloud.com/product/apsara-stack) distributed system
|
||||
- Expand capacity on-demand without manual data migration
|
||||
- Expand capacity on demand without manual data migration
|
||||
- Support single-node, single-replica, multi-node, and multi-replica architectures, and support hot and cold data tiering
|
||||
- Support access allow-list, one-key recovery, multi-layer network security protection, cloud disk encryption
|
||||
- Seamless integration with cloud log systems, databases, and data application tools
|
||||
|
@ -139,6 +139,7 @@ The following settings can be specified in configuration file for given endpoint
|
||||
|
||||
- `endpoint` — Specifies prefix of an endpoint. Mandatory.
|
||||
- `access_key_id` and `secret_access_key` — Specifies credentials to use with given endpoint. Optional.
|
||||
- `region` — Specifies S3 region name. Optional.
|
||||
- `use_environment_credentials` — If set to `true`, S3 client will try to obtain credentials from environment variables and Amazon EC2 metadata for given endpoint. Optional, default value is `false`.
|
||||
- `use_insecure_imds_request` — If set to `true`, S3 client will use insecure IMDS request while obtaining credentials from Amazon EC2 metadata. Optional, default value is `false`.
|
||||
- `header` — Adds specified HTTP header to a request to given endpoint. Optional, can be speficied multiple times.
|
||||
@ -152,6 +153,7 @@ The following settings can be specified in configuration file for given endpoint
|
||||
<endpoint>https://storage.yandexcloud.net/my-test-bucket-768/</endpoint>
|
||||
<!-- <access_key_id>ACCESS_KEY_ID</access_key_id> -->
|
||||
<!-- <secret_access_key>SECRET_ACCESS_KEY</secret_access_key> -->
|
||||
<!-- <region>us-west-1</region> -->
|
||||
<!-- <use_environment_credentials>false</use_environment_credentials> -->
|
||||
<!-- <use_insecure_imds_request>false</use_insecure_imds_request> -->
|
||||
<!-- <header>Authorization: Bearer SOME-TOKEN</header> -->
|
||||
|
@ -739,6 +739,7 @@ Configuration markup:
|
||||
<endpoint>https://storage.yandexcloud.net/my-bucket/root-path/</endpoint>
|
||||
<access_key_id>your_access_key_id</access_key_id>
|
||||
<secret_access_key>your_secret_access_key</secret_access_key>
|
||||
<region></region>
|
||||
<server_side_encryption_customer_key_base64>your_base64_encoded_customer_key</server_side_encryption_customer_key_base64>
|
||||
<proxy>
|
||||
<uri>http://proxy1</uri>
|
||||
@ -764,6 +765,7 @@ Required parameters:
|
||||
- `secret_access_key` — S3 secret access key.
|
||||
|
||||
Optional parameters:
|
||||
- `region` — S3 region name.
|
||||
- `use_environment_credentials` — Reads AWS credentials from the Environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY and AWS_SESSION_TOKEN if they exist. Default value is `false`.
|
||||
- `use_insecure_imds_request` — If set to `true`, S3 client will use insecure IMDS request while obtaining credentials from Amazon EC2 metadata. Default value is `false`.
|
||||
- `proxy` — Proxy configuration for S3 endpoint. Each `uri` element inside `proxy` block should contain a proxy URL.
|
||||
|
@ -21,6 +21,7 @@ Columns:
|
||||
- `bytes_allocated` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Amount of RAM allocated for the dictionary.
|
||||
- `query_count` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of queries since the dictionary was loaded or since the last successful reboot.
|
||||
- `hit_rate` ([Float64](../../sql-reference/data-types/float.md)) — For cache dictionaries, the percentage of uses for which the value was in the cache.
|
||||
- `found_rate` ([Float64](../../sql-reference/data-types/float.md)) — The percentage of uses for which the value was found.
|
||||
- `element_count` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of items stored in the dictionary.
|
||||
- `load_factor` ([Float64](../../sql-reference/data-types/float.md)) — Percentage filled in the dictionary (for a hashed dictionary, the percentage filled in the hash table).
|
||||
- `source` ([String](../../sql-reference/data-types/string.md)) — Text describing the [data source](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md) for the dictionary.
|
||||
|
@ -12,6 +12,9 @@ The result depends on the order of running the query, and is nondeterministic.
|
||||
|
||||
When using multiple `quantile*` functions with different levels in a query, the internal states are not combined (that is, the query works less efficiently than it could). In this case, use the [quantiles](../../../sql-reference/aggregate-functions/reference/quantiles.md#quantiles) function.
|
||||
|
||||
!!! note "Note"
|
||||
Using `quantileTDigestWeighted` [is not recommended for tiny data sets](https://github.com/tdunning/t-digest/issues/167#issuecomment-828650275) and can lead to significat error. In this case, consider possibility of using [`quantileTDigest`](../../../sql-reference/aggregate-functions/reference/quantiletdigest.md) instead.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
|
@ -38,3 +38,4 @@ We recommend using this function in almost all scenarios.
|
||||
- [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64)
|
||||
- [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniqhll12.md#agg_function-uniqhll12)
|
||||
- [uniqExact](../../../sql-reference/aggregate-functions/reference/uniqexact.md#agg_function-uniqexact)
|
||||
- [uniqThetaSketch](../../../sql-reference/aggregate-functions/reference/uniqthetasketch.md#agg_function-uniqthetasketch)
|
||||
|
@ -49,3 +49,4 @@ Compared to the [uniq](../../../sql-reference/aggregate-functions/reference/uniq
|
||||
- [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64)
|
||||
- [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniqhll12.md#agg_function-uniqhll12)
|
||||
- [uniqExact](../../../sql-reference/aggregate-functions/reference/uniqexact.md#agg_function-uniqexact)
|
||||
- [uniqThetaSketch](../../../sql-reference/aggregate-functions/reference/uniqthetasketch.md#agg_function-uniqthetasketch)
|
||||
|
@ -23,3 +23,4 @@ The function takes a variable number of parameters. Parameters can be `Tuple`, `
|
||||
- [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq)
|
||||
- [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniqcombined)
|
||||
- [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniqhll12)
|
||||
- [uniqThetaSketch](../../../sql-reference/aggregate-functions/reference/uniqthetasketch.md#agg_function-uniqthetasketch)
|
||||
|
@ -37,3 +37,4 @@ We don’t recommend using this function. In most cases, use the [uniq](../../..
|
||||
- [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq)
|
||||
- [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md#agg_function-uniqcombined)
|
||||
- [uniqExact](../../../sql-reference/aggregate-functions/reference/uniqexact.md#agg_function-uniqexact)
|
||||
- [uniqThetaSketch](../../../sql-reference/aggregate-functions/reference/uniqthetasketch.md#agg_function-uniqthetasketch)
|
||||
|
@ -0,0 +1,39 @@
|
||||
---
|
||||
toc_priority: 195
|
||||
---
|
||||
|
||||
# uniqThetaSketch {#agg_function-uniqthetasketch}
|
||||
|
||||
Calculates the approximate number of different argument values, using the [Theta Sketch Framework](https://datasketches.apache.org/docs/Theta/ThetaSketchFramework.html).
|
||||
|
||||
``` sql
|
||||
uniqThetaSketch(x[, ...])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
The function takes a variable number of parameters. Parameters can be `Tuple`, `Array`, `Date`, `DateTime`, `String`, or numeric types.
|
||||
|
||||
**Returned value**
|
||||
|
||||
- A [UInt64](../../../sql-reference/data-types/int-uint.md)-type number.
|
||||
|
||||
**Implementation details**
|
||||
|
||||
Function:
|
||||
|
||||
- Calculates a hash for all parameters in the aggregate, then uses it in calculations.
|
||||
|
||||
- Uses the [KMV](https://datasketches.apache.org/docs/Theta/InverseEstimate.html) algorithm to approximate the number of different argument values.
|
||||
|
||||
4096(2^12) 64-bit sketch are used. The size of the state is about 41 KB.
|
||||
|
||||
- The relative error is 3.125% (95% confidence), see the [relative error table](https://datasketches.apache.org/docs/Theta/ThetaErrorTable.html) for detail.
|
||||
|
||||
**See Also**
|
||||
|
||||
- [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq)
|
||||
- [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md#agg_function-uniqcombined)
|
||||
- [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64)
|
||||
- [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniqhll12.md#agg_function-uniqhll12)
|
||||
- [uniqExact](../../../sql-reference/aggregate-functions/reference/uniqexact.md#agg_function-uniqexact)
|
@ -47,9 +47,9 @@ Union
|
||||
|
||||
### EXPLAIN AST {#explain-ast}
|
||||
|
||||
Dump query AST.
|
||||
Dump query AST. Supports all types of queries, not only `SELECT`.
|
||||
|
||||
Example:
|
||||
Examples:
|
||||
|
||||
```sql
|
||||
EXPLAIN AST SELECT 1;
|
||||
@ -63,6 +63,22 @@ SelectWithUnionQuery (children 1)
|
||||
Literal UInt64_1
|
||||
```
|
||||
|
||||
```sql
|
||||
EXPLAIN AST ALTER TABLE t1 DELETE WHERE date = today();
|
||||
```
|
||||
|
||||
```sql
|
||||
explain
|
||||
AlterQuery t1 (children 1)
|
||||
ExpressionList (children 1)
|
||||
AlterCommand 27 (children 1)
|
||||
Function equals (children 1)
|
||||
ExpressionList (children 2)
|
||||
Identifier date
|
||||
Function today (children 1)
|
||||
ExpressionList
|
||||
```
|
||||
|
||||
### EXPLAIN SYNTAX {#explain-syntax}
|
||||
|
||||
Returns query after syntax optimizations.
|
||||
|
@ -22,7 +22,7 @@ toc_title: "\u30AF\u30E9\u30A6\u30C9"
|
||||
|
||||
## Alibaba Cloud {#alibaba-cloud}
|
||||
|
||||
ClickHouseのためのAlibaba Cloudの管理サービス [中国サイト](https://www.aliyun.com/product/clickhouse) (2021年5月に国際サイトで利用可能になります) 次の主な機能を提供します:
|
||||
[ClickHouseのためのAlibaba Cloudの管理サービス](https://www.alibabacloud.com/product/clickhouse) 次の主な機能を提供します:
|
||||
|
||||
- Alibaba Cloud Apsara分散システムをベースにした信頼性の高いクラウドディスクストレージエンジン
|
||||
- 手動でのデータ移行を必要とせずに、オン・デマンドで容量を拡張
|
||||
|
@ -82,6 +82,7 @@ SELECT * FROM s3_engine_table LIMIT 2;
|
||||
|
||||
Необязательные настройки:
|
||||
- `access_key_id` и `secret_access_key` — указывают учетные данные для использования с данной точкой приема запроса.
|
||||
- `region` — название региона S3.
|
||||
- `use_environment_credentials` — если `true`, S3-клиент будет пытаться получить учетные данные из переменных среды и метаданных Amazon EC2 для данной точки приема запроса. Значение по умолчанию - `false`.
|
||||
- `header` — добавляет указанный HTTP-заголовок к запросу на заданную точку приема запроса. Может быть определен несколько раз.
|
||||
- `server_side_encryption_customer_key_base64` — устанавливает необходимые заголовки для доступа к объектам S3 с шифрованием SSE-C.
|
||||
@ -94,6 +95,7 @@ SELECT * FROM s3_engine_table LIMIT 2;
|
||||
<endpoint>https://storage.yandexcloud.net/my-test-bucket-768/</endpoint>
|
||||
<!-- <access_key_id>ACCESS_KEY_ID</access_key_id> -->
|
||||
<!-- <secret_access_key>SECRET_ACCESS_KEY</secret_access_key> -->
|
||||
<!-- <region>us-west-1</region> -->
|
||||
<!-- <use_environment_credentials>false</use_environment_credentials> -->
|
||||
<!-- <header>Authorization: Bearer SOME-TOKEN</header> -->
|
||||
<!-- <server_side_encryption_customer_key_base64>BASE64-ENCODED-KEY</server_side_encryption_customer_key_base64> -->
|
||||
|
@ -727,6 +727,7 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd'
|
||||
<endpoint>https://storage.yandexcloud.net/my-bucket/root-path/</endpoint>
|
||||
<access_key_id>your_access_key_id</access_key_id>
|
||||
<secret_access_key>your_secret_access_key</secret_access_key>
|
||||
<region></region>
|
||||
<proxy>
|
||||
<uri>http://proxy1</uri>
|
||||
<uri>http://proxy2</uri>
|
||||
@ -753,6 +754,7 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd'
|
||||
|
||||
Необязательные параметры:
|
||||
|
||||
- `region` — название региона S3.
|
||||
- `use_environment_credentials` — признак, нужно ли считывать учетные данные AWS из сетевого окружения, а также из переменных окружения `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` и `AWS_SESSION_TOKEN`, если они есть. Значение по умолчанию: `false`.
|
||||
- `use_insecure_imds_request` — признак, нужно ли использовать менее безопасное соединение при выполнении запроса к IMDS при получении учётных данных из метаданных Amazon EC2. Значение по умолчанию: `false`.
|
||||
- `proxy` — конфигурация прокси-сервера для конечной точки S3. Каждый элемент `uri` внутри блока `proxy` должен содержать URL прокси-сервера.
|
||||
|
@ -1,248 +0,0 @@
|
||||
---
|
||||
toc_priority: 58
|
||||
toc_title: "Советы по эксплуатации"
|
||||
---
|
||||
|
||||
# Советы по эксплуатации {#sovety-po-ekspluatatsii}
|
||||
|
||||
## CPU Scaling Governor {#cpu-scaling-governor}
|
||||
|
||||
Всегда используйте `performance` scaling governor. `ondemand` scaling governor работает намного хуже при постоянно высоком спросе.
|
||||
|
||||
``` bash
|
||||
$ echo 'performance' | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
|
||||
```
|
||||
|
||||
## Ограничение CPU {#ogranichenie-cpu}
|
||||
|
||||
Процессоры могут перегреваться. С помощью `dmesg` можно увидеть, если тактовая частота процессора была ограничена из-за перегрева.
|
||||
Также ограничение может устанавливаться снаружи на уровне дата-центра. С помощью `turbostat` можно за этим наблюдать под нагрузкой.
|
||||
|
||||
## Оперативная память {#operativnaia-pamiat}
|
||||
|
||||
Для небольших объёмов данных (до ~200 Гб в сжатом виде) лучше всего использовать столько памяти не меньше, чем объём данных.
|
||||
Для больших объёмов данных, при выполнении интерактивных (онлайн) запросов, стоит использовать разумный объём оперативной памяти (128 Гб или более) для того, чтобы горячее подмножество данных поместилось в кеше страниц.
|
||||
Даже для объёмов данных в ~50 Тб на сервер, использование 128 Гб оперативной памяти намного лучше для производительности выполнения запросов, чем 64 Гб.
|
||||
|
||||
Не выключайте overcommit. Значение `cat /proc/sys/vm/overcommit_memory` должно быть 0 or 1. Выполните:
|
||||
|
||||
``` bash
|
||||
$ echo 0 | sudo tee /proc/sys/vm/overcommit_memory
|
||||
```
|
||||
|
||||
## Huge Pages {#huge-pages}
|
||||
|
||||
Механизм прозрачных huge pages нужно отключить. Он мешает работе аллокаторов памяти, что приводит к значительной деградации производительности.
|
||||
|
||||
``` bash
|
||||
$ echo 'madvise' | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
|
||||
```
|
||||
|
||||
С помощью `perf top` можно наблюдать за временем, проведенном в ядре операционной системы для управления памятью.
|
||||
Постоянные huge pages так же не нужно аллоцировать.
|
||||
|
||||
## Подсистема хранения {#podsistema-khraneniia}
|
||||
|
||||
Если ваш бюджет позволяет использовать SSD, используйте SSD.
|
||||
В противном случае используйте HDD. SATA HDDs 7200 RPM подойдут.
|
||||
|
||||
Предпочитайте много серверов с локальными жесткими дисками вместо меньшего числа серверов с подключенными дисковыми полками.
|
||||
Но для хранения архивов с редкими запросами полки всё же подходят.
|
||||
|
||||
## RAID {#raid}
|
||||
|
||||
При использовании HDD можно объединить их RAID-10, RAID-5, RAID-6 или RAID-50.
|
||||
Лучше использовать программный RAID в Linux (`mdadm`). Лучше не использовать LVM.
|
||||
При создании RAID-10, нужно выбрать `far` расположение.
|
||||
Если бюджет позволяет, лучше выбрать RAID-10.
|
||||
|
||||
На более чем 4 дисках вместо RAID-5 нужно использовать RAID-6 (предпочтительнее) или RAID-50.
|
||||
При использовании RAID-5, RAID-6 или RAID-50, нужно всегда увеличивать stripe_cache_size, так как значение по умолчанию выбрано не самым удачным образом.
|
||||
|
||||
``` bash
|
||||
$ echo 4096 | sudo tee /sys/block/md2/md/stripe_cache_size
|
||||
```
|
||||
|
||||
Точное число стоит вычислять из числа устройств и размер блока по формуле: `2 * num_devices * chunk_size_in_bytes / 4096`.
|
||||
|
||||
Размер блока в 1024 Кб подходит для всех конфигураций RAID.
|
||||
Никогда не указывайте слишком маленький или слишком большой размер блока.
|
||||
|
||||
На SSD можно использовать RAID-0.
|
||||
Вне зависимости от использования RAID, всегда используйте репликацию для безопасности данных.
|
||||
|
||||
Включите NCQ с длинной очередью. Для HDD стоит выбрать планировщик CFQ, а для SSD — noop. Не стоит уменьшать настройку readahead.
|
||||
На HDD стоит включать кеш записи.
|
||||
|
||||
## Файловая система {#failovaia-sistema}
|
||||
|
||||
Ext4 самый проверенный вариант. Укажите опции монтирования `noatime,nobarrier`.
|
||||
XFS также подходит, но не так тщательно протестирована в сочетании с ClickHouse.
|
||||
Большинство других файловых систем также должны нормально работать. Файловые системы с отложенной аллокацией работают лучше.
|
||||
|
||||
## Ядро Linux {#iadro-linux}
|
||||
|
||||
Не используйте слишком старое ядро Linux.
|
||||
|
||||
## Сеть {#set}
|
||||
|
||||
При использовании IPv6, стоит увеличить размер кеша маршрутов.
|
||||
Ядра Linux до 3.2 имели массу проблем в реализации IPv6.
|
||||
|
||||
Предпочитайте как минимум 10 Гбит сеть. 1 Гбит также будет работать, но намного хуже для починки реплик с десятками терабайт данных или для обработки распределенных запросов с большим объёмом промежуточных данных.
|
||||
|
||||
## ZooKeeper {#zookeeper}
|
||||
|
||||
Вероятно вы уже используете ZooKeeper для других целей. Можно использовать ту же инсталляцию ZooKeeper, если она не сильно перегружена.
|
||||
|
||||
Лучше использовать свежую версию ZooKeeper, как минимум 3.4.9. Версия в стабильных дистрибутивах Linux может быть устаревшей.
|
||||
|
||||
Никогда не используете написанные вручную скрипты для переноса данных между разными ZooKeeper кластерами, потому что результат будет некорректный для sequential нод. Никогда не используйте утилиту «zkcopy», по той же причине: https://github.com/ksprojects/zkcopy/issues/15
|
||||
|
||||
Если вы хотите разделить существующий ZooKeeper кластер на два, правильный способ - увеличить количество его реплик, а затем переконфигурировать его как два независимых кластера.
|
||||
|
||||
Не запускайте ZooKeeper на тех же серверах, что и ClickHouse. Потому что ZooKeeper очень чувствителен к задержкам, а ClickHouse может использовать все доступные системные ресурсы.
|
||||
|
||||
С настройками по умолчанию, ZooKeeper является бомбой замедленного действия:
|
||||
|
||||
> Сервер ZooKeeper не будет удалять файлы со старыми снепшоты и логами при использовании конфигурации по умолчанию (см. autopurge), это является ответственностью оператора.
|
||||
|
||||
Эту бомбу нужно обезвредить.
|
||||
|
||||
Далее описана конфигурация ZooKeeper (3.5.1), используемая в боевом окружении Яндекс.Метрики на момент 20 мая 2017 года:
|
||||
|
||||
zoo.cfg:
|
||||
|
||||
``` bash
|
||||
# http://hadoop.apache.org/zookeeper/docs/current/zookeeperAdmin.html
|
||||
|
||||
# The number of milliseconds of each tick
|
||||
tickTime=2000
|
||||
# The number of ticks that the initial
|
||||
# synchronization phase can take
|
||||
initLimit=30000
|
||||
# The number of ticks that can pass between
|
||||
# sending a request and getting an acknowledgement
|
||||
syncLimit=10
|
||||
|
||||
maxClientCnxns=2000
|
||||
|
||||
maxSessionTimeout=60000000
|
||||
# the directory where the snapshot is stored.
|
||||
dataDir=/opt/zookeeper/{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/data
|
||||
# Place the dataLogDir to a separate physical disc for better performance
|
||||
dataLogDir=/opt/zookeeper/{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/logs
|
||||
|
||||
autopurge.snapRetainCount=10
|
||||
autopurge.purgeInterval=1
|
||||
|
||||
|
||||
# To avoid seeks ZooKeeper allocates space in the transaction log file in
|
||||
# blocks of preAllocSize kilobytes. The default block size is 64M. One reason
|
||||
# for changing the size of the blocks is to reduce the block size if snapshots
|
||||
# are taken more often. (Also, see snapCount).
|
||||
preAllocSize=131072
|
||||
|
||||
# Clients can submit requests faster than ZooKeeper can process them,
|
||||
# especially if there are a lot of clients. To prevent ZooKeeper from running
|
||||
# out of memory due to queued requests, ZooKeeper will throttle clients so that
|
||||
# there is no more than globalOutstandingLimit outstanding requests in the
|
||||
# system. The default limit is 1,000.ZooKeeper logs transactions to a
|
||||
# transaction log. After snapCount transactions are written to a log file a
|
||||
# snapshot is started and a new transaction log file is started. The default
|
||||
# snapCount is 10,000.
|
||||
snapCount=3000000
|
||||
|
||||
# If this option is defined, requests will be will logged to a trace file named
|
||||
# traceFile.year.month.day.
|
||||
#traceFile=
|
||||
|
||||
# Leader accepts client connections. Default value is "yes". The leader machine
|
||||
# coordinates updates. For higher update throughput at thes slight expense of
|
||||
# read throughput the leader can be configured to not accept clients and focus
|
||||
# on coordination.
|
||||
leaderServes=yes
|
||||
|
||||
standaloneEnabled=false
|
||||
dynamicConfigFile=/etc/zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/conf/zoo.cfg.dynamic
|
||||
```
|
||||
|
||||
Версия Java:
|
||||
|
||||
``` text
|
||||
Java(TM) SE Runtime Environment (build 1.8.0_25-b17)
|
||||
Java HotSpot(TM) 64-Bit Server VM (build 25.25-b02, mixed mode)
|
||||
```
|
||||
|
||||
Параметры JVM:
|
||||
|
||||
``` bash
|
||||
NAME=zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}
|
||||
ZOOCFGDIR=/etc/$NAME/conf
|
||||
|
||||
# TODO this is really ugly
|
||||
# How to find out, which jars are needed?
|
||||
# seems, that log4j requires the log4j.properties file to be in the classpath
|
||||
CLASSPATH="$ZOOCFGDIR:/usr/build/classes:/usr/build/lib/*.jar:/usr/share/zookeeper/zookeeper-3.5.1-metrika.jar:/usr/share/zookeeper/slf4j-log4j12-1.7.5.jar:/usr/share/zookeeper/slf4j-api-1.7.5.jar:/usr/share/zookeeper/servlet-api-2.5-20081211.jar:/usr/share/zookeeper/netty-3.7.0.Final.jar:/usr/share/zookeeper/log4j-1.2.16.jar:/usr/share/zookeeper/jline-2.11.jar:/usr/share/zookeeper/jetty-util-6.1.26.jar:/usr/share/zookeeper/jetty-6.1.26.jar:/usr/share/zookeeper/javacc.jar:/usr/share/zookeeper/jackson-mapper-asl-1.9.11.jar:/usr/share/zookeeper/jackson-core-asl-1.9.11.jar:/usr/share/zookeeper/commons-cli-1.2.jar:/usr/src/java/lib/*.jar:/usr/etc/zookeeper"
|
||||
|
||||
ZOOCFG="$ZOOCFGDIR/zoo.cfg"
|
||||
ZOO_LOG_DIR=/var/log/$NAME
|
||||
USER=zookeeper
|
||||
GROUP=zookeeper
|
||||
PIDDIR=/var/run/$NAME
|
||||
PIDFILE=$PIDDIR/$NAME.pid
|
||||
SCRIPTNAME=/etc/init.d/$NAME
|
||||
JAVA=/usr/bin/java
|
||||
ZOOMAIN="org.apache.zookeeper.server.quorum.QuorumPeerMain"
|
||||
ZOO_LOG4J_PROP="INFO,ROLLINGFILE"
|
||||
JMXLOCALONLY=false
|
||||
JAVA_OPTS="-Xms{{ '{{' }} cluster.get('xms','128M') {{ '{{' }} '}}' }} \
|
||||
-Xmx{{ '{{' }} cluster.get('xmx','1G') {{ '{{' }} '}}' }} \
|
||||
-Xloggc:/var/log/$NAME/zookeeper-gc.log \
|
||||
-XX:+UseGCLogFileRotation \
|
||||
-XX:NumberOfGCLogFiles=16 \
|
||||
-XX:GCLogFileSize=16M \
|
||||
-verbose:gc \
|
||||
-XX:+PrintGCTimeStamps \
|
||||
-XX:+PrintGCDateStamps \
|
||||
-XX:+PrintGCDetails
|
||||
-XX:+PrintTenuringDistribution \
|
||||
-XX:+PrintGCApplicationStoppedTime \
|
||||
-XX:+PrintGCApplicationConcurrentTime \
|
||||
-XX:+PrintSafepointStatistics \
|
||||
-XX:+UseParNewGC \
|
||||
-XX:+UseConcMarkSweepGC \
|
||||
-XX:+CMSParallelRemarkEnabled"
|
||||
```
|
||||
|
||||
Salt init:
|
||||
|
||||
``` text
|
||||
description "zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }} centralized coordination service"
|
||||
|
||||
start on runlevel [2345]
|
||||
stop on runlevel [!2345]
|
||||
|
||||
respawn
|
||||
|
||||
limit nofile 8192 8192
|
||||
|
||||
pre-start script
|
||||
[ -r "/etc/zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/conf/environment" ] || exit 0
|
||||
. /etc/zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/conf/environment
|
||||
[ -d $ZOO_LOG_DIR ] || mkdir -p $ZOO_LOG_DIR
|
||||
chown $USER:$GROUP $ZOO_LOG_DIR
|
||||
end script
|
||||
|
||||
script
|
||||
. /etc/zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }}/conf/environment
|
||||
[ -r /etc/default/zookeeper ] && . /etc/default/zookeeper
|
||||
if [ -z "$JMXDISABLE" ]; then
|
||||
JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY"
|
||||
fi
|
||||
exec start-stop-daemon --start -c $USER --exec $JAVA --name zookeeper-{{ '{{' }} cluster['name'] {{ '{{' }} '}}' }} \
|
||||
-- -cp $CLASSPATH $JAVA_OPTS -Dzookeeper.log.dir=${ZOO_LOG_DIR} \
|
||||
-Dzookeeper.root.logger=${ZOO_LOG4J_PROP} $ZOOMAIN $ZOOCFG
|
||||
end script
|
||||
```
|
||||
|
1
docs/ru/operations/tips.md
Symbolic link
1
docs/ru/operations/tips.md
Symbolic link
@ -0,0 +1 @@
|
||||
../../en/operations/tips.md
|
@ -12,6 +12,9 @@ toc_priority: 208
|
||||
|
||||
Внутренние состояния функций `quantile*` не объединяются, если они используются в одном запросе. Если вам необходимо вычислить квантили нескольких уровней, используйте функцию [quantiles](#quantiles), это повысит эффективность запроса.
|
||||
|
||||
!!! note "Примечание"
|
||||
Использование `quantileTDigestWeighted` [не рекомендуется для небольших наборов данных](https://github.com/tdunning/t-digest/issues/167#issuecomment-828650275) и может привести к значительной ошибке. Рассмотрите возможность использования [`quantileTDigest`](../../../sql-reference/aggregate-functions/reference/quantiletdigest.md) в таких случаях.
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
|
@ -47,9 +47,9 @@ Union
|
||||
|
||||
### EXPLAIN AST {#explain-ast}
|
||||
|
||||
Дамп AST запроса.
|
||||
Дамп AST запроса. Поддерживает все типы запросов, не только `SELECT`.
|
||||
|
||||
Пример:
|
||||
Примеры:
|
||||
|
||||
```sql
|
||||
EXPLAIN AST SELECT 1;
|
||||
@ -63,6 +63,22 @@ SelectWithUnionQuery (children 1)
|
||||
Literal UInt64_1
|
||||
```
|
||||
|
||||
```sql
|
||||
EXPLAIN AST ALTER TABLE t1 DELETE WHERE date = today();
|
||||
```
|
||||
|
||||
```sql
|
||||
explain
|
||||
AlterQuery t1 (children 1)
|
||||
ExpressionList (children 1)
|
||||
AlterCommand 27 (children 1)
|
||||
Function equals (children 1)
|
||||
ExpressionList (children 2)
|
||||
Identifier date
|
||||
Function today (children 1)
|
||||
ExpressionList
|
||||
```
|
||||
|
||||
### EXPLAIN SYNTAX {#explain-syntax}
|
||||
|
||||
Возвращает текст запроса после применения синтаксических оптимизаций.
|
||||
|
@ -31,7 +31,7 @@ toc_title: 云
|
||||
|
||||
## 阿里云 {#alibaba-cloud}
|
||||
|
||||
阿里云的 ClickHouse 托管服务 [中国站](https://www.aliyun.com/product/clickhouse) (国际站于2021年5月初开放) 提供以下主要功能:
|
||||
[阿里云的 ClickHouse 托管服务](https://www.alibabacloud.com/zh/product/clickhouse) 提供以下主要功能:
|
||||
|
||||
- 基于阿里飞天分布式系统的高可靠云盘存储引擎
|
||||
- 按需扩容,无需手动进行数据搬迁
|
||||
|
@ -1399,17 +1399,27 @@ private:
|
||||
// when `lambda()` function gets substituted into a wrong place.
|
||||
// To avoid dealing with these cases, run the check only for the
|
||||
// queries we were able to successfully execute.
|
||||
// The final caveat is that sometimes WITH queries are not executed,
|
||||
// Another caveat is that sometimes WITH queries are not executed,
|
||||
// if they are not referenced by the main SELECT, so they can still
|
||||
// have the aforementioned problems. Disable this check for such
|
||||
// queries, for lack of a better solution.
|
||||
if (!have_error && queryHasWithClause(parsed_query.get()))
|
||||
// There is also a problem that fuzzer substitutes positive Int64
|
||||
// literals or Decimal literals, which are then parsed back as
|
||||
// UInt64, and suddenly duplicate alias substitition starts or stops
|
||||
// working (ASTWithAlias::formatImpl) or something like that.
|
||||
// So we compare not even the first and second formatting of the
|
||||
// query, but second and third.
|
||||
// If you have to add any more workarounds to this check, just remove
|
||||
// it altogether, it's not so useful.
|
||||
if (!have_error && !queryHasWithClause(parsed_query.get()))
|
||||
{
|
||||
ASTPtr parsed_formatted_query;
|
||||
ASTPtr ast_2;
|
||||
try
|
||||
{
|
||||
const auto * tmp_pos = query_to_send.c_str();
|
||||
parsed_formatted_query = parseQuery(tmp_pos, tmp_pos + query_to_send.size(), false /* allow_multi_statements */);
|
||||
|
||||
ast_2 = parseQuery(tmp_pos, tmp_pos + query_to_send.size(),
|
||||
false /* allow_multi_statements */);
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
@ -1419,27 +1429,30 @@ private:
|
||||
}
|
||||
}
|
||||
|
||||
if (parsed_formatted_query)
|
||||
if (ast_2)
|
||||
{
|
||||
const auto formatted_twice = parsed_formatted_query->formatForErrorMessage();
|
||||
|
||||
if (formatted_twice != query_to_send)
|
||||
const auto text_2 = ast_2->formatForErrorMessage();
|
||||
const auto * tmp_pos = text_2.c_str();
|
||||
const auto ast_3 = parseQuery(tmp_pos, tmp_pos + text_2.size(),
|
||||
false /* allow_multi_statements */);
|
||||
const auto text_3 = ast_3->formatForErrorMessage();
|
||||
if (text_3 != text_2)
|
||||
{
|
||||
fmt::print(stderr, "The query formatting is broken.\n");
|
||||
|
||||
printChangedSettings();
|
||||
|
||||
fmt::print(
|
||||
stderr,
|
||||
fmt::print(stderr,
|
||||
"Got the following (different) text after formatting the fuzzed query and parsing it back:\n'{}'\n, expected:\n'{}'\n",
|
||||
formatted_twice,
|
||||
query_to_send);
|
||||
text_3, text_2);
|
||||
fmt::print(stderr, "In more detail:\n");
|
||||
fmt::print(stderr, "AST-1:\n'{}'\n", parsed_query->dumpTree());
|
||||
fmt::print(stderr, "AST-1 (generated by fuzzer):\n'{}'\n", parsed_query->dumpTree());
|
||||
fmt::print(stderr, "Text-1 (AST-1 formatted):\n'{}'\n", query_to_send);
|
||||
fmt::print(stderr, "AST-2 (Text-1 parsed):\n'{}'\n", parsed_formatted_query->dumpTree());
|
||||
fmt::print(stderr, "Text-2 (AST-2 formatted):\n'{}'\n", formatted_twice);
|
||||
fmt::print(stderr, "Text-1 must be equal to Text-2, but it is not.\n");
|
||||
fmt::print(stderr, "AST-2 (Text-1 parsed):\n'{}'\n", ast_2->dumpTree());
|
||||
fmt::print(stderr, "Text-2 (AST-2 formatted):\n'{}'\n", text_2);
|
||||
fmt::print(stderr, "AST-3 (Text-2 parsed):\n'{}'\n", ast_3->dumpTree());
|
||||
fmt::print(stderr, "Text-3 (AST-3 formatted):\n'{}'\n", text_3);
|
||||
fmt::print(stderr, "Text-3 must be equal to Text-2, but it is not.\n");
|
||||
|
||||
exit(1);
|
||||
}
|
||||
@ -1456,6 +1469,11 @@ private:
|
||||
server_exception.reset();
|
||||
client_exception.reset();
|
||||
have_error = false;
|
||||
|
||||
// We have to reinitialize connection after errors, because it
|
||||
// might have gotten into a wrong state and we'll get false
|
||||
// positives about "Unknown packet from server".
|
||||
connection->forceConnected(connection_parameters.timeouts);
|
||||
}
|
||||
else if (ast_to_process->formatForErrorMessage().size() > 500)
|
||||
{
|
||||
|
@ -4,11 +4,14 @@
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include <pcg-random/pcg_random.hpp>
|
||||
|
||||
#include <Common/randomSeed.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <Core/Field.h>
|
||||
#include <Parsers/IAST.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
|
@ -774,7 +774,7 @@ UInt128 diffHash(const CommitDiff & file_changes)
|
||||
}
|
||||
|
||||
UInt128 hash_of_diff;
|
||||
hasher.get128(hash_of_diff.low, hash_of_diff.high);
|
||||
hasher.get128(hash_of_diff.items[0], hash_of_diff.items[1]);
|
||||
|
||||
return hash_of_diff;
|
||||
}
|
||||
|
@ -23,6 +23,8 @@ public:
|
||||
|
||||
SharedLibraryHandler(const SharedLibraryHandler & other);
|
||||
|
||||
SharedLibraryHandler & operator=(const SharedLibraryHandler & other) = delete;
|
||||
|
||||
~SharedLibraryHandler();
|
||||
|
||||
BlockInputStreamPtr loadAll();
|
||||
|
@ -365,16 +365,20 @@ static void transformFixedString(const UInt8 * src, UInt8 * dst, size_t size, UI
|
||||
}
|
||||
}
|
||||
|
||||
static void transformUUID(const UInt128 & src, UInt128 & dst, UInt64 seed)
|
||||
static void transformUUID(const UUID & src_uuid, UUID & dst_uuid, UInt64 seed)
|
||||
{
|
||||
const UInt128 & src = src_uuid.toUnderType();
|
||||
UInt128 & dst = dst_uuid.toUnderType();
|
||||
|
||||
SipHash hash;
|
||||
hash.update(seed);
|
||||
hash.update(reinterpret_cast<const char *>(&src), sizeof(UInt128));
|
||||
hash.update(reinterpret_cast<const char *>(&src), sizeof(UUID));
|
||||
|
||||
/// Saving version and variant from an old UUID
|
||||
hash.get128(reinterpret_cast<char *>(&dst));
|
||||
dst.high = (dst.high & 0x1fffffffffffffffull) | (src.high & 0xe000000000000000ull);
|
||||
dst.low = (dst.low & 0xffffffffffff0fffull) | (src.low & 0x000000000000f000ull);
|
||||
|
||||
dst.items[1] = (dst.items[1] & 0x1fffffffffffffffull) | (src.items[1] & 0xe000000000000000ull);
|
||||
dst.items[0] = (dst.items[0] & 0xffffffffffff0fffull) | (src.items[0] & 0x000000000000f000ull);
|
||||
}
|
||||
|
||||
class FixedStringModel : public IModel
|
||||
@ -426,10 +430,10 @@ public:
|
||||
|
||||
ColumnPtr generate(const IColumn & column) override
|
||||
{
|
||||
const ColumnUInt128 & src_column = assert_cast<const ColumnUInt128 &>(column);
|
||||
const ColumnUUID & src_column = assert_cast<const ColumnUUID &>(column);
|
||||
const auto & src_data = src_column.getData();
|
||||
|
||||
auto res_column = ColumnUInt128::create();
|
||||
auto res_column = ColumnUUID::create();
|
||||
auto & res_data = res_column->getData();
|
||||
|
||||
res_data.resize(src_data.size());
|
||||
|
@ -109,7 +109,7 @@ void ODBCColumnsInfoHandler::handleRequest(HTTPServerRequest & request, HTTPServ
|
||||
validateODBCConnectionString(connection_string),
|
||||
getContext()->getSettingsRef().odbc_bridge_connection_pool_size);
|
||||
|
||||
nanodbc::catalog catalog(*connection);
|
||||
nanodbc::catalog catalog(connection->get());
|
||||
std::string catalog_name;
|
||||
|
||||
/// In XDBC tables it is allowed to pass either database_name or schema_name in table definion, but not both of them.
|
||||
|
@ -46,7 +46,7 @@ void IdentifierQuoteHandler::handleRequest(HTTPServerRequest & request, HTTPServ
|
||||
validateODBCConnectionString(connection_string),
|
||||
getContext()->getSettingsRef().odbc_bridge_connection_pool_size);
|
||||
|
||||
auto identifier = getIdentifierQuote(*connection);
|
||||
auto identifier = getIdentifierQuote(connection->get());
|
||||
|
||||
WriteBufferFromHTTPServerResponse out(response, request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD, keep_alive_timeout);
|
||||
try
|
||||
|
@ -18,13 +18,10 @@
|
||||
#include <Processors/Formats/InputStreamFromInputFormat.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <Server/HTTP/HTMLForm.h>
|
||||
#include "ODBCConnectionFactory.h"
|
||||
|
||||
#include <mutex>
|
||||
#include <memory>
|
||||
|
||||
#include <nanodbc/nanodbc.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -133,12 +130,12 @@ void ODBCHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse
|
||||
|
||||
auto quoting_style = IdentifierQuotingStyle::None;
|
||||
#if USE_ODBC
|
||||
quoting_style = getQuotingStyle(*connection);
|
||||
quoting_style = getQuotingStyle(connection->get());
|
||||
#endif
|
||||
auto & read_buf = request.getStream();
|
||||
auto input_format = FormatFactory::instance().getInput(format, read_buf, *sample_block, getContext(), max_block_size);
|
||||
auto input_stream = std::make_shared<InputStreamFromInputFormat>(input_format);
|
||||
ODBCBlockOutputStream output_stream(*connection, db_name, table_name, *sample_block, getContext(), quoting_style);
|
||||
ODBCBlockOutputStream output_stream(std::move(connection), db_name, table_name, *sample_block, getContext(), quoting_style);
|
||||
copyData(*input_stream, output_stream);
|
||||
writeStringBinary("Ok.", out);
|
||||
}
|
||||
@ -148,7 +145,7 @@ void ODBCHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse
|
||||
LOG_TRACE(log, "Query: {}", query);
|
||||
|
||||
BlockOutputStreamPtr writer = FormatFactory::instance().getOutputStreamParallelIfPossible(format, out, *sample_block, getContext());
|
||||
ODBCBlockInputStream inp(*connection, query, *sample_block, max_block_size);
|
||||
ODBCBlockInputStream inp(std::move(connection), query, *sample_block, max_block_size);
|
||||
copyData(inp, *writer);
|
||||
}
|
||||
}
|
||||
|
@ -21,14 +21,13 @@ namespace ErrorCodes
|
||||
|
||||
|
||||
ODBCBlockInputStream::ODBCBlockInputStream(
|
||||
nanodbc::connection & connection_, const std::string & query_str, const Block & sample_block, const UInt64 max_block_size_)
|
||||
nanodbc::ConnectionHolderPtr connection, const std::string & query_str, const Block & sample_block, const UInt64 max_block_size_)
|
||||
: log(&Poco::Logger::get("ODBCBlockInputStream"))
|
||||
, max_block_size{max_block_size_}
|
||||
, connection(connection_)
|
||||
, query(query_str)
|
||||
{
|
||||
description.init(sample_block);
|
||||
result = execute(connection, NANODBC_TEXT(query));
|
||||
result = execute(connection->get(), NANODBC_TEXT(query));
|
||||
}
|
||||
|
||||
|
||||
|
@ -4,7 +4,7 @@
|
||||
#include <Core/Block.h>
|
||||
#include <DataStreams/IBlockInputStream.h>
|
||||
#include <Core/ExternalResultDescription.h>
|
||||
#include <nanodbc/nanodbc.h>
|
||||
#include "ODBCConnectionFactory.h"
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -13,7 +13,7 @@ namespace DB
|
||||
class ODBCBlockInputStream final : public IBlockInputStream
|
||||
{
|
||||
public:
|
||||
ODBCBlockInputStream(nanodbc::connection & connection_, const std::string & query_str, const Block & sample_block, const UInt64 max_block_size_);
|
||||
ODBCBlockInputStream(nanodbc::ConnectionHolderPtr connection, const std::string & query_str, const Block & sample_block, const UInt64 max_block_size_);
|
||||
|
||||
String getName() const override { return "ODBC"; }
|
||||
|
||||
@ -36,7 +36,6 @@ private:
|
||||
const UInt64 max_block_size;
|
||||
ExternalResultDescription description;
|
||||
|
||||
nanodbc::connection & connection;
|
||||
nanodbc::result result;
|
||||
String query;
|
||||
bool finished = false;
|
||||
|
@ -1,5 +1,6 @@
|
||||
#include "ODBCBlockOutputStream.h"
|
||||
|
||||
#include <Common/hex.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <Core/Field.h>
|
||||
#include <common/LocalDate.h>
|
||||
@ -37,17 +38,16 @@ namespace
|
||||
query.IAST::format(settings);
|
||||
return buf.str();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ODBCBlockOutputStream::ODBCBlockOutputStream(nanodbc::connection & connection_,
|
||||
ODBCBlockOutputStream::ODBCBlockOutputStream(nanodbc::ConnectionHolderPtr connection_,
|
||||
const std::string & remote_database_name_,
|
||||
const std::string & remote_table_name_,
|
||||
const Block & sample_block_,
|
||||
ContextPtr local_context_,
|
||||
IdentifierQuotingStyle quoting_)
|
||||
: log(&Poco::Logger::get("ODBCBlockOutputStream"))
|
||||
, connection(connection_)
|
||||
, connection(std::move(connection_))
|
||||
, db_name(remote_database_name_)
|
||||
, table_name(remote_table_name_)
|
||||
, sample_block(sample_block_)
|
||||
@ -69,7 +69,7 @@ void ODBCBlockOutputStream::write(const Block & block)
|
||||
writer->write(block);
|
||||
|
||||
std::string query = getInsertQuery(db_name, table_name, block.getColumnsWithTypeAndName(), quoting) + values_buf.str();
|
||||
execute(connection, query);
|
||||
execute(connection->get(), query);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -5,7 +5,7 @@
|
||||
#include <Core/ExternalResultDescription.h>
|
||||
#include <Parsers/IdentifierQuotingStyle.h>
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
#include <nanodbc/nanodbc.h>
|
||||
#include "ODBCConnectionFactory.h"
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -16,7 +16,7 @@ class ODBCBlockOutputStream : public IBlockOutputStream
|
||||
|
||||
public:
|
||||
ODBCBlockOutputStream(
|
||||
nanodbc::connection & connection_,
|
||||
nanodbc::ConnectionHolderPtr connection_,
|
||||
const std::string & remote_database_name_,
|
||||
const std::string & remote_table_name_,
|
||||
const Block & sample_block_,
|
||||
@ -29,7 +29,7 @@ public:
|
||||
private:
|
||||
Poco::Logger * log;
|
||||
|
||||
nanodbc::connection & connection;
|
||||
nanodbc::ConnectionHolderPtr connection;
|
||||
std::string db_name;
|
||||
std::string table_name;
|
||||
Block sample_block;
|
||||
|
@ -6,53 +6,51 @@
|
||||
#include <common/BorrowedObjectPool.h>
|
||||
#include <unordered_map>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NO_FREE_CONNECTION;
|
||||
}
|
||||
}
|
||||
|
||||
namespace nanodbc
|
||||
{
|
||||
|
||||
static constexpr inline auto ODBC_CONNECT_TIMEOUT = 100;
|
||||
|
||||
using ConnectionPtr = std::shared_ptr<nanodbc::connection>;
|
||||
using ConnectionPtr = std::unique_ptr<nanodbc::connection>;
|
||||
using Pool = BorrowedObjectPool<ConnectionPtr>;
|
||||
using PoolPtr = std::shared_ptr<Pool>;
|
||||
|
||||
class ConnectionHolder
|
||||
{
|
||||
|
||||
public:
|
||||
ConnectionHolder(const std::string & connection_string_, PoolPtr pool_) : connection_string(connection_string_), pool(pool_) {}
|
||||
ConnectionHolder(PoolPtr pool_, ConnectionPtr connection_) : pool(pool_), connection(std::move(connection_)) {}
|
||||
|
||||
~ConnectionHolder()
|
||||
ConnectionHolder(const ConnectionHolder & other) = delete;
|
||||
|
||||
~ConnectionHolder() { pool->returnObject(std::move(connection)); }
|
||||
|
||||
nanodbc::connection & get() const
|
||||
{
|
||||
if (connection)
|
||||
pool->returnObject(std::move(connection));
|
||||
}
|
||||
|
||||
nanodbc::connection & operator*()
|
||||
{
|
||||
if (!connection)
|
||||
{
|
||||
pool->borrowObject(connection, [&]()
|
||||
{
|
||||
return std::make_shared<nanodbc::connection>(connection_string, ODBC_CONNECT_TIMEOUT);
|
||||
});
|
||||
}
|
||||
|
||||
assert(connection != nullptr);
|
||||
return *connection;
|
||||
}
|
||||
|
||||
private:
|
||||
std::string connection_string;
|
||||
PoolPtr pool;
|
||||
ConnectionPtr connection;
|
||||
};
|
||||
|
||||
using ConnectionHolderPtr = std::unique_ptr<ConnectionHolder>;
|
||||
}
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
static constexpr inline auto ODBC_CONNECT_TIMEOUT = 100;
|
||||
static constexpr inline auto ODBC_POOL_WAIT_TIMEOUT = 10000;
|
||||
|
||||
class ODBCConnectionFactory final : private boost::noncopyable
|
||||
{
|
||||
public:
|
||||
@ -62,14 +60,32 @@ public:
|
||||
return ret;
|
||||
}
|
||||
|
||||
nanodbc::ConnectionHolder get(const std::string & connection_string, size_t pool_size)
|
||||
nanodbc::ConnectionHolderPtr get(const std::string & connection_string, size_t pool_size)
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
|
||||
if (!factory.count(connection_string))
|
||||
factory.emplace(std::make_pair(connection_string, std::make_shared<nanodbc::Pool>(pool_size)));
|
||||
|
||||
return nanodbc::ConnectionHolder(connection_string, factory[connection_string]);
|
||||
auto & pool = factory[connection_string];
|
||||
|
||||
nanodbc::ConnectionPtr connection;
|
||||
auto connection_available = pool->tryBorrowObject(connection, []() { return nullptr; }, ODBC_POOL_WAIT_TIMEOUT);
|
||||
|
||||
if (!connection_available)
|
||||
throw Exception("Unable to fetch connection within the timeout", ErrorCodes::NO_FREE_CONNECTION);
|
||||
|
||||
try
|
||||
{
|
||||
if (!connection || !connection->connected())
|
||||
connection = std::make_unique<nanodbc::connection>(connection_string, ODBC_CONNECT_TIMEOUT);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
pool->returnObject(std::move(connection));
|
||||
}
|
||||
|
||||
return std::make_unique<nanodbc::ConnectionHolder>(factory[connection_string], std::move(connection));
|
||||
}
|
||||
|
||||
private:
|
||||
|
@ -53,7 +53,7 @@ void SchemaAllowedHandler::handleRequest(HTTPServerRequest & request, HTTPServer
|
||||
validateODBCConnectionString(connection_string),
|
||||
getContext()->getSettingsRef().odbc_bridge_connection_pool_size);
|
||||
|
||||
bool result = isSchemaAllowed(*connection);
|
||||
bool result = isSchemaAllowed(connection->get());
|
||||
|
||||
WriteBufferFromHTTPServerResponse out(response, request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD, keep_alive_timeout);
|
||||
try
|
||||
|
@ -3,7 +3,6 @@
|
||||
#if USE_ODBC
|
||||
|
||||
#include <common/logger_useful.h>
|
||||
#include <nanodbc/nanodbc.h>
|
||||
#include <sql.h>
|
||||
#include <sqlext.h>
|
||||
|
||||
|
@ -13,6 +13,7 @@
|
||||
#include <Poco/Net/HTTPServer.h>
|
||||
#include <Poco/Net/NetException.h>
|
||||
#include <Poco/Util/HelpFormatter.h>
|
||||
#include <Poco/Environment.h>
|
||||
#include <ext/scope_guard.h>
|
||||
#include <common/defines.h>
|
||||
#include <common/logger_useful.h>
|
||||
@ -385,6 +386,11 @@ void Server::initialize(Poco::Util::Application & self)
|
||||
{
|
||||
BaseDaemon::initialize(self);
|
||||
logger().information("starting up");
|
||||
|
||||
LOG_INFO(&logger(), "OS Name = {}, OS Version = {}, OS Architecture = {}",
|
||||
Poco::Environment::osName(),
|
||||
Poco::Environment::osVersion(),
|
||||
Poco::Environment::osArchitecture());
|
||||
}
|
||||
|
||||
std::string Server::getDefaultCorePath() const
|
||||
@ -879,7 +885,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
global_context->setMMappedFileCache(mmap_cache_size);
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
size_t compiled_expression_cache_size = config().getUInt64("compiled_expression_cache_size", 500);
|
||||
constexpr size_t compiled_expression_cache_size_default = 1024 * 1024 * 1024;
|
||||
size_t compiled_expression_cache_size = config().getUInt64("compiled_expression_cache_size", compiled_expression_cache_size_default);
|
||||
CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_size);
|
||||
#endif
|
||||
|
||||
|
@ -329,6 +329,8 @@
|
||||
-->
|
||||
<mmap_cache_size>1000</mmap_cache_size>
|
||||
|
||||
<!-- Cache size for compiled expressions.-->
|
||||
<compiled_expression_cache_size>1073741824</compiled_expression_cache_size>
|
||||
|
||||
<!-- Path to data directory, with trailing slash. -->
|
||||
<path>/var/lib/clickhouse/</path>
|
||||
@ -518,6 +520,33 @@
|
||||
<!-- Reallocate memory for machine code ("text") using huge pages. Highly experimental. -->
|
||||
<remap_executable>false</remap_executable>
|
||||
|
||||
<![CDATA[
|
||||
Uncomment below in order to use JDBC table engine and function.
|
||||
|
||||
To install and run JDBC bridge in background:
|
||||
* [Debian/Ubuntu]
|
||||
export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge
|
||||
export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '<release>' | sed -e 's|.*>\(.*\)<.*|\1|')
|
||||
wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb
|
||||
apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb
|
||||
clickhouse-jdbc-bridge &
|
||||
|
||||
* [CentOS/RHEL]
|
||||
export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge
|
||||
export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '<release>' | sed -e 's|.*>\(.*\)<.*|\1|')
|
||||
wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm
|
||||
yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm
|
||||
clickhouse-jdbc-bridge &
|
||||
|
||||
Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information.
|
||||
]]>
|
||||
<!--
|
||||
<jdbc_bridge>
|
||||
<host>127.0.0.1</host>
|
||||
<port>9019</port>
|
||||
</jdbc_bridge>
|
||||
-->
|
||||
|
||||
<!-- Configuration of clusters that could be used in Distributed tables.
|
||||
https://clickhouse.tech/docs/en/operations/table_engines/distributed/
|
||||
-->
|
||||
|
@ -355,8 +355,9 @@ String DiskAccessStorage::getStorageParamsJSON() const
|
||||
std::lock_guard lock{mutex};
|
||||
Poco::JSON::Object json;
|
||||
json.set("path", directory_path);
|
||||
if (readonly)
|
||||
json.set("readonly", readonly.load());
|
||||
bool readonly_loaded = readonly;
|
||||
if (readonly_loaded)
|
||||
json.set("readonly", Poco::Dynamic::Var{true});
|
||||
std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
|
||||
oss.exceptions(std::ios::failbit);
|
||||
Poco::JSON::Stringifier::stringify(json, oss);
|
||||
|
@ -77,7 +77,7 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str
|
||||
if (enable_tls_lc_str == "starttls")
|
||||
params.enable_tls = LDAPClient::Params::TLSEnable::YES_STARTTLS;
|
||||
else if (config.getBool(ldap_server_config + ".enable_tls"))
|
||||
params.enable_tls = LDAPClient::Params::TLSEnable::YES;
|
||||
params.enable_tls = LDAPClient::Params::TLSEnable::YES; //-V1048
|
||||
else
|
||||
params.enable_tls = LDAPClient::Params::TLSEnable::NO;
|
||||
}
|
||||
@ -96,7 +96,7 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str
|
||||
else if (tls_minimum_protocol_version_lc_str == "tls1.1")
|
||||
params.tls_minimum_protocol_version = LDAPClient::Params::TLSProtocolVersion::TLS1_1;
|
||||
else if (tls_minimum_protocol_version_lc_str == "tls1.2")
|
||||
params.tls_minimum_protocol_version = LDAPClient::Params::TLSProtocolVersion::TLS1_2;
|
||||
params.tls_minimum_protocol_version = LDAPClient::Params::TLSProtocolVersion::TLS1_2; //-V1048
|
||||
else
|
||||
throw Exception("Bad value for 'tls_minimum_protocol_version' entry, allowed values are: 'ssl2', 'ssl3', 'tls1.0', 'tls1.1', 'tls1.2'", ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
@ -113,7 +113,7 @@ auto parseLDAPServer(const Poco::Util::AbstractConfiguration & config, const Str
|
||||
else if (tls_require_cert_lc_str == "try")
|
||||
params.tls_require_cert = LDAPClient::Params::TLSRequireCert::TRY;
|
||||
else if (tls_require_cert_lc_str == "demand")
|
||||
params.tls_require_cert = LDAPClient::Params::TLSRequireCert::DEMAND;
|
||||
params.tls_require_cert = LDAPClient::Params::TLSRequireCert::DEMAND; //-V1048
|
||||
else
|
||||
throw Exception("Bad value for 'tls_require_cert' entry, allowed values are: 'never', 'allow', 'try', 'demand'", ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
|
@ -136,7 +136,7 @@ GrantedRoles::Elements GrantedRoles::getElements() const
|
||||
boost::range::set_difference(roles, roles_with_admin_option, std::back_inserter(element.ids));
|
||||
if (!element.empty())
|
||||
{
|
||||
element.admin_option = false;
|
||||
element.admin_option = false; //-V1048
|
||||
elements.emplace_back(std::move(element));
|
||||
}
|
||||
|
||||
|
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Access/IAccessEntity.h>
|
||||
#include <common/types.h>
|
||||
#include <Core/Types.h>
|
||||
#include <Core/UUID.h>
|
||||
#include <ext/scope_guard.h>
|
||||
#include <functional>
|
||||
|
@ -45,7 +45,7 @@ struct Quota : public IAccessEntity
|
||||
|
||||
struct ResourceTypeInfo
|
||||
{
|
||||
const char * const raw_name;
|
||||
const char * const raw_name = "";
|
||||
const String name; /// Lowercased with underscores, e.g. "result_rows".
|
||||
const String keyword; /// Uppercased with spaces, e.g. "RESULT ROWS".
|
||||
const bool output_as_float = false;
|
||||
|
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Core/UUID.h>
|
||||
#include <Core/Types.h>
|
||||
#include <boost/container/flat_set.hpp>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
|
@ -2,6 +2,7 @@
|
||||
|
||||
#include <Access/IAccessEntity.h>
|
||||
#include <Access/RolesOrUsersSet.h>
|
||||
#include <Core/Types.h>
|
||||
#include <array>
|
||||
|
||||
|
||||
|
@ -7,7 +7,7 @@
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
#include "Core/DecimalFunctions.h"
|
||||
#include <Core/DecimalFunctions.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
|
@ -5,18 +5,18 @@
|
||||
|
||||
namespace DB
|
||||
{
|
||||
template <class T>
|
||||
template <typename T>
|
||||
using AvgWeightedFieldType = std::conditional_t<IsDecimalNumber<T>,
|
||||
std::conditional_t<std::is_same_v<T, Decimal256>, Decimal256, Decimal128>,
|
||||
std::conditional_t<DecimalOrExtendedInt<T>,
|
||||
Float64, // no way to do UInt128 * UInt128, better cast to Float64
|
||||
NearestFieldType<T>>>;
|
||||
|
||||
template <class T, class U>
|
||||
template <typename T, typename U>
|
||||
using MaxFieldType = std::conditional_t<(sizeof(AvgWeightedFieldType<T>) > sizeof(AvgWeightedFieldType<U>)),
|
||||
AvgWeightedFieldType<T>, AvgWeightedFieldType<U>>;
|
||||
|
||||
template <class Value, class Weight>
|
||||
template <typename Value, typename Weight>
|
||||
class AggregateFunctionAvgWeighted final :
|
||||
public AggregateFunctionAvgBase<
|
||||
MaxFieldType<Value, Weight>, AvgWeightedFieldType<Weight>, AggregateFunctionAvgWeighted<Value, Weight>>
|
||||
|
@ -2,7 +2,6 @@
|
||||
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <AggregateFunctions/Helpers.h>
|
||||
@ -20,7 +19,7 @@ namespace ErrorCodes
|
||||
|
||||
/** Tracks the leftmost and rightmost (x, y) data points.
|
||||
*/
|
||||
struct AggregateFunctionBoundingRatioData
|
||||
struct AggregateFunctionBoundingRatioData //-V730
|
||||
{
|
||||
struct Point
|
||||
{
|
||||
|
@ -10,7 +10,6 @@
|
||||
#include <Common/HashTable/HashMap.h>
|
||||
#include <Common/SipHash.h>
|
||||
|
||||
#include <Common/FieldVisitors.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
@ -296,12 +296,7 @@ public:
|
||||
if (size)
|
||||
{
|
||||
typename ColumnVector<T>::Container & data_to = assert_cast<ColumnVector<T> &>(arr_to.getData()).getData();
|
||||
if constexpr (is_big_int_v<T>)
|
||||
// is data_to empty? we should probably use std::vector::insert then
|
||||
for (auto it = this->data(place).value.begin(); it != this->data(place).value.end(); it++)
|
||||
data_to.push_back(*it);
|
||||
else
|
||||
data_to.insert(this->data(place).value.begin(), this->data(place).value.end());
|
||||
data_to.insert(this->data(place).value.begin(), this->data(place).value.end());
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -21,6 +21,7 @@ namespace ErrorCodes
|
||||
namespace
|
||||
{
|
||||
|
||||
/// TODO Proper support for Decimal256.
|
||||
template <typename T, typename LimitNumberOfElements>
|
||||
struct MovingSum
|
||||
{
|
||||
|
@ -38,7 +38,7 @@ struct MovingData
|
||||
using Array = PODArray<T, 32, Allocator>;
|
||||
|
||||
Array value; /// Prefix sums.
|
||||
T sum = 0;
|
||||
T sum{};
|
||||
|
||||
void NO_SANITIZE_UNDEFINED add(T val, Arena * arena)
|
||||
{
|
||||
@ -69,9 +69,9 @@ struct MovingAvgData : public MovingData<T>
|
||||
T NO_SANITIZE_UNDEFINED get(size_t idx, UInt64 window_size) const
|
||||
{
|
||||
if (idx < window_size)
|
||||
return this->value[idx] / window_size;
|
||||
return this->value[idx] / T(window_size);
|
||||
else
|
||||
return (this->value[idx] - this->value[idx - window_size]) / window_size;
|
||||
return (this->value[idx] - this->value[idx - window_size]) / T(window_size);
|
||||
}
|
||||
};
|
||||
|
||||
|
@ -220,7 +220,7 @@ private:
|
||||
}
|
||||
|
||||
public:
|
||||
AggregateFunctionHistogramData()
|
||||
AggregateFunctionHistogramData() //-V730
|
||||
: size(0)
|
||||
, lower_bound(std::numeric_limits<Mean>::max())
|
||||
, upper_bound(std::numeric_limits<Mean>::lowest())
|
||||
|
@ -6,7 +6,6 @@
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <Common/PODArray_fwd.h>
|
||||
#include <common/types.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
@ -21,7 +20,7 @@
|
||||
|
||||
#include <Common/ArenaAllocator.h>
|
||||
|
||||
#include <iostream>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
|
@ -181,7 +181,7 @@ public:
|
||||
/** For strings. Short strings are stored in the object itself, and long strings are allocated separately.
|
||||
* NOTE It could also be suitable for arrays of numbers.
|
||||
*/
|
||||
struct SingleValueDataString
|
||||
struct SingleValueDataString //-V730
|
||||
{
|
||||
private:
|
||||
using Self = SingleValueDataString;
|
||||
|
@ -100,13 +100,14 @@ AggregateFunctionPtr createAggregateFunctionQuantile(const std::string & name, c
|
||||
if (which.idx == TypeIndex::Decimal32) return std::make_shared<Function<Decimal32, false>>(argument_types, params);
|
||||
if (which.idx == TypeIndex::Decimal64) return std::make_shared<Function<Decimal64, false>>(argument_types, params);
|
||||
if (which.idx == TypeIndex::Decimal128) return std::make_shared<Function<Decimal128, false>>(argument_types, params);
|
||||
if (which.idx == TypeIndex::Decimal256) return std::make_shared<Function<Decimal256, false>>(argument_types, params);
|
||||
if (which.idx == TypeIndex::DateTime64) return std::make_shared<Function<DateTime64, false>>(argument_types, params);
|
||||
//if (which.idx == TypeIndex::Decimal256) return std::make_shared<Function<Decimal256, false>>(argument_types, params);
|
||||
}
|
||||
|
||||
if constexpr (supportBigInt<Function>())
|
||||
{
|
||||
if (which.idx == TypeIndex::Int128) return std::make_shared<Function<Int128, true>>(argument_types, params);
|
||||
if (which.idx == TypeIndex::UInt128) return std::make_shared<Function<Int128, true>>(argument_types, params);
|
||||
if (which.idx == TypeIndex::Int256) return std::make_shared<Function<Int256, true>>(argument_types, params);
|
||||
if (which.idx == TypeIndex::UInt256) return std::make_shared<Function<UInt256, true>>(argument_types, params);
|
||||
}
|
||||
|
@ -36,7 +36,7 @@ AggregateFunctionPtr createAggregateFunctionStatisticsBinary(const std::string &
|
||||
assertNoParameters(name, parameters);
|
||||
assertBinary(name, argument_types);
|
||||
|
||||
AggregateFunctionPtr res(createWithTwoNumericTypes<FunctionTemplate>(*argument_types[0], *argument_types[1], argument_types));
|
||||
AggregateFunctionPtr res(createWithTwoBasicNumericTypes<FunctionTemplate>(*argument_types[0], *argument_types[1], argument_types));
|
||||
if (!res)
|
||||
throw Exception("Illegal types " + argument_types[0]->getName() + " and " + argument_types[1]->getName()
|
||||
+ " of arguments for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
@ -13,7 +13,7 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace
|
||||
namespace detail
|
||||
{
|
||||
|
||||
/// This function returns true if both values are large and comparable.
|
||||
@ -72,7 +72,7 @@ public:
|
||||
Float64 factor = static_cast<Float64>(count * source.count) / total_count;
|
||||
Float64 delta = mean - source.mean;
|
||||
|
||||
if (areComparable(count, source.count))
|
||||
if (detail::areComparable(count, source.count))
|
||||
mean = (source.count * source.mean + count * mean) / total_count;
|
||||
else
|
||||
mean = source.mean + delta * (static_cast<Float64>(count) / total_count);
|
||||
@ -302,7 +302,7 @@ public:
|
||||
Float64 left_delta = left_mean - source.left_mean;
|
||||
Float64 right_delta = right_mean - source.right_mean;
|
||||
|
||||
if (areComparable(count, source.count))
|
||||
if (detail::areComparable(count, source.count))
|
||||
{
|
||||
left_mean = (source.count * source.left_mean + count * left_mean) / total_count;
|
||||
right_mean = (source.count * source.right_mean + count * right_mean) / total_count;
|
||||
|
@ -41,7 +41,7 @@ AggregateFunctionPtr createAggregateFunctionStatisticsBinary(const std::string &
|
||||
assertNoParameters(name, parameters);
|
||||
assertBinary(name, argument_types);
|
||||
|
||||
AggregateFunctionPtr res(createWithTwoNumericTypes<FunctionTemplate>(*argument_types[0], *argument_types[1], argument_types));
|
||||
AggregateFunctionPtr res(createWithTwoBasicNumericTypes<FunctionTemplate>(*argument_types[0], *argument_types[1], argument_types));
|
||||
if (!res)
|
||||
throw Exception("Illegal types " + argument_types[0]->getName() + " and " + argument_types[1]->getName()
|
||||
+ " of arguments for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
|
@ -131,11 +131,10 @@ public:
|
||||
static_cast<ResultType>(static_cast<const ColVecT2 &>(*columns[1]).getData()[row_num]));
|
||||
else
|
||||
{
|
||||
if constexpr (std::is_same_v<T1, Decimal256>)
|
||||
if constexpr (IsDecimalNumber<T1>)
|
||||
{
|
||||
this->data(place).add(static_cast<ResultType>(
|
||||
static_cast<const ColVecT1 &>(*columns[0]).getData()[row_num].value
|
||||
));
|
||||
static_cast<const ColVecT1 &>(*columns[0]).getData()[row_num].value));
|
||||
}
|
||||
else
|
||||
this->data(place).add(
|
||||
|
@ -2,9 +2,10 @@
|
||||
#include <AggregateFunctions/AggregateFunctionTopK.h>
|
||||
#include <AggregateFunctions/Helpers.h>
|
||||
#include <AggregateFunctions/FactoryHelpers.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include "registerAggregateFunctions.h"
|
||||
|
||||
|
||||
#define TOP_K_MAX_SIZE 0xFFFFFF
|
||||
|
||||
|
@ -10,7 +10,6 @@
|
||||
#include <Columns/ColumnArray.h>
|
||||
|
||||
#include <Common/SpaceSaving.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <Common/assert_cast.h>
|
||||
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
|
@ -132,6 +132,12 @@ void registerAggregateFunctionsUniq(AggregateFunctionFactory & factory)
|
||||
|
||||
factory.registerFunction("uniqExact",
|
||||
{createAggregateFunctionUniq<true, AggregateFunctionUniqExactData, AggregateFunctionUniqExactData<String>>, properties});
|
||||
|
||||
#if USE_DATASKETCHES
|
||||
factory.registerFunction("uniqThetaSketch",
|
||||
{createAggregateFunctionUniq<AggregateFunctionUniqThetaSketchData, AggregateFunctionUniqThetaSketchData>, properties});
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -22,6 +22,7 @@
|
||||
|
||||
#include <AggregateFunctions/UniquesHashSet.h>
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
#include <AggregateFunctions/ThetaSketchData.h>
|
||||
#include <AggregateFunctions/UniqVariadicHash.h>
|
||||
|
||||
|
||||
@ -69,7 +70,7 @@ struct AggregateFunctionUniqHLL12Data<String>
|
||||
};
|
||||
|
||||
template <>
|
||||
struct AggregateFunctionUniqHLL12Data<UInt128>
|
||||
struct AggregateFunctionUniqHLL12Data<UUID>
|
||||
{
|
||||
using Set = HyperLogLogWithSmallSetOptimization<UInt64, 16, 12>;
|
||||
Set set;
|
||||
@ -124,6 +125,19 @@ struct AggregateFunctionUniqExactData<String>
|
||||
};
|
||||
|
||||
|
||||
/// uniqThetaSketch
|
||||
#if USE_DATASKETCHES
|
||||
|
||||
struct AggregateFunctionUniqThetaSketchData
|
||||
{
|
||||
using Set = ThetaSketchData<UInt64>;
|
||||
Set set;
|
||||
|
||||
static String getName() { return "uniqThetaSketch"; }
|
||||
};
|
||||
|
||||
#endif
|
||||
|
||||
namespace detail
|
||||
{
|
||||
|
||||
@ -133,16 +147,14 @@ template <typename T> struct AggregateFunctionUniqTraits
|
||||
{
|
||||
static UInt64 hash(T x)
|
||||
{
|
||||
if constexpr (std::is_same_v<T, UInt128>)
|
||||
{
|
||||
return sipHash64(x);
|
||||
}
|
||||
else if constexpr (std::is_same_v<T, Float32> || std::is_same_v<T, Float64>)
|
||||
if constexpr (std::is_same_v<T, Float32> || std::is_same_v<T, Float64>)
|
||||
{
|
||||
return ext::bit_cast<UInt64>(x);
|
||||
}
|
||||
else if constexpr (sizeof(T) <= sizeof(UInt64))
|
||||
{
|
||||
return x;
|
||||
}
|
||||
else
|
||||
return DefaultHash64<T>(x);
|
||||
}
|
||||
@ -184,11 +196,17 @@ struct OneAdder
|
||||
UInt128 key;
|
||||
SipHash hash;
|
||||
hash.update(value.data, value.size);
|
||||
hash.get128(key.low, key.high);
|
||||
hash.get128(key);
|
||||
|
||||
data.set.insert(key);
|
||||
}
|
||||
}
|
||||
#if USE_DATASKETCHES
|
||||
else if constexpr (std::is_same_v<Data, AggregateFunctionUniqThetaSketchData>)
|
||||
{
|
||||
data.set.insertOriginal(column.getDataAt(row_num));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
};
|
||||
|
||||
|
@ -3,11 +3,13 @@
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
#include <AggregateFunctions/Helpers.h>
|
||||
|
||||
#include <Common/FieldVisitors.h>
|
||||
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
|
||||
#include <functional>
|
||||
#include "registerAggregateFunctions.h"
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
@ -1,7 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/CombinedCardinalityEstimator.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <Common/SipHash.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/assert_cast.h>
|
||||
|
@ -1,11 +1,11 @@
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
#include <AggregateFunctions/Helpers.h>
|
||||
#include <AggregateFunctions/AggregateFunctionUniqUpTo.h>
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <DataTypes/DataTypeFixedString.h>
|
||||
#include "registerAggregateFunctions.h"
|
||||
|
||||
|
||||
namespace DB
|
||||
|
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Common/FieldVisitors.h>
|
||||
#include <common/unaligned.h>
|
||||
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/assert_cast.h>
|
||||
|
||||
@ -35,7 +36,7 @@ namespace DB
|
||||
*/
|
||||
|
||||
template <typename T>
|
||||
struct __attribute__((__packed__)) AggregateFunctionUniqUpToData
|
||||
struct AggregateFunctionUniqUpToData
|
||||
{
|
||||
/** If count == threshold + 1 - this means that it is "overflowed" (values greater than threshold).
|
||||
* In this case (for example, after calling the merge function), the `data` array does not necessarily contain the initialized values
|
||||
@ -43,7 +44,17 @@ struct __attribute__((__packed__)) AggregateFunctionUniqUpToData
|
||||
* then set count to `threshold + 1`, and values from another state are not copied.
|
||||
*/
|
||||
UInt8 count = 0;
|
||||
T data[0];
|
||||
char data_ptr[0];
|
||||
|
||||
T load(size_t i) const
|
||||
{
|
||||
return unalignedLoad<T>(data_ptr + i * sizeof(T));
|
||||
}
|
||||
|
||||
void store(size_t i, const T & x)
|
||||
{
|
||||
unalignedStore<T>(data_ptr + i * sizeof(T), x);
|
||||
}
|
||||
|
||||
size_t size() const
|
||||
{
|
||||
@ -60,12 +71,12 @@ struct __attribute__((__packed__)) AggregateFunctionUniqUpToData
|
||||
|
||||
/// Linear search for the matching element.
|
||||
for (size_t i = 0; i < count; ++i)
|
||||
if (data[i] == x)
|
||||
if (load(i) == x)
|
||||
return;
|
||||
|
||||
/// Did not find the matching element. If there is room for one more element, insert it.
|
||||
if (count < threshold)
|
||||
data[count] = x;
|
||||
store(count, x);
|
||||
|
||||
/// After increasing count, the state may be overflowed.
|
||||
++count;
|
||||
@ -84,7 +95,7 @@ struct __attribute__((__packed__)) AggregateFunctionUniqUpToData
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < rhs.count; ++i)
|
||||
insert(rhs.data[i], threshold);
|
||||
insert(rhs.load(i), threshold);
|
||||
}
|
||||
|
||||
void write(WriteBuffer & wb, UInt8 threshold) const
|
||||
@ -93,7 +104,7 @@ struct __attribute__((__packed__)) AggregateFunctionUniqUpToData
|
||||
|
||||
/// Write values only if the state is not overflowed. Otherwise, they are not needed, and only the fact that the state is overflowed is important.
|
||||
if (count <= threshold)
|
||||
wb.write(reinterpret_cast<const char *>(data), count * sizeof(data[0]));
|
||||
wb.write(data_ptr, count * sizeof(T));
|
||||
}
|
||||
|
||||
void read(ReadBuffer & rb, UInt8 threshold)
|
||||
@ -101,7 +112,7 @@ struct __attribute__((__packed__)) AggregateFunctionUniqUpToData
|
||||
readBinary(count, rb);
|
||||
|
||||
if (count <= threshold)
|
||||
rb.read(reinterpret_cast<char *>(data), count * sizeof(data[0]));
|
||||
rb.read(data_ptr, count * sizeof(T));
|
||||
}
|
||||
|
||||
/// ALWAYS_INLINE is required to have better code layout for uniqUpTo function
|
||||
|
@ -15,12 +15,12 @@
|
||||
M(Float32) \
|
||||
M(Float64)
|
||||
|
||||
// No UInt128 here because of the name conflict
|
||||
#define FOR_NUMERIC_TYPES(M) \
|
||||
M(UInt8) \
|
||||
M(UInt16) \
|
||||
M(UInt32) \
|
||||
M(UInt64) \
|
||||
M(UInt128) \
|
||||
M(UInt256) \
|
||||
M(Int8) \
|
||||
M(Int16) \
|
||||
@ -109,6 +109,8 @@ static IAggregateFunction * createWithUnsignedIntegerType(const IDataType & argu
|
||||
if (which.idx == TypeIndex::UInt16) return new AggregateFunctionTemplate<UInt16, Data<UInt16>>(std::forward<TArgs>(args)...);
|
||||
if (which.idx == TypeIndex::UInt32) return new AggregateFunctionTemplate<UInt32, Data<UInt32>>(std::forward<TArgs>(args)...);
|
||||
if (which.idx == TypeIndex::UInt64) return new AggregateFunctionTemplate<UInt64, Data<UInt64>>(std::forward<TArgs>(args)...);
|
||||
if (which.idx == TypeIndex::UInt128) return new AggregateFunctionTemplate<UInt128, Data<UInt128>>(std::forward<TArgs>(args)...);
|
||||
if (which.idx == TypeIndex::UInt256) return new AggregateFunctionTemplate<UInt256, Data<UInt256>>(std::forward<TArgs>(args)...);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
@ -119,11 +121,11 @@ static IAggregateFunction * createWithNumericBasedType(const IDataType & argumen
|
||||
if (f)
|
||||
return f;
|
||||
|
||||
/// expects that DataTypeDate based on UInt16, DataTypeDateTime based on UInt32 and UUID based on UInt128
|
||||
/// expects that DataTypeDate based on UInt16, DataTypeDateTime based on UInt32
|
||||
WhichDataType which(argument_type);
|
||||
if (which.idx == TypeIndex::Date) return new AggregateFunctionTemplate<UInt16>(std::forward<TArgs>(args)...);
|
||||
if (which.idx == TypeIndex::DateTime) return new AggregateFunctionTemplate<UInt32>(std::forward<TArgs>(args)...);
|
||||
if (which.idx == TypeIndex::UUID) return new AggregateFunctionTemplate<UInt128>(std::forward<TArgs>(args)...);
|
||||
if (which.idx == TypeIndex::UUID) return new AggregateFunctionTemplate<UUID>(std::forward<TArgs>(args)...);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
@ -184,6 +186,29 @@ static IAggregateFunction * createWithTwoNumericTypes(const IDataType & first_ty
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
template <typename FirstType, template <typename, typename> class AggregateFunctionTemplate, typename... TArgs>
|
||||
static IAggregateFunction * createWithTwoBasicNumericTypesSecond(const IDataType & second_type, TArgs && ... args)
|
||||
{
|
||||
WhichDataType which(second_type);
|
||||
#define DISPATCH(TYPE) \
|
||||
if (which.idx == TypeIndex::TYPE) return new AggregateFunctionTemplate<FirstType, TYPE>(std::forward<TArgs>(args)...);
|
||||
FOR_BASIC_NUMERIC_TYPES(DISPATCH)
|
||||
#undef DISPATCH
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
template <template <typename, typename> class AggregateFunctionTemplate, typename... TArgs>
|
||||
static IAggregateFunction * createWithTwoBasicNumericTypes(const IDataType & first_type, const IDataType & second_type, TArgs && ... args)
|
||||
{
|
||||
WhichDataType which(first_type);
|
||||
#define DISPATCH(TYPE) \
|
||||
if (which.idx == TypeIndex::TYPE) \
|
||||
return createWithTwoBasicNumericTypesSecond<TYPE, AggregateFunctionTemplate>(second_type, std::forward<TArgs>(args)...);
|
||||
FOR_BASIC_NUMERIC_TYPES(DISPATCH)
|
||||
#undef DISPATCH
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
template <typename FirstType, template <typename, typename> class AggregateFunctionTemplate, typename... TArgs>
|
||||
static IAggregateFunction * createWithTwoNumericOrDateTypesSecond(const IDataType & second_type, TArgs && ... args)
|
||||
{
|
||||
@ -195,7 +220,7 @@ static IAggregateFunction * createWithTwoNumericOrDateTypesSecond(const IDataTyp
|
||||
if (which.idx == TypeIndex::Enum8) return new AggregateFunctionTemplate<FirstType, Int8>(std::forward<TArgs>(args)...);
|
||||
if (which.idx == TypeIndex::Enum16) return new AggregateFunctionTemplate<FirstType, Int16>(std::forward<TArgs>(args)...);
|
||||
|
||||
/// expects that DataTypeDate based on UInt16, DataTypeDateTime based on UInt32 and UUID based on UInt128
|
||||
/// expects that DataTypeDate based on UInt16, DataTypeDateTime based on UInt32
|
||||
if (which.idx == TypeIndex::Date) return new AggregateFunctionTemplate<FirstType, UInt16>(std::forward<TArgs>(args)...);
|
||||
if (which.idx == TypeIndex::DateTime) return new AggregateFunctionTemplate<FirstType, UInt32>(std::forward<TArgs>(args)...);
|
||||
|
||||
@ -216,7 +241,7 @@ static IAggregateFunction * createWithTwoNumericOrDateTypes(const IDataType & fi
|
||||
if (which.idx == TypeIndex::Enum16)
|
||||
return createWithTwoNumericOrDateTypesSecond<Int16, AggregateFunctionTemplate>(second_type, std::forward<TArgs>(args)...);
|
||||
|
||||
/// expects that DataTypeDate based on UInt16, DataTypeDateTime based on UInt32 and UUID based on UInt128
|
||||
/// expects that DataTypeDate based on UInt16, DataTypeDateTime based on UInt32
|
||||
if (which.idx == TypeIndex::Date)
|
||||
return createWithTwoNumericOrDateTypesSecond<UInt16, AggregateFunctionTemplate>(second_type, std::forward<TArgs>(args)...);
|
||||
if (which.idx == TypeIndex::DateTime)
|
||||
|
@ -263,7 +263,7 @@ struct QuantileExactLow : public QuantileExactBase<Value, QuantileExactLow<Value
|
||||
{
|
||||
// sort inputs in ascending order
|
||||
std::sort(array.begin(), array.end());
|
||||
size_t n = level < 1 ? level * array.size() : (array.size() - 1);
|
||||
|
||||
// if level is 0.5 then compute the "low" median of the sorted array
|
||||
// by the method of rounding.
|
||||
if (level == 0.5)
|
||||
@ -278,10 +278,14 @@ struct QuantileExactLow : public QuantileExactBase<Value, QuantileExactLow<Value
|
||||
return array[static_cast<size_t>((floor(s / 2)) - 1)];
|
||||
}
|
||||
}
|
||||
// else quantile is the nth index of the sorted array obtained by multiplying
|
||||
// level and size of array. Example if level = 0.1 and size of array is 10,
|
||||
// then return array[1].
|
||||
return array[n];
|
||||
else
|
||||
{
|
||||
// else quantile is the nth index of the sorted array obtained by multiplying
|
||||
// level and size of array. Example if level = 0.1 and size of array is 10,
|
||||
// then return array[1].
|
||||
size_t n = level < 1 ? level * array.size() : (array.size() - 1);
|
||||
return array[n];
|
||||
}
|
||||
}
|
||||
return std::numeric_limits<Value>::quiet_NaN();
|
||||
}
|
||||
@ -295,7 +299,7 @@ struct QuantileExactLow : public QuantileExactBase<Value, QuantileExactLow<Value
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
auto level = levels[indices[i]];
|
||||
size_t n = level < 1 ? level * array.size() : (array.size() - 1);
|
||||
|
||||
// if level is 0.5 then compute the "low" median of the sorted array
|
||||
// by the method of rounding.
|
||||
if (level == 0.5)
|
||||
@ -310,9 +314,13 @@ struct QuantileExactLow : public QuantileExactBase<Value, QuantileExactLow<Value
|
||||
result[indices[i]] = array[static_cast<size_t>(floor((s / 2) - 1))];
|
||||
}
|
||||
}
|
||||
// else quantile is the nth index of the sorted array obtained by multiplying
|
||||
// level and size of array. Example if level = 0.1 and size of array is 10.
|
||||
result[indices[i]] = array[n];
|
||||
else
|
||||
{
|
||||
// else quantile is the nth index of the sorted array obtained by multiplying
|
||||
// level and size of array. Example if level = 0.1 and size of array is 10.
|
||||
size_t n = level < 1 ? level * array.size() : (array.size() - 1);
|
||||
result[indices[i]] = array[n];
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
@ -337,7 +345,7 @@ struct QuantileExactHigh : public QuantileExactBase<Value, QuantileExactHigh<Val
|
||||
{
|
||||
// sort inputs in ascending order
|
||||
std::sort(array.begin(), array.end());
|
||||
size_t n = level < 1 ? level * array.size() : (array.size() - 1);
|
||||
|
||||
// if level is 0.5 then compute the "high" median of the sorted array
|
||||
// by the method of rounding.
|
||||
if (level == 0.5)
|
||||
@ -345,9 +353,13 @@ struct QuantileExactHigh : public QuantileExactBase<Value, QuantileExactHigh<Val
|
||||
auto s = array.size();
|
||||
return array[static_cast<size_t>(floor(s / 2))];
|
||||
}
|
||||
// else quantile is the nth index of the sorted array obtained by multiplying
|
||||
// level and size of array. Example if level = 0.1 and size of array is 10.
|
||||
return array[n];
|
||||
else
|
||||
{
|
||||
// else quantile is the nth index of the sorted array obtained by multiplying
|
||||
// level and size of array. Example if level = 0.1 and size of array is 10.
|
||||
size_t n = level < 1 ? level * array.size() : (array.size() - 1);
|
||||
return array[n];
|
||||
}
|
||||
}
|
||||
return std::numeric_limits<Value>::quiet_NaN();
|
||||
}
|
||||
@ -361,7 +373,7 @@ struct QuantileExactHigh : public QuantileExactBase<Value, QuantileExactHigh<Val
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
auto level = levels[indices[i]];
|
||||
size_t n = level < 1 ? level * array.size() : (array.size() - 1);
|
||||
|
||||
// if level is 0.5 then compute the "high" median of the sorted array
|
||||
// by the method of rounding.
|
||||
if (level == 0.5)
|
||||
@ -369,9 +381,13 @@ struct QuantileExactHigh : public QuantileExactBase<Value, QuantileExactHigh<Val
|
||||
auto s = array.size();
|
||||
result[indices[i]] = array[static_cast<size_t>(floor(s / 2))];
|
||||
}
|
||||
// else quantile is the nth index of the sorted array obtained by multiplying
|
||||
// level and size of array. Example if level = 0.1 and size of array is 10.
|
||||
result[indices[i]] = array[n];
|
||||
else
|
||||
{
|
||||
// else quantile is the nth index of the sorted array obtained by multiplying
|
||||
// level and size of array. Example if level = 0.1 and size of array is 10.
|
||||
size_t n = level < 1 ? level * array.size() : (array.size() - 1);
|
||||
result[indices[i]] = array[n];
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
|
@ -54,7 +54,7 @@ struct QuantileReservoirSampler
|
||||
/// Get the value of the `level` quantile. The level must be between 0 and 1.
|
||||
Value get(Float64 level)
|
||||
{
|
||||
return data.quantileInterpolated(level);
|
||||
return Value(data.quantileInterpolated(level));
|
||||
}
|
||||
|
||||
/// Get the `size` values of `levels` quantiles. Write `size` results starting with `result` address.
|
||||
@ -62,7 +62,7 @@ struct QuantileReservoirSampler
|
||||
void getMany(const Float64 * levels, const size_t * indices, size_t size, Value * result)
|
||||
{
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
result[indices[i]] = data.quantileInterpolated(levels[indices[i]]);
|
||||
result[indices[i]] = Value(data.quantileInterpolated(levels[indices[i]]));
|
||||
}
|
||||
|
||||
/// The same, but in the case of an empty state, NaN is returned.
|
||||
|
@ -131,12 +131,20 @@ public:
|
||||
size_t left_index = static_cast<size_t>(index);
|
||||
size_t right_index = left_index + 1;
|
||||
if (right_index == samples.size())
|
||||
return static_cast<double>(samples[left_index]);
|
||||
{
|
||||
if constexpr (DB::IsDecimalNumber<T>)
|
||||
return static_cast<double>(samples[left_index].value);
|
||||
else
|
||||
return static_cast<double>(samples[left_index]);
|
||||
}
|
||||
|
||||
double left_coef = right_index - index;
|
||||
double right_coef = index - left_index;
|
||||
|
||||
return static_cast<double>(samples[left_index]) * left_coef + static_cast<double>(samples[right_index]) * right_coef;
|
||||
if constexpr (DB::IsDecimalNumber<T>)
|
||||
return static_cast<double>(samples[left_index].value) * left_coef + static_cast<double>(samples[right_index].value) * right_coef;
|
||||
else
|
||||
return static_cast<double>(samples[left_index]) * left_coef + static_cast<double>(samples[right_index]) * right_coef;
|
||||
}
|
||||
|
||||
void merge(const ReservoirSampler<T, OnEmpty> & b)
|
||||
|
119
src/AggregateFunctions/ThetaSketchData.h
Normal file
119
src/AggregateFunctions/ThetaSketchData.h
Normal file
@ -0,0 +1,119 @@
|
||||
#pragma once
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include <Common/config.h>
|
||||
#endif
|
||||
|
||||
#if USE_DATASKETCHES
|
||||
|
||||
#include <boost/noncopyable.hpp>
|
||||
#include <memory>
|
||||
#include <theta_sketch.hpp> // Y_IGNORE
|
||||
#include <theta_union.hpp> // Y_IGNORE
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
|
||||
template <typename Key>
|
||||
class ThetaSketchData : private boost::noncopyable
|
||||
{
|
||||
private:
|
||||
std::unique_ptr<datasketches::update_theta_sketch> sk_update;
|
||||
std::unique_ptr<datasketches::theta_union> sk_union;
|
||||
|
||||
inline datasketches::update_theta_sketch * getSkUpdate()
|
||||
{
|
||||
if (!sk_update)
|
||||
sk_update = std::make_unique<datasketches::update_theta_sketch>(datasketches::update_theta_sketch::builder().build());
|
||||
return sk_update.get();
|
||||
}
|
||||
|
||||
inline datasketches::theta_union * getSkUnion()
|
||||
{
|
||||
if (!sk_union)
|
||||
sk_union = std::make_unique<datasketches::theta_union>(datasketches::theta_union::builder().build());
|
||||
return sk_union.get();
|
||||
}
|
||||
|
||||
public:
|
||||
using value_type = Key;
|
||||
|
||||
ThetaSketchData() = default;
|
||||
~ThetaSketchData() = default;
|
||||
|
||||
/// Insert original value without hash, as `datasketches::update_theta_sketch.update` will do the hash internal.
|
||||
void insertOriginal(const StringRef & value)
|
||||
{
|
||||
getSkUpdate()->update(value.data, value.size);
|
||||
}
|
||||
|
||||
/// Note that `datasketches::update_theta_sketch.update` will do the hash again.
|
||||
void insert(Key value)
|
||||
{
|
||||
getSkUpdate()->update(value);
|
||||
}
|
||||
|
||||
UInt64 size() const
|
||||
{
|
||||
if (sk_union)
|
||||
return static_cast<UInt64>(sk_union->get_result().get_estimate());
|
||||
else if (sk_update)
|
||||
return static_cast<UInt64>(sk_update->get_estimate());
|
||||
else
|
||||
return 0;
|
||||
}
|
||||
|
||||
void merge(const ThetaSketchData & rhs)
|
||||
{
|
||||
datasketches::theta_union * u = getSkUnion();
|
||||
|
||||
if (sk_update)
|
||||
{
|
||||
u->update(*sk_update);
|
||||
sk_update.reset(nullptr);
|
||||
}
|
||||
|
||||
if (rhs.sk_update)
|
||||
u->update(*rhs.sk_update);
|
||||
else if (rhs.sk_union)
|
||||
u->update(rhs.sk_union->get_result());
|
||||
}
|
||||
|
||||
/// You can only call for an empty object.
|
||||
void read(DB::ReadBuffer & in)
|
||||
{
|
||||
datasketches::compact_theta_sketch::vector_bytes bytes;
|
||||
readVectorBinary(bytes, in);
|
||||
if (!bytes.empty())
|
||||
{
|
||||
auto sk = datasketches::compact_theta_sketch::deserialize(bytes.data(), bytes.size());
|
||||
getSkUnion()->update(sk);
|
||||
}
|
||||
}
|
||||
|
||||
void write(DB::WriteBuffer & out) const
|
||||
{
|
||||
if (sk_update)
|
||||
{
|
||||
auto bytes = sk_update->compact().serialize();
|
||||
writeVectorBinary(bytes, out);
|
||||
}
|
||||
else if (sk_union)
|
||||
{
|
||||
auto bytes = sk_union->get_result().serialize();
|
||||
writeVectorBinary(bytes, out);
|
||||
}
|
||||
else
|
||||
{
|
||||
datasketches::compact_theta_sketch::vector_bytes bytes;
|
||||
writeVectorBinary(bytes, out);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
||||
#endif
|
@ -3,7 +3,6 @@
|
||||
#include <city.h>
|
||||
#include <Core/Defines.h>
|
||||
#include <Common/SipHash.h>
|
||||
#include <Common/UInt128.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
|
||||
@ -107,7 +106,7 @@ struct UniqVariadicHash<true, false>
|
||||
}
|
||||
|
||||
UInt128 key;
|
||||
hash.get128(key.low, key.high);
|
||||
hash.get128(key);
|
||||
return key;
|
||||
}
|
||||
};
|
||||
@ -131,7 +130,7 @@ struct UniqVariadicHash<true, true>
|
||||
}
|
||||
|
||||
UInt128 key;
|
||||
hash.get128(key.low, key.high);
|
||||
hash.get128(key);
|
||||
return key;
|
||||
}
|
||||
};
|
||||
|
@ -184,6 +184,7 @@ add_object_library(clickhouse_disks Disks)
|
||||
add_object_library(clickhouse_interpreters Interpreters)
|
||||
add_object_library(clickhouse_interpreters_mysql Interpreters/MySQL)
|
||||
add_object_library(clickhouse_interpreters_clusterproxy Interpreters/ClusterProxy)
|
||||
add_object_library(clickhouse_interpreters_jit Interpreters/JIT)
|
||||
add_object_library(clickhouse_columns Columns)
|
||||
add_object_library(clickhouse_storages Storages)
|
||||
add_object_library(clickhouse_storages_distributed Storages/Distributed)
|
||||
|
@ -85,8 +85,8 @@ private:
|
||||
{}
|
||||
|
||||
public:
|
||||
const char * getFamilyName() const override { return TypeName<T>::get(); }
|
||||
TypeIndex getDataType() const override { return TypeId<T>::value; }
|
||||
const char * getFamilyName() const override { return TypeName<T>; }
|
||||
TypeIndex getDataType() const override { return TypeId<T>; }
|
||||
|
||||
bool isNumeric() const override { return false; }
|
||||
bool canBeInsideNullable() const override { return true; }
|
||||
|
@ -665,7 +665,7 @@ UInt128 ColumnUnique<ColumnType>::IncrementalHash::getHash(const ColumnType & co
|
||||
column.updateHashWithValue(i, sip_hash);
|
||||
|
||||
std::lock_guard lock(mutex);
|
||||
sip_hash.get128(hash.low, hash.high);
|
||||
sip_hash.get128(hash);
|
||||
cur_hash = hash;
|
||||
num_added_rows.store(column_size);
|
||||
}
|
||||
|
@ -33,6 +33,7 @@ namespace ErrorCodes
|
||||
extern const int PARAMETER_OUT_OF_BOUND;
|
||||
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
@ -154,7 +155,7 @@ void ColumnVector<T>::getPermutation(bool reverse, size_t limit, int nan_directi
|
||||
else
|
||||
{
|
||||
/// A case for radix sort
|
||||
if constexpr (is_arithmetic_v<T> && !std::is_same_v<T, UInt128>)
|
||||
if constexpr (is_arithmetic_v<T> && !is_big_int_v<T>)
|
||||
{
|
||||
/// Thresholds on size. Lower threshold is arbitrary. Upper threshold is chosen by the type for histogram counters.
|
||||
if (s >= 256 && s <= std::numeric_limits<UInt32>::max())
|
||||
@ -292,28 +293,37 @@ MutableColumnPtr ColumnVector<T>::cloneResized(size_t size) const
|
||||
memcpy(new_col.data.data(), data.data(), count * sizeof(data[0]));
|
||||
|
||||
if (size > count)
|
||||
memset(static_cast<void *>(&new_col.data[count]), static_cast<int>(ValueType()), (size - count) * sizeof(ValueType));
|
||||
memset(static_cast<void *>(&new_col.data[count]), 0, (size - count) * sizeof(ValueType));
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
UInt64 ColumnVector<T>::get64(size_t n) const
|
||||
UInt64 ColumnVector<T>::get64(size_t n [[maybe_unused]]) const
|
||||
{
|
||||
return ext::bit_cast<UInt64>(data[n]);
|
||||
if constexpr (is_arithmetic_v<T>)
|
||||
return ext::bit_cast<UInt64>(data[n]);
|
||||
else
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get the value of {} as UInt64", TypeName<T>);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline Float64 ColumnVector<T>::getFloat64(size_t n) const
|
||||
inline Float64 ColumnVector<T>::getFloat64(size_t n [[maybe_unused]]) const
|
||||
{
|
||||
return static_cast<Float64>(data[n]);
|
||||
if constexpr (is_arithmetic_v<T>)
|
||||
return static_cast<Float64>(data[n]);
|
||||
else
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get the value of {} as Float64", TypeName<T>);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
Float32 ColumnVector<T>::getFloat32(size_t n) const
|
||||
Float32 ColumnVector<T>::getFloat32(size_t n [[maybe_unused]]) const
|
||||
{
|
||||
return static_cast<Float32>(data[n]);
|
||||
if constexpr (is_arithmetic_v<T>)
|
||||
return static_cast<Float32>(data[n]);
|
||||
else
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get the value of {} as Float32", TypeName<T>);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
@ -482,8 +492,6 @@ void ColumnVector<T>::gather(ColumnGathererStream & gatherer)
|
||||
template <typename T>
|
||||
void ColumnVector<T>::getExtremes(Field & min, Field & max) const
|
||||
{
|
||||
using FastRefT = std::conditional_t<is_big_int_v<T>, const T &, const T>;
|
||||
|
||||
size_t size = data.size();
|
||||
|
||||
if (size == 0)
|
||||
@ -504,7 +512,7 @@ void ColumnVector<T>::getExtremes(Field & min, Field & max) const
|
||||
T cur_min = NaNOrZero<T>();
|
||||
T cur_max = NaNOrZero<T>();
|
||||
|
||||
for (FastRefT x : data)
|
||||
for (const T & x : data)
|
||||
{
|
||||
if (isNaN(x))
|
||||
continue;
|
||||
@ -569,5 +577,6 @@ template class ColumnVector<Int128>;
|
||||
template class ColumnVector<Int256>;
|
||||
template class ColumnVector<Float32>;
|
||||
template class ColumnVector<Float64>;
|
||||
template class ColumnVector<UUID>;
|
||||
|
||||
}
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user