Merge branch 'better_ttl_merges_selection' into recompression_in_background

This commit is contained in:
alesapin 2020-09-02 13:30:04 +03:00
commit aac466ab1c
78 changed files with 594 additions and 515 deletions

View File

@ -121,3 +121,18 @@ template <> struct is_big_int<bInt256> { static constexpr bool value = true; };
template <typename T>
inline constexpr bool is_big_int_v = is_big_int<T>::value;
template <typename T>
inline std::string bigintToString(const T & x)
{
return x.str();
}
template <typename To, typename From>
inline To bigint_cast(const From & x [[maybe_unused]])
{
if constexpr ((is_big_int_v<From> && std::is_same_v<To, UInt8>) || (is_big_int_v<To> && std::is_same_v<From, UInt8>))
return static_cast<uint8_t>(x);
else
return static_cast<To>(x);
}

View File

@ -106,8 +106,8 @@ AggregateFunctionPtr createAggregateFunctionQuantile(const std::string & name, c
if constexpr (supportBigInt<Function>())
{
if (which.idx == TypeIndex::Int128) return std::make_shared<Function<Int128, true>>(argument_types, params);
if (which.idx == TypeIndex::bInt256) return std::make_shared<Function<bInt256, true>>(argument_types, params);
if (which.idx == TypeIndex::bUInt256) return std::make_shared<Function<bUInt256, true>>(argument_types, params);
if (which.idx == TypeIndex::Int256) return std::make_shared<Function<Int256, true>>(argument_types, params);
if (which.idx == TypeIndex::UInt256) return std::make_shared<Function<UInt256, true>>(argument_types, params);
}
throw Exception("Illegal type " + argument_type->getName() + " of argument for aggregate function " + name,

View File

@ -137,23 +137,23 @@ struct AggregateFunctionUniqUpToData<UInt128> : AggregateFunctionUniqUpToData<UI
};
template <>
struct AggregateFunctionUniqUpToData<bUInt256> : AggregateFunctionUniqUpToData<UInt64>
struct AggregateFunctionUniqUpToData<UInt256> : AggregateFunctionUniqUpToData<UInt64>
{
/// ALWAYS_INLINE is required to have better code layout for uniqUpTo function
void ALWAYS_INLINE add(const IColumn & column, size_t row_num, UInt8 threshold)
{
bUInt256 value = assert_cast<const ColumnVector<bUInt256> &>(column).getData()[row_num];
UInt256 value = assert_cast<const ColumnVector<UInt256> &>(column).getData()[row_num];
insert(sipHash64(value), threshold);
}
};
template <>
struct AggregateFunctionUniqUpToData<bInt256> : AggregateFunctionUniqUpToData<UInt64>
struct AggregateFunctionUniqUpToData<Int256> : AggregateFunctionUniqUpToData<UInt64>
{
/// ALWAYS_INLINE is required to have better code layout for uniqUpTo function
void ALWAYS_INLINE add(const IColumn & column, size_t row_num, UInt8 threshold)
{
bInt256 value = assert_cast<const ColumnVector<bInt256> &>(column).getData()[row_num];
Int256 value = assert_cast<const ColumnVector<Int256> &>(column).getData()[row_num];
insert(sipHash64(value), threshold);
}
};

View File

@ -20,13 +20,13 @@
M(UInt16) \
M(UInt32) \
M(UInt64) \
M(bUInt256) \
M(UInt256) \
M(Int8) \
M(Int16) \
M(Int32) \
M(Int64) \
M(Int128) \
M(bInt256) \
M(Int256) \
M(Float32) \
M(Float64)

View File

@ -61,7 +61,7 @@ StringRef ColumnDecimal<T>::serializeValueIntoArena(size_t n, Arena & arena, cha
else
{
char * pos = arena.allocContinue(BigInt<T>::size, begin);
return BigInt<bInt256>::serialize(data[n], pos);
return BigInt<Int256>::serialize(data[n], pos);
}
}
@ -75,8 +75,8 @@ const char * ColumnDecimal<T>::deserializeAndInsertFromArena(const char * pos)
}
else
{
data.push_back(BigInt<bInt256>::deserialize(pos));
return pos + BigInt<bInt256>::size;
data.push_back(BigInt<Int256>::deserialize(pos));
return pos + BigInt<Int256>::size;
}
}
@ -273,7 +273,7 @@ void ColumnDecimal<T>::insertData(const char * src, size_t /*length*/)
}
else
{
data.push_back(BigInt<bInt256>::deserialize(src));
data.push_back(BigInt<Int256>::deserialize(src));
}
}

View File

@ -598,13 +598,13 @@ template class ColumnVector<UInt16>;
template class ColumnVector<UInt32>;
template class ColumnVector<UInt64>;
template class ColumnVector<UInt128>;
template class ColumnVector<bUInt256>;
template class ColumnVector<UInt256>;
template class ColumnVector<Int8>;
template class ColumnVector<Int16>;
template class ColumnVector<Int32>;
template class ColumnVector<Int64>;
template class ColumnVector<Int128>;
template class ColumnVector<bInt256>;
template class ColumnVector<Int256>;
template class ColumnVector<Float32>;
template class ColumnVector<Float64>;
}

View File

@ -14,14 +14,14 @@ using ColumnUInt16 = ColumnVector<UInt16>;
using ColumnUInt32 = ColumnVector<UInt32>;
using ColumnUInt64 = ColumnVector<UInt64>;
using ColumnUInt128 = ColumnVector<UInt128>;
using ColumnUInt256 = ColumnVector<bUInt256>;
using ColumnUInt256 = ColumnVector<UInt256>;
using ColumnInt8 = ColumnVector<Int8>;
using ColumnInt16 = ColumnVector<Int16>;
using ColumnInt32 = ColumnVector<Int32>;
using ColumnInt64 = ColumnVector<Int64>;
using ColumnInt128 = ColumnVector<Int128>;
using ColumnInt256 = ColumnVector<bInt256>;
using ColumnInt256 = ColumnVector<Int256>;
using ColumnFloat32 = ColumnVector<Float32>;
using ColumnFloat64 = ColumnVector<Float64>;

View File

@ -48,8 +48,8 @@ String FieldVisitorDump::operator() (const DecimalField<Decimal32> & x) const {
String FieldVisitorDump::operator() (const DecimalField<Decimal64> & x) const { return formatQuotedWithPrefix(x, "Decimal64_"); }
String FieldVisitorDump::operator() (const DecimalField<Decimal128> & x) const { return formatQuotedWithPrefix(x, "Decimal128_"); }
String FieldVisitorDump::operator() (const DecimalField<Decimal256> & x) const { return formatQuotedWithPrefix(x, "Decimal256_"); }
String FieldVisitorDump::operator() (const bUInt256 & x) const { return formatQuotedWithPrefix(x, "UInt256_"); }
String FieldVisitorDump::operator() (const bInt256 & x) const { return formatQuotedWithPrefix(x, "Int256_"); }
String FieldVisitorDump::operator() (const UInt256 & x) const { return formatQuotedWithPrefix(x, "UInt256_"); }
String FieldVisitorDump::operator() (const Int256 & x) const { return formatQuotedWithPrefix(x, "Int256_"); }
String FieldVisitorDump::operator() (const Int128 & x) const { return formatQuotedWithPrefix(x, "Int128_"); }
String FieldVisitorDump::operator() (const UInt128 & x) const { return formatQuotedWithPrefix(UUID(x), "UUID_"); }
@ -141,8 +141,8 @@ String FieldVisitorToString::operator() (const AggregateFunctionStateData & x) c
{
return formatQuoted(x.data);
}
String FieldVisitorToString::operator() (const bUInt256 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const bInt256 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const UInt256 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const Int256 & x) const { return formatQuoted(x); }
String FieldVisitorToString::operator() (const Array & x) const
{
@ -286,16 +286,16 @@ void FieldVisitorHash::operator() (const AggregateFunctionStateData & x) const
hash.update(x.data.data(), x.data.size());
}
void FieldVisitorHash::operator() (const bUInt256 & x) const
void FieldVisitorHash::operator() (const UInt256 & x) const
{
UInt8 type = Field::Types::bUInt256;
UInt8 type = Field::Types::UInt256;
hash.update(type);
hash.update(x);
}
void FieldVisitorHash::operator() (const bInt256 & x) const
void FieldVisitorHash::operator() (const Int256 & x) const
{
UInt8 type = Field::Types::bInt256;
UInt8 type = Field::Types::Int256;
hash.update(type);
hash.update(x);
}

View File

@ -82,8 +82,8 @@ public:
String operator() (const DecimalField<Decimal256> & x) const;
String operator() (const AggregateFunctionStateData & x) const;
String operator() (const bUInt256 & x) const;
String operator() (const bInt256 & x) const;
String operator() (const UInt256 & x) const;
String operator() (const Int256 & x) const;
};
@ -106,8 +106,8 @@ public:
String operator() (const DecimalField<Decimal256> & x) const;
String operator() (const AggregateFunctionStateData & x) const;
String operator() (const bUInt256 & x) const;
String operator() (const bInt256 & x) const;
String operator() (const UInt256 & x) const;
String operator() (const Int256 & x) const;
};
@ -143,7 +143,7 @@ public:
T operator() (const Float64 & x) const
{
if constexpr (std::is_same_v<Decimal256, T>)
return bInt256(x);
return Int256(x);
else
return T(x);
}
@ -187,12 +187,10 @@ public:
{
if constexpr (IsDecimalNumber<T>)
return static_cast<T>(static_cast<typename T::NativeType>(x));
else if constexpr (std::is_same_v<T, UInt8>)
return static_cast<T>(static_cast<UInt16>(x));
else if constexpr (std::is_same_v<T, UInt128>)
throw Exception("No conversion to old UInt128 from " + demangle(typeid(U).name()), ErrorCodes::NOT_IMPLEMENTED);
else
return static_cast<T>(x);
return bigint_cast<T>(x);
}
};
@ -220,8 +218,8 @@ public:
void operator() (const DecimalField<Decimal256> & x) const;
void operator() (const AggregateFunctionStateData & x) const;
void operator() (const bUInt256 & x) const;
void operator() (const bInt256 & x) const;
void operator() (const UInt256 & x) const;
void operator() (const Int256 & x) const;
};

View File

@ -280,13 +280,13 @@ DEFINE_HASH(DB::UInt16)
DEFINE_HASH(DB::UInt32)
DEFINE_HASH(DB::UInt64)
DEFINE_HASH(DB::UInt128)
DEFINE_HASH(DB::bUInt256)
DEFINE_HASH(DB::UInt256)
DEFINE_HASH(DB::Int8)
DEFINE_HASH(DB::Int16)
DEFINE_HASH(DB::Int32)
DEFINE_HASH(DB::Int64)
DEFINE_HASH(DB::Int128)
DEFINE_HASH(DB::bInt256)
DEFINE_HASH(DB::Int256)
DEFINE_HASH(DB::Float32)
DEFINE_HASH(DB::Float64)
@ -297,7 +297,7 @@ template <>
struct DefaultHash<DB::UInt128> : public DB::UInt128Hash {};
template <>
struct DefaultHash<DB::UInt256> : public DB::UInt256Hash {};
struct DefaultHash<DB::DummyUInt256> : public DB::UInt256Hash {};
/// It is reasonable to use for UInt8, UInt16 with sufficient hash table size.

View File

@ -136,7 +136,7 @@ struct UInt128TrivialHash
/** Used for aggregation, for putting a large number of constant-length keys in a hash table.
*/
struct UInt256
struct DummyUInt256
{
/// Suppress gcc7 warnings: 'prev_key.DB::UInt256::a' may be used uninitialized in this function
@ -150,7 +150,7 @@ struct UInt256
UInt64 c;
UInt64 d;
bool operator== (const UInt256 rhs) const
bool operator== (const DummyUInt256 rhs) const
{
return a == rhs.a && b == rhs.b && c == rhs.c && d == rhs.d;
@ -164,7 +164,7 @@ struct UInt256
_mm_loadu_si128(reinterpret_cast<const __m128i *>(&rhs.c)))));*/
}
bool operator!= (const UInt256 rhs) const { return !operator==(rhs); }
bool operator!= (const DummyUInt256 rhs) const { return !operator==(rhs); }
bool operator== (const UInt64 rhs) const { return a == rhs && b == 0 && c == 0 && d == 0; }
bool operator!= (const UInt64 rhs) const { return !operator==(rhs); }
@ -173,12 +173,12 @@ struct UInt256
#pragma GCC diagnostic pop
#endif
UInt256 & operator= (const UInt64 rhs) { a = rhs; b = 0; c = 0; d = 0; return *this; }
DummyUInt256 & operator = (const UInt64 rhs) { a = rhs; b = 0; c = 0; d = 0; return *this; }
};
struct UInt256Hash
{
size_t operator()(UInt256 x) const
size_t operator()(DummyUInt256 x) const
{
/// NOTE suboptimal
return CityHash_v1_0_2::Hash128to64({CityHash_v1_0_2::Hash128to64({x.a, x.b}), CityHash_v1_0_2::Hash128to64({x.c, x.d})});
@ -189,7 +189,7 @@ struct UInt256Hash
struct UInt256HashCRC32
{
size_t operator()(UInt256 x) const
size_t operator()(DummyUInt256 x) const
{
UInt64 crc = -1ULL;
crc = _mm_crc32_u64(crc, x.a);

View File

@ -52,7 +52,7 @@ template <typename TInt, typename TUInt>
constexpr bool is_le_int_vs_uint = is_any_int_vs_uint<TInt, TUInt> && (sizeof(TInt) <= sizeof(TUInt));
static_assert(is_le_int_vs_uint<Int128, DB::UInt128>);
static_assert(is_le_int_vs_uint<Int128, bUInt256>);
static_assert(is_le_int_vs_uint<Int128, DB::UInt256>);
template <typename TInt, typename TUInt>
using bool_if_le_int_vs_uint_t = std::enable_if_t<is_le_int_vs_uint<TInt, TUInt>, bool>;
@ -92,39 +92,27 @@ using bool_if_gt_int_vs_uint = std::enable_if_t<is_gt_int_vs_uint<TInt, TUInt>,
template <typename TInt, typename TUInt>
inline bool_if_gt_int_vs_uint<TInt, TUInt> greaterOpTmpl(TInt a, TUInt b)
{
if constexpr (is_big_int_v<TInt> && std::is_same_v<TUInt, UInt8>)
return static_cast<TInt>(a) > static_cast<TInt>(static_cast<UInt16>(b));
else
return static_cast<TInt>(a) > static_cast<TInt>(b);
return bigint_cast<TInt>(a) > bigint_cast<TInt>(b);
}
template <typename TInt, typename TUInt>
inline bool_if_gt_int_vs_uint<TInt, TUInt> greaterOpTmpl(TUInt a, TInt b)
{
if constexpr (is_big_int_v<TInt> && std::is_same_v<TUInt, UInt8>)
return static_cast<TInt>(static_cast<UInt16>(a)) > static_cast<TInt>(b);
else if constexpr (is_big_int_v<TInt> && std::is_same_v<TUInt, DB::UInt128>)
return static_cast<bUInt256>(a) > b;
else
return static_cast<TInt>(a) > b;
using CastA = std::conditional_t<is_big_int_v<TInt> && std::is_same_v<TUInt, DB::UInt128>, DB::UInt256, TInt>;
return bigint_cast<CastA>(a) > b;
}
template <typename TInt, typename TUInt>
inline bool_if_gt_int_vs_uint<TInt, TUInt> equalsOpTmpl(TInt a, TUInt b)
{
if constexpr (is_big_int_v<TInt> && std::is_same_v<TUInt, UInt8>)
return static_cast<TInt>(a) == static_cast<TInt>(static_cast<UInt16>(b));
else
return static_cast<TInt>(a) == static_cast<TInt>(b);
return bigint_cast<TInt>(a) == bigint_cast<TInt>(b);
}
template <typename TInt, typename TUInt>
inline bool_if_gt_int_vs_uint<TInt, TUInt> equalsOpTmpl(TUInt a, TInt b)
{
if constexpr (is_big_int_v<TInt> && std::is_same_v<TUInt, UInt8>)
return static_cast<TInt>(static_cast<UInt16>(a)) == static_cast<TInt>(b);
else
return static_cast<TInt>(a) == static_cast<TInt>(b);
return bigint_cast<TInt>(a) == bigint_cast<TInt>(b);
}
@ -200,16 +188,14 @@ inline bool_if_not_safe_conversion<A, B> greaterOp(A a, B b)
template <typename A, typename B>
inline bool_if_safe_conversion<A, B> greaterOp(A a, B b)
{
if constexpr (is_big_int_v<A> && std::is_same_v<B, UInt8>)
return a > static_cast<UInt16>(b);
else if constexpr (is_big_int_v<B> && std::is_same_v<A, UInt8>)
return static_cast<UInt16>(a) > b;
else if constexpr (std::is_same_v<A, DB::UInt128> && is_big_int_v<B>)
// hack for UInt128 and bUInt128
return static_cast<B>(a) > b;
else if constexpr (std::is_same_v<B, DB::UInt128> && is_big_int_v<A>)
// hack for UInt128 and bUInt128
return a > static_cast<A>(b);
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
return bigint_cast<CastA>(a) > bigint_cast<CastB>(b);
else
return a > b;
}
@ -317,17 +303,9 @@ inline bool_if_not_safe_conversion<A, B> equalsOp(A a, B b)
template <typename A, typename B>
inline bool_if_safe_conversion<A, B> equalsOp(A a, B b)
{
using LargestType = std::conditional_t<sizeof(A) >= sizeof(B), A, B>;
using LargestType = std::conditional_t<(sizeof(A) > sizeof(B)) || ((sizeof(A) == sizeof(B)) && !std::is_same_v<A, DB::UInt128>), A, B>;
if constexpr (is_big_int_v<LargestType> && std::is_same_v<A, UInt8>)
return static_cast<LargestType>(static_cast<UInt16>(a)) == static_cast<LargestType>(b);
else if constexpr (is_big_int_v<LargestType> && std::is_same_v<B, UInt8>)
return static_cast<LargestType>(a) == static_cast<LargestType>(static_cast<UInt16>(b));
else if constexpr (std::is_same_v<LargestType, DB::UInt128> && is_big_int_v<B>)
// hack for UInt128 and bUInt128
return static_cast<B>(a) == static_cast<B>(b);
else
return static_cast<LargestType>(a) == static_cast<LargestType>(b);
return bigint_cast<LargestType>(a) == bigint_cast<LargestType>(b);
}
template <>
@ -443,16 +421,14 @@ inline bool_if_not_safe_conversion<A, B> notEqualsOp(A a, B b)
template <typename A, typename B>
inline bool_if_safe_conversion<A, B> notEqualsOp(A a, B b)
{
if constexpr (std::is_same_v<A, UInt8> && is_big_int_v<B>)
return static_cast<UInt16>(a) != b;
else if constexpr (std::is_same_v<B, UInt8> && is_big_int_v<A>)
return a != static_cast<UInt16>(b);
else if constexpr (std::is_same_v<A, DB::UInt128> && is_big_int_v<B>)
// hack for UInt128 and bUInt128
return static_cast<B>(a) != b;
else if constexpr (std::is_same_v<B, DB::UInt128> && is_big_int_v<A>)
// hack for UInt128 and bUInt128
return a != static_cast<A>(b);
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
return bigint_cast<CastA>(a) != bigint_cast<CastB>(b);
else
return a != b;
}
@ -467,16 +443,14 @@ inline bool_if_not_safe_conversion<A, B> lessOp(A a, B b)
template <typename A, typename B>
inline bool_if_safe_conversion<A, B> lessOp(A a, B b)
{
if constexpr (std::is_same_v<A, UInt8> && is_big_int_v<B>)
return static_cast<UInt16>(a) < b;
else if constexpr (std::is_same_v<B, UInt8> && is_big_int_v<A>)
return a < static_cast<UInt16>(b);
else if constexpr (std::is_same_v<A, DB::UInt128> && is_big_int_v<B>)
// hack for UInt128 and bUInt128
return static_cast<B>(a) < b;
else if constexpr (std::is_same_v<B, DB::UInt128> && is_big_int_v<A>)
// hack for UInt128 and bUInt128
return a < static_cast<A>(b);
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
return bigint_cast<CastA>(a) < bigint_cast<CastB>(b);
else
return a < b;
}
@ -493,16 +467,14 @@ inline bool_if_not_safe_conversion<A, B> lessOrEqualsOp(A a, B b)
template <typename A, typename B>
inline bool_if_safe_conversion<A, B> lessOrEqualsOp(A a, B b)
{
if constexpr (std::is_same_v<A, UInt8> && is_big_int_v<B>)
return static_cast<UInt16>(a) <= b;
else if constexpr (std::is_same_v<B, UInt8> && is_big_int_v<A>)
return a <= static_cast<UInt16>(b);
else if constexpr (std::is_same_v<A, DB::UInt128> && is_big_int_v<B>)
// hack for UInt128 and bUInt128
return static_cast<B>(a) <= b;
else if constexpr (std::is_same_v<B, DB::UInt128> && is_big_int_v<A>)
// hack for UInt128 and bUInt128
return a <= static_cast<A>(b);
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
return bigint_cast<CastA>(a) <= bigint_cast<CastB>(b);
else
return a <= b;
}
@ -519,16 +491,14 @@ inline bool_if_not_safe_conversion<A, B> greaterOrEqualsOp(A a, B b)
template <typename A, typename B>
inline bool_if_safe_conversion<A, B> greaterOrEqualsOp(A a, B b)
{
if constexpr (std::is_same_v<A, UInt8> && is_big_int_v<B>)
return static_cast<UInt16>(a) >= b;
else if constexpr (std::is_same_v<B, UInt8> && is_big_int_v<A>)
return a >= static_cast<UInt16>(b);
else if constexpr (std::is_same_v<A, DB::UInt128> && is_big_int_v<B>)
// hack for UInt128 and bUInt128
return static_cast<B>(a) >= b;
else if constexpr (std::is_same_v<B, DB::UInt128> && is_big_int_v<A>)
// hack for UInt128 and bUInt128
return a >= static_cast<A>(b);
using CastA1 = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB1 = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, DB::UInt128>, B, CastA1>;
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, DB::UInt128>, A, CastB1>;
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
return bigint_cast<CastA>(a) >= bigint_cast<CastB>(b);
else
return a >= b;
}

View File

@ -37,7 +37,7 @@ inline bool allowDecimalComparison(const DataTypePtr & left_type, const DataType
template <size_t > struct ConstructDecInt { using Type = Int32; };
template <> struct ConstructDecInt<8> { using Type = Int64; };
template <> struct ConstructDecInt<16> { using Type = Int128; };
template <> struct ConstructDecInt<48> { using Type = bInt256; };
template <> struct ConstructDecInt<48> { using Type = Int256; };
template <typename T, typename U>
struct DecCompareInt
@ -228,18 +228,14 @@ private:
CompareInt x;
if constexpr (is_big_int_v<CompareInt> && IsDecimalNumber<A>)
x = a.value;
else if constexpr (is_big_int_v<CompareInt> && std::is_same_v<A, UInt8>)
x = static_cast<UInt16>(a);
else
x = static_cast<CompareInt>(a);
x = bigint_cast<CompareInt>(a);
CompareInt y;
if constexpr (is_big_int_v<CompareInt> && IsDecimalNumber<B>)
y = b.value;
else if constexpr (is_big_int_v<CompareInt> && std::is_same_v<B, UInt8>)
y = static_cast<UInt16>(b);
else
y = static_cast<CompareInt>(b);
y = bigint_cast<CompareInt>(b);
if constexpr (_check_overflow)
{

View File

@ -36,7 +36,7 @@ inline auto scaleMultiplier(UInt32 scale)
return common::exp10_i64(scale);
else if constexpr (std::is_same_v<T, Int128> || std::is_same_v<T, Decimal128>)
return common::exp10_i128(scale);
else if constexpr (std::is_same_v<T, bInt256> || std::is_same_v<T, Decimal256>)
else if constexpr (std::is_same_v<T, Int256> || std::is_same_v<T, Decimal256>)
return common::exp10_i256(scale);
}

View File

@ -218,16 +218,16 @@ void readBinary(Tuple & x, ReadBuffer & buf)
x.push_back(value);
break;
}
case Field::Types::bUInt256:
case Field::Types::UInt256:
{
bUInt256 value;
UInt256 value;
DB::readBinary(value, buf);
x.push_back(value);
break;
}
case Field::Types::bInt256:
case Field::Types::Int256:
{
bInt256 value;
Int256 value;
DB::readBinary(value, buf);
x.push_back(value);
break;
@ -301,14 +301,14 @@ void writeBinary(const Tuple & x, WriteBuffer & buf)
DB::writeStringBinary(get<std::string>(elem), buf);
break;
}
case Field::Types::bUInt256:
case Field::Types::UInt256:
{
DB::writeBinary(get<bUInt256>(elem), buf);
DB::writeBinary(get<UInt256>(elem), buf);
break;
}
case Field::Types::bInt256:
case Field::Types::Int256:
{
DB::writeBinary(get<bInt256>(elem), buf);
DB::writeBinary(get<Int256>(elem), buf);
break;
}
case Field::Types::Array:
@ -411,14 +411,14 @@ Field Field::restoreFromDump(const std::string_view & dump_)
prefix = std::string_view{"Int256_"};
if (dump.starts_with(prefix))
{
bInt256 value = parseFromString<bInt256>(dump.substr(prefix.length()));
Int256 value = parseFromString<Int256>(dump.substr(prefix.length()));
return value;
}
prefix = std::string_view{"UInt256_"};
if (dump.starts_with(prefix))
{
bUInt256 value = parseFromString<bUInt256>(dump.substr(prefix.length()));
UInt256 value = parseFromString<UInt256>(dump.substr(prefix.length()));
return value;
}

View File

@ -185,8 +185,8 @@ template <> struct NearestFieldTypeImpl<long long> { using Type = Int64; };
template <> struct NearestFieldTypeImpl<unsigned long> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<unsigned long long> { using Type = UInt64; };
template <> struct NearestFieldTypeImpl<bUInt256> { using Type = bUInt256; };
template <> struct NearestFieldTypeImpl<bInt256> { using Type = bInt256; };
template <> struct NearestFieldTypeImpl<UInt256> { using Type = UInt256; };
template <> struct NearestFieldTypeImpl<Int256> { using Type = Int256; };
template <> struct NearestFieldTypeImpl<Int128> { using Type = Int128; };
template <> struct NearestFieldTypeImpl<Decimal32> { using Type = DecimalField<Decimal32>; };
@ -255,8 +255,8 @@ public:
Decimal128 = 21,
AggregateFunctionState = 22,
Decimal256 = 23,
bUInt256 = 24,
bInt256 = 25,
UInt256 = 24,
Int256 = 25,
};
static const int MIN_NON_POD = 16;
@ -279,8 +279,8 @@ public:
case Decimal128: return "Decimal128";
case Decimal256: return "Decimal256";
case AggregateFunctionState: return "AggregateFunctionState";
case bUInt256: return "UInt256";
case bInt256: return "Int256";
case UInt256: return "UInt256";
case Int256: return "Int256";
}
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
@ -467,8 +467,8 @@ public:
case Types::Decimal128: return get<DecimalField<Decimal128>>() < rhs.get<DecimalField<Decimal128>>();
case Types::Decimal256: return get<DecimalField<Decimal256>>() < rhs.get<DecimalField<Decimal256>>();
case Types::AggregateFunctionState: return get<AggregateFunctionStateData>() < rhs.get<AggregateFunctionStateData>();
case Types::bUInt256: return get<bUInt256>() < rhs.get<bUInt256>();
case Types::bInt256: return get<bInt256>() < rhs.get<bInt256>();
case Types::UInt256: return get<UInt256>() < rhs.get<UInt256>();
case Types::Int256: return get<Int256>() < rhs.get<Int256>();
}
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
@ -502,8 +502,8 @@ public:
case Types::Decimal128: return get<DecimalField<Decimal128>>() <= rhs.get<DecimalField<Decimal128>>();
case Types::Decimal256: return get<DecimalField<Decimal256>>() <= rhs.get<DecimalField<Decimal256>>();
case Types::AggregateFunctionState: return get<AggregateFunctionStateData>() <= rhs.get<AggregateFunctionStateData>();
case Types::bUInt256: return get<bUInt256>() <= rhs.get<bUInt256>();
case Types::bInt256: return get<bInt256>() <= rhs.get<bInt256>();
case Types::UInt256: return get<UInt256>() <= rhs.get<UInt256>();
case Types::Int256: return get<Int256>() <= rhs.get<Int256>();
}
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
@ -541,8 +541,8 @@ public:
case Types::Decimal128: return get<DecimalField<Decimal128>>() == rhs.get<DecimalField<Decimal128>>();
case Types::Decimal256: return get<DecimalField<Decimal256>>() == rhs.get<DecimalField<Decimal256>>();
case Types::AggregateFunctionState: return get<AggregateFunctionStateData>() == rhs.get<AggregateFunctionStateData>();
case Types::bUInt256: return get<bUInt256>() == rhs.get<bUInt256>();
case Types::bInt256: return get<bInt256>() == rhs.get<bInt256>();
case Types::UInt256: return get<UInt256>() == rhs.get<UInt256>();
case Types::Int256: return get<Int256>() == rhs.get<Int256>();
}
throw Exception("Bad type of Field", ErrorCodes::BAD_TYPE_OF_FIELD);
@ -579,8 +579,8 @@ public:
case Types::Decimal256: return f(field.template get<DecimalField<Decimal256>>());
case Types::AggregateFunctionState: return f(field.template get<AggregateFunctionStateData>());
case Types::Int128: return f(field.template get<Int128>());
case Types::bUInt256: return f(field.template get<bUInt256>());
case Types::bInt256: return f(field.template get<bInt256>());
case Types::UInt256: return f(field.template get<UInt256>());
case Types::Int256: return f(field.template get<Int256>());
#if !__clang__
#pragma GCC diagnostic pop
#endif
@ -601,7 +601,7 @@ private:
Null, UInt64, UInt128, Int64, Int128, Float64, String, Array, Tuple,
DecimalField<Decimal32>, DecimalField<Decimal64>, DecimalField<Decimal128>, DecimalField<Decimal256>,
AggregateFunctionStateData,
bUInt256, bInt256
UInt256, Int256
> storage;
Types::Which which;
@ -732,8 +732,8 @@ template <> struct Field::TypeToEnum<DecimalField<Decimal64>>{ static const Type
template <> struct Field::TypeToEnum<DecimalField<Decimal128>>{ static const Types::Which value = Types::Decimal128; };
template <> struct Field::TypeToEnum<DecimalField<Decimal256>>{ static const Types::Which value = Types::Decimal256; };
template <> struct Field::TypeToEnum<AggregateFunctionStateData>{ static const Types::Which value = Types::AggregateFunctionState; };
template <> struct Field::TypeToEnum<bUInt256> { static const Types::Which value = Types::bUInt256; };
template <> struct Field::TypeToEnum<bInt256> { static const Types::Which value = Types::bInt256; };
template <> struct Field::TypeToEnum<UInt256> { static const Types::Which value = Types::UInt256; };
template <> struct Field::TypeToEnum<Int256> { static const Types::Which value = Types::Int256; };
template <> struct Field::EnumToType<Field::Types::Null> { using Type = Null; };
template <> struct Field::EnumToType<Field::Types::UInt64> { using Type = UInt64; };
@ -749,8 +749,8 @@ template <> struct Field::EnumToType<Field::Types::Decimal64> { using Type = Dec
template <> struct Field::EnumToType<Field::Types::Decimal128> { using Type = DecimalField<Decimal128>; };
template <> struct Field::EnumToType<Field::Types::Decimal256> { using Type = DecimalField<Decimal256>; };
template <> struct Field::EnumToType<Field::Types::AggregateFunctionState> { using Type = DecimalField<AggregateFunctionStateData>; };
template <> struct Field::EnumToType<Field::Types::bUInt256> { using Type = bUInt256; };
template <> struct Field::EnumToType<Field::Types::bInt256> { using Type = bInt256; };
template <> struct Field::EnumToType<Field::Types::UInt256> { using Type = UInt256; };
template <> struct Field::EnumToType<Field::Types::Int256> { using Type = Int256; };
inline constexpr bool isInt64FieldType(Field::Types::Which t)
{

View File

@ -8,7 +8,7 @@ namespace DB
{
using TypeListNativeNumbers = TypeList<UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Int64, Float32, Float64>;
using TypeListExtendedNumbers = TypeList<Int128, bUInt256, bInt256>;
using TypeListExtendedNumbers = TypeList<Int128, UInt256, Int256>;
using TypeListDecimalNumbers = TypeList<Decimal32, Decimal64, Decimal128, Decimal256>;
using TypeListGeneralNumbers = typename TypeListConcat<TypeListNativeNumbers, TypeListExtendedNumbers>::Type;

View File

@ -22,13 +22,13 @@ enum class TypeIndex
UInt32,
UInt64,
UInt128,
bUInt256,
UInt256,
Int8,
Int16,
Int32,
Int64,
Int128,
bInt256,
Int256,
Float32,
Float64,
Date,
@ -58,14 +58,14 @@ using UInt8 = ::UInt8;
using UInt16 = ::UInt16;
using UInt32 = ::UInt32;
using UInt64 = ::UInt64;
using bUInt256 = ::bUInt256;
using UInt256 = ::bUInt256;
using Int8 = ::Int8;
using Int16 = ::Int16;
using Int32 = ::Int32;
using Int64 = ::Int64;
using Int128 = ::Int128;
using bInt256 = ::bInt256;
using Int256 = ::bInt256;
using Float32 = float;
using Float64 = double;
@ -81,13 +81,13 @@ template <> inline constexpr bool IsNumber<UInt8> = true;
template <> inline constexpr bool IsNumber<UInt16> = true;
template <> inline constexpr bool IsNumber<UInt32> = true;
template <> inline constexpr bool IsNumber<UInt64> = true;
template <> inline constexpr bool IsNumber<bUInt256> = true;
template <> inline constexpr bool IsNumber<UInt256> = true;
template <> inline constexpr bool IsNumber<Int8> = true;
template <> inline constexpr bool IsNumber<Int16> = true;
template <> inline constexpr bool IsNumber<Int32> = true;
template <> inline constexpr bool IsNumber<Int64> = true;
template <> inline constexpr bool IsNumber<Int128> = true;
template <> inline constexpr bool IsNumber<bInt256> = true;
template <> inline constexpr bool IsNumber<Int256> = true;
template <> inline constexpr bool IsNumber<Float32> = true;
template <> inline constexpr bool IsNumber<Float64> = true;
@ -97,13 +97,13 @@ template <> struct TypeName<UInt8> { static constexpr const char * get() { ret
template <> struct TypeName<UInt16> { static constexpr const char * get() { return "UInt16"; } };
template <> struct TypeName<UInt32> { static constexpr const char * get() { return "UInt32"; } };
template <> struct TypeName<UInt64> { static constexpr const char * get() { return "UInt64"; } };
template <> struct TypeName<bUInt256> { static constexpr const char * get() { return "UInt256"; } };
template <> struct TypeName<UInt256> { static constexpr const char * get() { return "UInt256"; } };
template <> struct TypeName<Int8> { static constexpr const char * get() { return "Int8"; } };
template <> struct TypeName<Int16> { static constexpr const char * get() { return "Int16"; } };
template <> struct TypeName<Int32> { static constexpr const char * get() { return "Int32"; } };
template <> struct TypeName<Int64> { static constexpr const char * get() { return "Int64"; } };
template <> struct TypeName<Int128> { static constexpr const char * get() { return "Int128"; } };
template <> struct TypeName<bInt256> { static constexpr const char * get() { return "Int256"; } };
template <> struct TypeName<Int256> { static constexpr const char * get() { return "Int256"; } };
template <> struct TypeName<Float32> { static constexpr const char * get() { return "Float32"; } };
template <> struct TypeName<Float64> { static constexpr const char * get() { return "Float64"; } };
template <> struct TypeName<String> { static constexpr const char * get() { return "String"; } };
@ -113,13 +113,13 @@ template <> struct TypeId<UInt8> { static constexpr const TypeIndex value = T
template <> struct TypeId<UInt16> { static constexpr const TypeIndex value = TypeIndex::UInt16; };
template <> struct TypeId<UInt32> { static constexpr const TypeIndex value = TypeIndex::UInt32; };
template <> struct TypeId<UInt64> { static constexpr const TypeIndex value = TypeIndex::UInt64; };
template <> struct TypeId<bUInt256> { static constexpr const TypeIndex value = TypeIndex::bUInt256; };
template <> struct TypeId<UInt256> { static constexpr const TypeIndex value = TypeIndex::UInt256; };
template <> struct TypeId<Int8> { static constexpr const TypeIndex value = TypeIndex::Int8; };
template <> struct TypeId<Int16> { static constexpr const TypeIndex value = TypeIndex::Int16; };
template <> struct TypeId<Int32> { static constexpr const TypeIndex value = TypeIndex::Int32; };
template <> struct TypeId<Int64> { static constexpr const TypeIndex value = TypeIndex::Int64; };
template <> struct TypeId<Int128> { static constexpr const TypeIndex value = TypeIndex::Int128; };
template <> struct TypeId<bInt256> { static constexpr const TypeIndex value = TypeIndex::bInt256; };
template <> struct TypeId<Int256> { static constexpr const TypeIndex value = TypeIndex::Int256; };
template <> struct TypeId<Float32> { static constexpr const TypeIndex value = TypeIndex::Float32; };
template <> struct TypeId<Float64> { static constexpr const TypeIndex value = TypeIndex::Float64; };
@ -158,16 +158,12 @@ struct Decimal
if constexpr (std::is_same_v<U, Decimal<Int32>> ||
std::is_same_v<U, Decimal<Int64>> ||
std::is_same_v<U, Decimal<Int128>> ||
std::is_same_v<U, Decimal<bInt256>>)
std::is_same_v<U, Decimal<Int256>>)
{
return convertTo<typename U::NativeType>();
}
else if constexpr (is_big_int_v<NativeType>)
{
return value. template convert_to<U>();
}
else
return static_cast<U>(value);
return bigint_cast<U>(value);
}
const Decimal<T> & operator += (const T & x) { value += x; return *this; }
@ -193,7 +189,7 @@ template <typename T> inline Decimal<T> operator- (const Decimal<T> & x) { retur
using Decimal32 = Decimal<Int32>;
using Decimal64 = Decimal<Int64>;
using Decimal128 = Decimal<Int128>;
using Decimal256 = Decimal<bInt256>;
using Decimal256 = Decimal<Int256>;
using DateTime64 = Decimal64;
@ -217,11 +213,11 @@ template <typename T> struct NativeType { using Type = T; };
template <> struct NativeType<Decimal32> { using Type = Int32; };
template <> struct NativeType<Decimal64> { using Type = Int64; };
template <> struct NativeType<Decimal128> { using Type = Int128; };
template <> struct NativeType<Decimal256> { using Type = bInt256; };
template <> struct NativeType<Decimal256> { using Type = Int256; };
template <typename T> constexpr bool OverBigInt = false;
template <> inline constexpr bool OverBigInt<bInt256> = true;
template <> inline constexpr bool OverBigInt<bUInt256> = true;
template <> inline constexpr bool OverBigInt<Int256> = true;
template <> inline constexpr bool OverBigInt<UInt256> = true;
template <> inline constexpr bool OverBigInt<Decimal256> = true;
inline constexpr const char * getTypeName(TypeIndex idx)
@ -234,13 +230,13 @@ inline constexpr const char * getTypeName(TypeIndex idx)
case TypeIndex::UInt32: return TypeName<UInt32>::get();
case TypeIndex::UInt64: return TypeName<UInt64>::get();
case TypeIndex::UInt128: return "UInt128";
case TypeIndex::bUInt256: return TypeName<bUInt256>::get();
case TypeIndex::UInt256: return TypeName<UInt256>::get();
case TypeIndex::Int8: return TypeName<Int8>::get();
case TypeIndex::Int16: return TypeName<Int16>::get();
case TypeIndex::Int32: return TypeName<Int32>::get();
case TypeIndex::Int64: return TypeName<Int64>::get();
case TypeIndex::Int128: return TypeName<Int128>::get();
case TypeIndex::bInt256: return TypeName<bInt256>::get();
case TypeIndex::Int256: return TypeName<Int256>::get();
case TypeIndex::Float32: return TypeName<Float32>::get();
case TypeIndex::Float64: return TypeName<Float64>::get();
case TypeIndex::Date: return "Date";

View File

@ -26,14 +26,14 @@ bool callOnBasicType(TypeIndex number, F && f)
case TypeIndex::UInt16: return f(TypePair<T, UInt16>());
case TypeIndex::UInt32: return f(TypePair<T, UInt32>());
case TypeIndex::UInt64: return f(TypePair<T, UInt64>());
case TypeIndex::bUInt256: return f(TypePair<T, bUInt256>());
case TypeIndex::UInt256: return f(TypePair<T, UInt256>());
case TypeIndex::Int8: return f(TypePair<T, Int8>());
case TypeIndex::Int16: return f(TypePair<T, Int16>());
case TypeIndex::Int32: return f(TypePair<T, Int32>());
case TypeIndex::Int64: return f(TypePair<T, Int64>());
case TypeIndex::Int128: return f(TypePair<T, Int128>());
case TypeIndex::bInt256: return f(TypePair<T, bInt256>());
case TypeIndex::Int256: return f(TypePair<T, Int256>());
case TypeIndex::Enum8: return f(TypePair<T, Int8>());
case TypeIndex::Enum16: return f(TypePair<T, Int16>());
@ -94,14 +94,14 @@ inline bool callOnBasicTypes(TypeIndex type_num1, TypeIndex type_num2, F && f)
case TypeIndex::UInt16: return callOnBasicType<UInt16, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::UInt32: return callOnBasicType<UInt32, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::UInt64: return callOnBasicType<UInt64, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::bUInt256: return callOnBasicType<bUInt256, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::UInt256: return callOnBasicType<UInt256, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::Int8: return callOnBasicType<Int8, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::Int16: return callOnBasicType<Int16, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::Int32: return callOnBasicType<Int32, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::Int64: return callOnBasicType<Int64, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::Int128: return callOnBasicType<Int128, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::bInt256: return callOnBasicType<bInt256, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::Int256: return callOnBasicType<Int256, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::Enum8: return callOnBasicType<Int8, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
case TypeIndex::Enum16: return callOnBasicType<Int16, _int, _float, _decimal, _datetime>(type_num2, std::forward<F>(f));
@ -171,14 +171,14 @@ bool callOnIndexAndDataType(TypeIndex number, F && f)
case TypeIndex::UInt16: return f(TypePair<DataTypeNumber<UInt16>, T>());
case TypeIndex::UInt32: return f(TypePair<DataTypeNumber<UInt32>, T>());
case TypeIndex::UInt64: return f(TypePair<DataTypeNumber<UInt64>, T>());
case TypeIndex::bUInt256: return f(TypePair<DataTypeNumber<bUInt256>, T>());
case TypeIndex::UInt256: return f(TypePair<DataTypeNumber<UInt256>, T>());
case TypeIndex::Int8: return f(TypePair<DataTypeNumber<Int8>, T>());
case TypeIndex::Int16: return f(TypePair<DataTypeNumber<Int16>, T>());
case TypeIndex::Int32: return f(TypePair<DataTypeNumber<Int32>, T>());
case TypeIndex::Int64: return f(TypePair<DataTypeNumber<Int64>, T>());
case TypeIndex::Int128: return f(TypePair<DataTypeNumber<Int128>, T>());
case TypeIndex::bInt256: return f(TypePair<DataTypeNumber<bInt256>, T>());
case TypeIndex::Int256: return f(TypePair<DataTypeNumber<Int256>, T>());
case TypeIndex::Float32: return f(TypePair<DataTypeNumber<Float32>, T>());
case TypeIndex::Float64: return f(TypePair<DataTypeNumber<Float64>, T>());

View File

@ -257,14 +257,14 @@ template class DataTypeNumberBase<UInt8>;
template class DataTypeNumberBase<UInt16>;
template class DataTypeNumberBase<UInt32>;
template class DataTypeNumberBase<UInt64>;
template class DataTypeNumberBase<UInt128>; // used only in UUID
template class DataTypeNumberBase<bUInt256>;
template class DataTypeNumberBase<UInt128>; // base for UUID
template class DataTypeNumberBase<UInt256>;
template class DataTypeNumberBase<Int8>;
template class DataTypeNumberBase<Int16>;
template class DataTypeNumberBase<Int32>;
template class DataTypeNumberBase<Int64>;
template class DataTypeNumberBase<Int128>;
template class DataTypeNumberBase<bInt256>;
template class DataTypeNumberBase<Int256>;
template class DataTypeNumberBase<Float32>;
template class DataTypeNumberBase<Float64>;

View File

@ -171,7 +171,7 @@ convertToDecimal(const typename FromDataType::FieldType & value, UInt32 scale)
else
{
if constexpr (is_big_int_v<FromFieldType>)
return convertDecimals<DataTypeDecimal<Decimal256>, ToDataType>(static_cast<bInt256>(value), 0, scale);
return convertDecimals<DataTypeDecimal<Decimal256>, ToDataType>(static_cast<Int256>(value), 0, scale);
else if constexpr (std::is_same_v<FromFieldType, UInt64>)
return convertDecimals<DataTypeDecimal<Decimal128>, ToDataType>(value, 0, scale);
else

View File

@ -39,7 +39,7 @@ using DataTypeFloat32 = DataTypeNumber<Float32>;
using DataTypeFloat64 = DataTypeNumber<Float64>;
using DataTypeInt128 = DataTypeNumber<Int128>;
using DataTypeUInt256 = DataTypeNumber<bUInt256>;
using DataTypeInt256 = DataTypeNumber<bInt256>;
using DataTypeUInt256 = DataTypeNumber<UInt256>;
using DataTypeInt256 = DataTypeNumber<Int256>;
}

View File

@ -124,12 +124,12 @@ DataTypePtr FieldToDataType::operator() (const AggregateFunctionStateData & x) c
return DataTypeFactory::instance().get(name);
}
DataTypePtr FieldToDataType::operator() (const bUInt256 &) const
DataTypePtr FieldToDataType::operator() (const UInt256 &) const
{
throw Exception("There are no UInt256 literals in SQL", ErrorCodes::NOT_IMPLEMENTED);
}
DataTypePtr FieldToDataType::operator() (const bInt256 &) const
DataTypePtr FieldToDataType::operator() (const Int256 &) const
{
throw Exception("There are no Int256 literals in SQL", ErrorCodes::NOT_IMPLEMENTED);
}

View File

@ -31,8 +31,8 @@ public:
DataTypePtr operator() (const DecimalField<Decimal128> & x) const;
DataTypePtr operator() (const DecimalField<Decimal256> & x) const;
DataTypePtr operator() (const AggregateFunctionStateData & x) const;
DataTypePtr operator() (const bUInt256 & x) const;
DataTypePtr operator() (const bInt256 & x) const;
DataTypePtr operator() (const UInt256 & x) const;
DataTypePtr operator() (const Int256 & x) const;
};
}

View File

@ -482,8 +482,8 @@ struct WhichDataType
bool isUInt32() const { return idx == TypeIndex::UInt32; }
bool isUInt64() const { return idx == TypeIndex::UInt64; }
bool isUInt128() const { return idx == TypeIndex::UInt128; }
bool isbUInt256() const { return idx == TypeIndex::bUInt256; }
bool isUInt() const { return isUInt8() || isUInt16() || isUInt32() || isUInt64() || isUInt128() || isbUInt256(); }
bool isUInt256() const { return idx == TypeIndex::UInt256; }
bool isUInt() const { return isUInt8() || isUInt16() || isUInt32() || isUInt64() || isUInt128() || isUInt256(); }
bool isNativeUInt() const { return isUInt8() || isUInt16() || isUInt32() || isUInt64(); }
bool isInt8() const { return idx == TypeIndex::Int8; }
@ -491,8 +491,8 @@ struct WhichDataType
bool isInt32() const { return idx == TypeIndex::Int32; }
bool isInt64() const { return idx == TypeIndex::Int64; }
bool isInt128() const { return idx == TypeIndex::Int128; }
bool isbInt256() const { return idx == TypeIndex::bInt256; }
bool isInt() const { return isInt8() || isInt16() || isInt32() || isInt64() || isInt128() || isbInt256(); }
bool isInt256() const { return idx == TypeIndex::Int256; }
bool isInt() const { return isInt8() || isInt16() || isInt32() || isInt64() || isInt128() || isInt256(); }
bool isNativeInt() const { return isInt8() || isInt16() || isInt32() || isInt64(); }
bool isDecimal32() const { return idx == TypeIndex::Decimal32; }
@ -529,7 +529,7 @@ struct WhichDataType
bool isFunction() const { return idx == TypeIndex::Function; }
bool isAggregateFunction() const { return idx == TypeIndex::AggregateFunction; }
bool IsBigIntOrDeimal() const { return isInt128() || isbInt256() || isbUInt256() || isDecimal256(); }
bool IsBigIntOrDeimal() const { return isInt128() || isInt256() || isUInt256() || isDecimal256(); }
};
/// IDataType helpers (alternative for IDataType virtual methods with single point of truth)

View File

@ -55,9 +55,9 @@ template <> struct Construct<false, false, 1> { using Type = UInt8; };
template <> struct Construct<false, false, 2> { using Type = UInt16; };
template <> struct Construct<false, false, 4> { using Type = UInt32; };
template <> struct Construct<false, false, 8> { using Type = UInt64; };
template <> struct Construct<false, false, 16> { using Type = bUInt256; };
template <> struct Construct<false, false, 32> { using Type = bUInt256; };
template <> struct Construct<false, false, 48> { using Type = bUInt256; };
template <> struct Construct<false, false, 16> { using Type = UInt256; };
template <> struct Construct<false, false, 32> { using Type = UInt256; };
template <> struct Construct<false, false, 48> { using Type = UInt256; };
template <> struct Construct<false, true, 1> { using Type = Float32; };
template <> struct Construct<false, true, 2> { using Type = Float32; };
template <> struct Construct<false, true, 4> { using Type = Float32; };
@ -68,7 +68,7 @@ template <> struct Construct<true, false, 4> { using Type = Int32; };
template <> struct Construct<true, false, 8> { using Type = Int64; };
template <> struct Construct<true, false, 16> { using Type = Int128; };
template <> struct Construct<true, false, 32> { using Type = Int128; };
template <> struct Construct<true, false, 48> { using Type = bInt256; };
template <> struct Construct<true, false, 48> { using Type = Int256; };
template <> struct Construct<true, true, 1> { using Type = Float32; };
template <> struct Construct<true, true, 2> { using Type = Float32; };
template <> struct Construct<true, true, 4> { using Type = Float32; };
@ -228,10 +228,7 @@ using ResultOfGreatest = std::conditional_t<LeastGreatestSpecialCase<A, B>,
template <typename T>
static inline auto littleBits(const T & x)
{
if constexpr (is_big_int_v<T>)
return x. template convert_to<UInt32>();
else
return UInt8(x);
return bigint_cast<UInt8>(x);
}
}

View File

@ -456,8 +456,8 @@ public:
}
bool readInt128(Int128 &) override { cannotConvertType("Int128"); }
bool readbInt256(bInt256 &) override { cannotConvertType("Int256"); }
bool readbUInt256(bUInt256 &) override { cannotConvertType("UInt256"); }
bool readInt256(Int256 &) override { cannotConvertType("Int256"); }
bool readUInt256(UInt256 &) override { cannotConvertType("UInt256"); }
bool readFloat32(Float32 &) override
{

View File

@ -62,8 +62,8 @@ public:
bool readNumber(UInt64 & value) { return current_converter->readUInt64(value); }
bool readNumber(Int128 & value) { return current_converter->readInt128(value); }
bool readNumber(UInt128 & value) { return current_converter->readUInt128(value); }
bool readNumber(bInt256 & value) { return current_converter->readbInt256(value); }
bool readNumber(bUInt256 & value) { return current_converter->readbUInt256(value); }
bool readNumber(Int256 & value) { return current_converter->readInt256(value); }
bool readNumber(UInt256 & value) { return current_converter->readUInt256(value); }
bool readNumber(Float32 & value) { return current_converter->readFloat32(value); }
bool readNumber(Float64 & value) { return current_converter->readFloat64(value); }
@ -152,8 +152,8 @@ private:
virtual bool readInt128(Int128 &) = 0;
virtual bool readUInt128(UInt128 &) = 0;
virtual bool readbInt256(bInt256 &) = 0;
virtual bool readbUInt256(bUInt256 &) = 0;
virtual bool readInt256(Int256 &) = 0;
virtual bool readUInt256(UInt256 &) = 0;
virtual bool readFloat32(Float32 &) = 0;
virtual bool readFloat64(Float64 &) = 0;
@ -232,8 +232,8 @@ public:
bool readNumber(UInt64 &) { return false; }
bool readNumber(Int128 &) { return false; }
bool readNumber(UInt128 &) { return false; }
bool readNumber(bInt256 &) { return false; }
bool readNumber(bUInt256 &) { return false; }
bool readNumber(Int256 &) { return false; }
bool readNumber(UInt256 &) { return false; }
bool readNumber(Float32 &) { return false; }
bool readNumber(Float64 &) { return false; }
bool readStringInto(PaddedPODArray<UInt8> &) { return false; }

View File

@ -319,8 +319,8 @@ public:
virtual void writeUInt64(UInt64) override { cannotConvertType("UInt64"); }
virtual void writeInt128(Int128) override { cannotConvertType("Int128"); }
virtual void writeUInt128(const UInt128 &) override { cannotConvertType("UInt128"); }
virtual void writebInt256(const bInt256 &) override { cannotConvertType("Int256"); }
virtual void writebUInt256(const bUInt256 &) override { cannotConvertType("UInt256"); }
virtual void writeInt256(const Int256 &) override { cannotConvertType("Int256"); }
virtual void writeUInt256(const UInt256 &) override { cannotConvertType("UInt256"); }
virtual void writeFloat32(Float32) override { cannotConvertType("Float32"); }
virtual void writeFloat64(Float64) override { cannotConvertType("Float64"); }
virtual void prepareEnumMapping8(const std::vector<std::pair<std::string, Int8>> &) override {}

View File

@ -65,8 +65,8 @@ public:
bool writeNumber(Int128 value) { return writeValueIfPossible(&IConverter::writeInt128, value); }
bool writeNumber(UInt128 value) { return writeValueIfPossible(&IConverter::writeUInt128, value); }
bool writeNumber(bInt256 value) { return writeValueIfPossible(&IConverter::writebInt256, value); }
bool writeNumber(bUInt256 value) { return writeValueIfPossible(&IConverter::writebUInt256, value); }
bool writeNumber(Int256 value) { return writeValueIfPossible(&IConverter::writeInt256, value); }
bool writeNumber(UInt256 value) { return writeValueIfPossible(&IConverter::writeUInt256, value); }
bool writeNumber(Float32 value) { return writeValueIfPossible(&IConverter::writeFloat32, value); }
bool writeNumber(Float64 value) { return writeValueIfPossible(&IConverter::writeFloat64, value); }
@ -156,8 +156,8 @@ private:
virtual void writeInt128(Int128) = 0;
virtual void writeUInt128(const UInt128 &) = 0;
virtual void writebInt256(const bInt256 &) = 0;
virtual void writebUInt256(const bUInt256 &) = 0;
virtual void writeInt256(const Int256 &) = 0;
virtual void writeUInt256(const UInt256 &) = 0;
virtual void writeFloat32(Float32) = 0;
virtual void writeFloat64(Float64) = 0;
@ -267,8 +267,8 @@ public:
bool writeNumber(UInt64 /* value */) { return false; }
bool writeNumber(Int128 /* value */) { return false; }
bool writeNumber(UInt128 /* value */) { return false; }
bool writeNumber(bInt256 /* value */) { return false; }
bool writeNumber(bUInt256 /* value */) { return false; }
bool writeNumber(Int256 /* value */) { return false; }
bool writeNumber(UInt256 /* value */) { return false; }
bool writeNumber(Float32 /* value */) { return false; }
bool writeNumber(Float64 /* value */) { return false; }
bool writeString(const StringRef & /* value */) { return false; }

View File

@ -50,7 +50,11 @@ inline auto checkedDivision(A a, B b)
{
throwIfDivisionLeadsToFPE(a, b);
if constexpr (is_big_int_v<A> && is_big_int_v<B>)
if constexpr (is_big_int_v<A> && std::is_floating_point_v<B>)
return bigint_cast<B>(a) / b;
else if constexpr (is_big_int_v<B> && std::is_floating_point_v<A>)
return a / bigint_cast<A>(b);
else if constexpr (is_big_int_v<A> && is_big_int_v<B>)
return static_cast<A>(a / b);
else if constexpr (!is_big_int_v<A> && is_big_int_v<B>)
return static_cast<A>(B(a) / b);
@ -70,28 +74,18 @@ struct DivideIntegralImpl
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
if constexpr (is_big_int_v<A> && std::is_floating_point_v<B>)
return Result(static_cast<B>(a) / b);
else if constexpr (is_big_int_v<B> && std::is_floating_point_v<A>)
return a / static_cast<Result>(b);
else if constexpr (is_big_int_v<A> && std::is_same_v<B, UInt8>)
return static_cast<Result>(checkedDivision(make_signed_t<A>(a), Int16(b)));
else if constexpr (is_big_int_v<B> && std::is_same_v<A, UInt8>)
return checkedDivision(Int16(a), make_signed_t<B>(b));
using CastA = std::conditional_t<is_big_int_v<B> && std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB = std::conditional_t<is_big_int_v<A> && std::is_same_v<B, UInt8>, uint8_t, B>;
/// Otherwise overflow may occur due to integer promotion. Example: int8_t(-1) / uint64_t(2).
/// NOTE: overflow is still possible when dividing large signed number to large unsigned number or vice-versa. But it's less harmful.
else if constexpr (is_signed_v<A> || is_signed_v<B>)
if constexpr (is_integer_v<A> && is_integer_v<B> && (is_signed_v<A> || is_signed_v<B>))
{
if constexpr (is_integer_v<A> && is_integer_v<B>)
{
return checkedDivision(make_signed_t<A>(a),
sizeof(A) > sizeof(B) ? make_signed_t<A>(b) : make_signed_t<B>(b));
}
else
return checkedDivision(a, b);
return checkedDivision(make_signed_t<CastA>(a),
sizeof(A) > sizeof(B) ? make_signed_t<A>(CastB(b)) : make_signed_t<CastB>(b));
}
else
return checkedDivision(a, b);
return bigint_cast<Result>(checkedDivision(CastA(a), CastB(b)));
}
#if USE_EMBEDDED_COMPILER
@ -99,19 +93,6 @@ struct DivideIntegralImpl
#endif
};
template <typename Result, typename A, typename B>
inline Result applyBigIntModulo(A a, B b)
{
if constexpr (std::is_same_v<A, UInt8>)
return UInt16(a) % b;
else if constexpr (std::is_same_v<B, UInt8>)
return static_cast<UInt16>(a % UInt16(b));
else if constexpr (sizeof(A) > sizeof(B))
return static_cast<Result>(a % A(b));
else
return static_cast<Result>(B(a) % b);
}
template <typename A, typename B>
struct ModuloImpl
{
@ -120,7 +101,6 @@ struct ModuloImpl
using IntegerBType = typename NumberTraits::ToInteger<B>::Type;
static const constexpr bool allow_fixed_string = false;
static const constexpr bool is_special = is_big_int_v<IntegerAType> || is_big_int_v<IntegerBType>;
template <typename Result = ResultType>
static inline Result apply(A a, B b)
@ -134,8 +114,19 @@ struct ModuloImpl
{
throwIfDivisionLeadsToFPE(IntegerAType(a), IntegerBType(b));
if constexpr (is_special)
return applyBigIntModulo<Result>(IntegerAType(a), IntegerBType(b));
if constexpr (is_big_int_v<IntegerAType> || is_big_int_v<IntegerBType>)
{
using CastA = std::conditional_t<std::is_same_v<IntegerAType, UInt8>, uint8_t, IntegerAType>;
using CastB = std::conditional_t<std::is_same_v<IntegerBType, UInt8>, uint8_t, IntegerBType>;
CastA int_a(a);
CastB int_b(b);
if constexpr (is_big_int_v<IntegerBType> && sizeof(IntegerAType) <= sizeof(IntegerBType))
return bigint_cast<Result>(bigint_cast<CastB>(int_a) % int_b);
else
return bigint_cast<Result>(int_a % int_b);
}
else
return IntegerAType(a) % IntegerBType(b);
}

View File

@ -634,13 +634,13 @@ private:
|| executeNumRightType<T0, UInt32>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, UInt64>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, UInt128>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, bUInt256>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, UInt256>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Int8>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Int16>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Int32>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Int64>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Int128>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, bInt256>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Int256>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Float32>(block, result, col_left, col_right_untyped)
|| executeNumRightType<T0, Float64>(block, result, col_left, col_right_untyped))
return true;
@ -656,13 +656,13 @@ private:
|| executeNumConstRightType<T0, UInt32>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, UInt64>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, UInt128>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, bUInt256>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, UInt256>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Int8>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Int16>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Int32>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Int64>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Int128>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, bInt256>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Int256>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Float32>(block, result, col_left_const, col_right_untyped)
|| executeNumConstRightType<T0, Float64>(block, result, col_left_const, col_right_untyped))
return true;
@ -1223,13 +1223,13 @@ public:
|| executeNumLeftType<UInt32>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<UInt64>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<UInt128>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<bUInt256>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<UInt256>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Int8>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Int16>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Int32>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Int64>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Int128>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<bInt256>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Int256>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Float32>(block, result, col_left_untyped, col_right_untyped)
|| executeNumLeftType<Float64>(block, result, col_left_untyped, col_right_untyped)))
throw Exception("Illegal column " + col_left_untyped->getName()

View File

@ -150,13 +150,10 @@ struct ConvertImpl
}
else if constexpr (is_big_int_v<FromFieldType> || is_big_int_v<ToFieldType>)
{
using CastFrom = std::conditional_t<std::is_same_v<FromFieldType, UInt8>, uint8_t, FromFieldType>;
using CastTo = std::conditional_t<std::is_same_v<ToFieldType, UInt8>, uint8_t, ToFieldType>;
if constexpr (std::is_same_v<FromFieldType, UInt128> || std::is_same_v<ToFieldType, UInt128>)
throw Exception("Unexpected UInt128 to big int conversion", ErrorCodes::NOT_IMPLEMENTED);
else
vec_to[i] = static_cast<CastTo>(static_cast<CastFrom>(vec_from[i]));
vec_to[i] = bigint_cast<ToFieldType>(vec_from[i]);
}
else if constexpr (std::is_same_v<ToFieldType, UInt128> && sizeof(FromFieldType) <= sizeof(UInt64))
vec_to[i] = static_cast<ToFieldType>(static_cast<UInt64>(vec_from[i]));
@ -1630,13 +1627,13 @@ using FunctionToUInt8 = FunctionConvert<DataTypeUInt8, NameToUInt8, ToNumberMono
using FunctionToUInt16 = FunctionConvert<DataTypeUInt16, NameToUInt16, ToNumberMonotonicity<UInt16>>;
using FunctionToUInt32 = FunctionConvert<DataTypeUInt32, NameToUInt32, ToNumberMonotonicity<UInt32>>;
using FunctionToUInt64 = FunctionConvert<DataTypeUInt64, NameToUInt64, ToNumberMonotonicity<UInt64>>;
using FunctionToUInt256 = FunctionConvert<DataTypeUInt256, NameToUInt256, ToNumberMonotonicity<bUInt256>>;
using FunctionToUInt256 = FunctionConvert<DataTypeUInt256, NameToUInt256, ToNumberMonotonicity<UInt256>>;
using FunctionToInt8 = FunctionConvert<DataTypeInt8, NameToInt8, ToNumberMonotonicity<Int8>>;
using FunctionToInt16 = FunctionConvert<DataTypeInt16, NameToInt16, ToNumberMonotonicity<Int16>>;
using FunctionToInt32 = FunctionConvert<DataTypeInt32, NameToInt32, ToNumberMonotonicity<Int32>>;
using FunctionToInt64 = FunctionConvert<DataTypeInt64, NameToInt64, ToNumberMonotonicity<Int64>>;
using FunctionToInt128 = FunctionConvert<DataTypeInt128, NameToInt128, ToNumberMonotonicity<Int128>>;
using FunctionToInt256 = FunctionConvert<DataTypeInt256, NameToInt256, ToNumberMonotonicity<bInt256>>;
using FunctionToInt256 = FunctionConvert<DataTypeInt256, NameToInt256, ToNumberMonotonicity<Int256>>;
using FunctionToFloat32 = FunctionConvert<DataTypeFloat32, NameToFloat32, ToNumberMonotonicity<Float32>>;
using FunctionToFloat64 = FunctionConvert<DataTypeFloat64, NameToFloat64, ToNumberMonotonicity<Float64>>;
using FunctionToDate = FunctionConvert<DataTypeDate, NameToDate, ToDateMonotonicity>;

View File

@ -968,13 +968,13 @@ private:
else if (which.isUInt32()) executeIntType<UInt32, first>(icolumn, vec_to);
else if (which.isUInt64()) executeIntType<UInt64, first>(icolumn, vec_to);
else if (which.isUInt128() || which.isUUID()) executeBigIntType<UInt128, first>(icolumn, vec_to);
else if (which.isbUInt256()) executeBigIntType<bUInt256, first>(icolumn, vec_to);
else if (which.isUInt256()) executeBigIntType<UInt256, first>(icolumn, vec_to);
else if (which.isInt8()) executeIntType<Int8, first>(icolumn, vec_to);
else if (which.isInt16()) executeIntType<Int16, first>(icolumn, vec_to);
else if (which.isInt32()) executeIntType<Int32, first>(icolumn, vec_to);
else if (which.isInt64()) executeIntType<Int64, first>(icolumn, vec_to);
else if (which.isInt128()) executeBigIntType<Int128, first>(icolumn, vec_to);
else if (which.isbInt256()) executeBigIntType<bInt256, first>(icolumn, vec_to);
else if (which.isInt256()) executeBigIntType<Int256, first>(icolumn, vec_to);
else if (which.isEnum8()) executeIntType<Int8, first>(icolumn, vec_to);
else if (which.isEnum16()) executeIntType<Int16, first>(icolumn, vec_to);
else if (which.isDate()) executeIntType<UInt16, first>(icolumn, vec_to);

View File

@ -45,21 +45,14 @@ void writeSlice(const NumericArraySlice<T> & slice, NumericArraySink<U> & sink)
if constexpr (OverBigInt<T> || OverBigInt<U>)
{
if constexpr (std::is_same_v<T, UInt8> || std::is_same_v<U, UInt8>)
{
if constexpr (IsDecimalNumber<T>)
dst = static_cast<NativeU>(static_cast<UInt16>(src.value));
else
dst = static_cast<NativeU>(static_cast<UInt16>(src));
}
else if constexpr (std::is_same_v<U, UInt128>)
if constexpr (std::is_same_v<U, UInt128>)
{
throw Exception("No conversion between UInt128 and " + demangle(typeid(T).name()), ErrorCodes::NOT_IMPLEMENTED);
}
else if constexpr (IsDecimalNumber<T>)
dst = static_cast<NativeU>(src.value);
dst = bigint_cast<NativeU>(src.value);
else
dst = static_cast<NativeU>(src);
dst = bigint_cast<NativeU>(src);
}
else
dst = static_cast<NativeU>(src);

View File

@ -14,20 +14,11 @@ struct BitAndImpl
{
using ResultType = typename NumberTraits::ResultOfBit<A, B>::Type;
static constexpr const bool allow_fixed_string = true;
static constexpr bool need_uint8_cast = is_big_int_v<ResultType> && (std::is_same_v<A, UInt8> || std::is_same_v<B, UInt8>);
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
if constexpr (need_uint8_cast)
{
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
return static_cast<Result>(static_cast<CastA>(a)) & static_cast<Result>(static_cast<CastB>(b));
}
else
return static_cast<Result>(a) & static_cast<Result>(b);
return bigint_cast<Result>(a) & bigint_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER

View File

@ -13,20 +13,11 @@ struct BitOrImpl
{
using ResultType = typename NumberTraits::ResultOfBit<A, B>::Type;
static constexpr const bool allow_fixed_string = true;
static constexpr bool need_uint8_cast = is_big_int_v<ResultType> && (std::is_same_v<A, UInt8> || std::is_same_v<B, UInt8>);
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
if constexpr (need_uint8_cast)
{
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
return static_cast<Result>(static_cast<CastA>(a)) | static_cast<Result>(static_cast<CastB>(b));
}
else
return static_cast<Result>(a) | static_cast<Result>(b);
return bigint_cast<Result>(a) | bigint_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER

View File

@ -18,12 +18,10 @@ struct BitShiftLeftImpl
template <typename Result = ResultType>
static inline NO_SANITIZE_UNDEFINED Result apply(A a [[maybe_unused]], B b [[maybe_unused]])
{
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
if constexpr (is_big_int_v<B>)
throw Exception("BitShiftLeftImpl is not implemented for big integers as second argument", ErrorCodes::NOT_IMPLEMENTED);
else if constexpr (is_big_int_v<ResultType>)
return static_cast<Result>(a) << static_cast<CastB>(b);
else if constexpr (is_big_int_v<A>)
return static_cast<Result>(a) << bigint_cast<UInt32>(b);
else
return static_cast<Result>(a) << static_cast<Result>(b);
}

View File

@ -18,12 +18,10 @@ struct BitShiftRightImpl
template <typename Result = ResultType>
static inline NO_SANITIZE_UNDEFINED Result apply(A a [[maybe_unused]], B b [[maybe_unused]])
{
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
if constexpr (is_big_int_v<B>)
throw Exception("BitRotate is not implemented for big integers as second argument", ErrorCodes::NOT_IMPLEMENTED);
else if constexpr (is_big_int_v<ResultType>)
return static_cast<Result>(a) >> static_cast<CastB>(b);
else if constexpr (is_big_int_v<A>)
return static_cast<Result>(a) >> bigint_cast<UInt32>(b);
else
return static_cast<Result>(a) >> static_cast<Result>(b);
}

View File

@ -13,20 +13,11 @@ struct BitXorImpl
{
using ResultType = typename NumberTraits::ResultOfBit<A, B>::Type;
static constexpr bool allow_fixed_string = true;
static constexpr bool need_uint8_cast = is_big_int_v<ResultType> && (std::is_same_v<A, UInt8> || std::is_same_v<B, UInt8>);
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
if constexpr (need_uint8_cast)
{
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
return static_cast<Result>(static_cast<CastA>(a)) ^ static_cast<Result>(static_cast<CastB>(b));
}
else
return static_cast<Result>(a) ^ static_cast<Result>(b);
return bigint_cast<Result>(a) ^ bigint_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER

View File

@ -12,22 +12,12 @@ struct GreatestBaseImpl
{
using ResultType = NumberTraits::ResultOfGreatest<A, B>;
static const constexpr bool allow_fixed_string = false;
static constexpr bool need_uint8_cast = is_big_int_v<ResultType> && (std::is_same_v<A, UInt8> || std::is_same_v<B, UInt8>);
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
if constexpr (need_uint8_cast)
{
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
return static_cast<Result>(static_cast<CastA>(a)) > static_cast<Result>(static_cast<CastB>(b)) ?
static_cast<Result>(static_cast<CastA>(a)) : static_cast<Result>(static_cast<CastB>(b));
}
else
return static_cast<Result>(a) > static_cast<Result>(b) ?
static_cast<Result>(a) : static_cast<Result>(b);
return bigint_cast<Result>(a) > bigint_cast<Result>(b) ?
bigint_cast<Result>(a) : bigint_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER

View File

@ -44,15 +44,6 @@ using namespace GatherUtils;
* then, else - numeric types for which there is a general type, or dates, datetimes, or strings, or arrays of these types.
*/
template <typename From, typename To>
inline To special_cast(From x)
{
if constexpr (is_big_int_v<To> && std::is_same_v<From, UInt8>)
return static_cast<To>(static_cast<UInt16>(x));
else
return static_cast<To>(x);
}
template <typename A, typename B, typename ResultType>
struct NumIfImpl
{
@ -68,7 +59,7 @@ struct NumIfImpl
typename ColVecResult::Container & res = col_res->getData();
for (size_t i = 0; i < size; ++i)
res[i] = cond[i] ? special_cast<A, ResultType>(a[i]) : special_cast<B, ResultType>(b[i]);
res[i] = cond[i] ? bigint_cast<ResultType>(a[i]) : bigint_cast<ResultType>(b[i]);
block.getByPosition(result).column = std::move(col_res);
}
@ -79,7 +70,7 @@ struct NumIfImpl
typename ColVecResult::Container & res = col_res->getData();
for (size_t i = 0; i < size; ++i)
res[i] = cond[i] ? special_cast<A, ResultType>(a[i]) : special_cast<B, ResultType>(b);
res[i] = cond[i] ? bigint_cast<ResultType>(a[i]) : bigint_cast<ResultType>(b);
block.getByPosition(result).column = std::move(col_res);
}
@ -90,7 +81,7 @@ struct NumIfImpl
typename ColVecResult::Container & res = col_res->getData();
for (size_t i = 0; i < size; ++i)
res[i] = cond[i] ? special_cast<A, ResultType>(a) : special_cast<B, ResultType>(b[i]);
res[i] = cond[i] ? bigint_cast<ResultType>(a) : bigint_cast<ResultType>(b[i]);
block.getByPosition(result).column = std::move(col_res);
}
@ -101,7 +92,7 @@ struct NumIfImpl
typename ColVecResult::Container & res = col_res->getData();
for (size_t i = 0; i < size; ++i)
res[i] = cond[i] ? special_cast<A, ResultType>(a) : special_cast<B, ResultType>(b);
res[i] = cond[i] ? bigint_cast<ResultType>(a) : bigint_cast<ResultType>(b);
block.getByPosition(result).column = std::move(col_res);
}
};

View File

@ -12,22 +12,12 @@ struct LeastBaseImpl
{
using ResultType = NumberTraits::ResultOfLeast<A, B>;
static const constexpr bool allow_fixed_string = false;
static constexpr bool need_uint8_cast = is_big_int_v<ResultType> && (std::is_same_v<A, UInt8> || std::is_same_v<B, UInt8>);
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
if constexpr (need_uint8_cast)
{
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, A>;
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, B>;
return static_cast<Result>(static_cast<CastA>(a)) < static_cast<Result>(static_cast<CastB>(b)) ?
static_cast<Result>(static_cast<CastA>(a)) : static_cast<Result>(static_cast<CastB>(b));
}
else
/** gcc 4.9.2 successfully vectorizes a loop from this function. */
return static_cast<Result>(a) < static_cast<Result>(b) ? static_cast<Result>(a) : static_cast<Result>(b);
/** gcc 4.9.2 successfully vectorizes a loop from this function. */
return bigint_cast<Result>(a) < bigint_cast<Result>(b) ? bigint_cast<Result>(a) : bigint_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER

View File

@ -17,10 +17,10 @@ struct MinusImpl
{
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
{
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, std::conditional_t<std::is_floating_point_v<B>, B, A>>;
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, std::conditional_t<std::is_floating_point_v<A>, A, B>>;
using CastA = std::conditional_t<std::is_floating_point_v<B>, B, A>;
using CastB = std::conditional_t<std::is_floating_point_v<A>, A, B>;
return static_cast<Result>(static_cast<CastA>(a)) - static_cast<Result>(static_cast<CastB>(b));
return bigint_cast<Result>(bigint_cast<CastA>(a)) - bigint_cast<Result>(bigint_cast<CastB>(b));
}
else
return static_cast<Result>(a) - b;

View File

@ -17,10 +17,10 @@ struct MultiplyImpl
{
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
{
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, std::conditional_t<std::is_floating_point_v<B>, B, A>>;
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, std::conditional_t<std::is_floating_point_v<A>, A, B>>;
using CastA = std::conditional_t<std::is_floating_point_v<B>, B, A>;
using CastB = std::conditional_t<std::is_floating_point_v<A>, A, B>;
return static_cast<Result>(static_cast<CastA>(a)) * static_cast<Result>(static_cast<CastB>(b));
return bigint_cast<Result>(bigint_cast<CastA>(a)) * bigint_cast<Result>(bigint_cast<CastB>(b));
}
else
return static_cast<Result>(a) * b;

View File

@ -18,10 +18,10 @@ struct PlusImpl
/// Next everywhere, static_cast - so that there is no wrong result in expressions of the form Int64 c = UInt32(a) * Int32(-1).
if constexpr (is_big_int_v<A> || is_big_int_v<B>)
{
using CastA = std::conditional_t<std::is_same_v<A, UInt8>, uint8_t, std::conditional_t<std::is_floating_point_v<B>, B, A>>;
using CastB = std::conditional_t<std::is_same_v<B, UInt8>, uint8_t, std::conditional_t<std::is_floating_point_v<A>, A, B>>;
using CastA = std::conditional_t<std::is_floating_point_v<B>, B, A>;
using CastB = std::conditional_t<std::is_floating_point_v<A>, A, B>;
return static_cast<Result>(static_cast<CastA>(a)) + static_cast<Result>(static_cast<CastB>(b));
return bigint_cast<Result>(bigint_cast<CastA>(a)) + bigint_cast<Result>(bigint_cast<CastB>(b));
}
else
return static_cast<Result>(a) + b;

View File

@ -13,169 +13,169 @@ static const std::map<std::pair<std::string, std::string>, std::string> answer =
{{"UInt8", "UInt16"}, "UInt16"},
{{"UInt8", "UInt32"}, "UInt32"},
{{"UInt8", "UInt64"}, "UInt64"},
{{"UInt8", "bUInt256"}, "bUInt256"},
{{"UInt8", "UInt256"}, "UInt256"},
{{"UInt8", "Int8"}, "Int16"},
{{"UInt8", "Int16"}, "Int16"},
{{"UInt8", "Int32"}, "Int32"},
{{"UInt8", "Int64"}, "Int64"},
{{"UInt8", "Int128"}, "Int128"},
{{"UInt8", "bInt256"}, "bInt256"},
{{"UInt8", "Int256"}, "Int256"},
{{"UInt8", "Float32"}, "Float32"},
{{"UInt8", "Float64"}, "Float64"},
{{"UInt16", "UInt8"}, "UInt16"},
{{"UInt16", "UInt16"}, "UInt16"},
{{"UInt16", "UInt32"}, "UInt32"},
{{"UInt16", "UInt64"}, "UInt64"},
{{"UInt16", "bUInt256"}, "bUInt256"},
{{"UInt16", "UInt256"}, "UInt256"},
{{"UInt16", "Int8"}, "Int32"},
{{"UInt16", "Int16"}, "Int32"},
{{"UInt16", "Int32"}, "Int32"},
{{"UInt16", "Int64"}, "Int64"},
{{"UInt16", "Int128"}, "Int128"},
{{"UInt16", "bInt256"}, "bInt256"},
{{"UInt16", "Int256"}, "Int256"},
{{"UInt16", "Float32"}, "Float32"},
{{"UInt16", "Float64"}, "Float64"},
{{"UInt32", "UInt8"}, "UInt32"},
{{"UInt32", "UInt16"}, "UInt32"},
{{"UInt32", "UInt32"}, "UInt32"},
{{"UInt32", "UInt64"}, "UInt64"},
{{"UInt32", "bUInt256"}, "bUInt256"},
{{"UInt32", "UInt256"}, "UInt256"},
{{"UInt32", "Int8"}, "Int64"},
{{"UInt32", "Int16"}, "Int64"},
{{"UInt32", "Int32"}, "Int64"},
{{"UInt32", "Int64"}, "Int64"},
{{"UInt32", "Int128"}, "Int128"},
{{"UInt32", "bInt256"}, "bInt256"},
{{"UInt32", "Int256"}, "Int256"},
{{"UInt32", "Float32"}, "Float64"},
{{"UInt32", "Float64"}, "Float64"},
{{"UInt64", "UInt8"}, "UInt64"},
{{"UInt64", "UInt16"}, "UInt64"},
{{"UInt64", "UInt32"}, "UInt64"},
{{"UInt64", "UInt64"}, "UInt64"},
{{"UInt64", "bUInt256"}, "bUInt256"},
{{"UInt64", "UInt256"}, "UInt256"},
{{"UInt64", "Int8"}, "Int128"},
{{"UInt64", "Int16"}, "Int128"},
{{"UInt64", "Int32"}, "Int128"},
{{"UInt64", "Int64"}, "Int128"},
{{"UInt64", "Int128"}, "Int128"},
{{"UInt64", "bInt256"}, "bInt256"},
{{"UInt64", "Int256"}, "Int256"},
{{"UInt64", "Float32"}, "Error"},
{{"UInt64", "Float64"}, "Error"},
{{"bUInt256", "UInt8"}, "bUInt256"},
{{"bUInt256", "UInt16"}, "bUInt256"},
{{"bUInt256", "UInt32"}, "bUInt256"},
{{"bUInt256", "UInt64"}, "bUInt256"},
{{"bUInt256", "bUInt256"}, "bUInt256"},
{{"bUInt256", "Int8"}, "Error"},
{{"bUInt256", "Int16"}, "Error"},
{{"bUInt256", "Int32"}, "Error"},
{{"bUInt256", "Int64"}, "Error"},
{{"bUInt256", "Int128"}, "Error"},
{{"bUInt256", "bInt256"}, "Error"},
{{"bUInt256", "Float32"}, "Error"},
{{"bUInt256", "Float64"}, "Error"},
{{"UInt256", "UInt8"}, "UInt256"},
{{"UInt256", "UInt16"}, "UInt256"},
{{"UInt256", "UInt32"}, "UInt256"},
{{"UInt256", "UInt64"}, "UInt256"},
{{"UInt256", "UInt256"}, "UInt256"},
{{"UInt256", "Int8"}, "Error"},
{{"UInt256", "Int16"}, "Error"},
{{"UInt256", "Int32"}, "Error"},
{{"UInt256", "Int64"}, "Error"},
{{"UInt256", "Int128"}, "Error"},
{{"UInt256", "Int256"}, "Error"},
{{"UInt256", "Float32"}, "Error"},
{{"UInt256", "Float64"}, "Error"},
{{"Int8", "UInt8"}, "Int16"},
{{"Int8", "UInt16"}, "Int32"},
{{"Int8", "UInt32"}, "Int64"},
{{"Int8", "UInt64"}, "Int128"},
{{"Int8", "bUInt256"}, "Error"},
{{"Int8", "UInt256"}, "Error"},
{{"Int8", "Int8"}, "Int8"},
{{"Int8", "Int16"}, "Int16"},
{{"Int8", "Int32"}, "Int32"},
{{"Int8", "Int64"}, "Int64"},
{{"Int8", "Int128"}, "Int128"},
{{"Int8", "bInt256"}, "bInt256"},
{{"Int8", "Int256"}, "Int256"},
{{"Int8", "Float32"}, "Float32"},
{{"Int8", "Float64"}, "Float64"},
{{"Int16", "UInt8"}, "Int16"},
{{"Int16", "UInt16"}, "Int32"},
{{"Int16", "UInt32"}, "Int64"},
{{"Int16", "UInt64"}, "Int128"},
{{"Int16", "bUInt256"}, "Error"},
{{"Int16", "UInt256"}, "Error"},
{{"Int16", "Int8"}, "Int16"},
{{"Int16", "Int16"}, "Int16"},
{{"Int16", "Int32"}, "Int32"},
{{"Int16", "Int64"}, "Int64"},
{{"Int16", "Int128"}, "Int128"},
{{"Int16", "bInt256"}, "bInt256"},
{{"Int16", "Int256"}, "Int256"},
{{"Int16", "Float32"}, "Float32"},
{{"Int16", "Float64"}, "Float64"},
{{"Int32", "UInt8"}, "Int32"},
{{"Int32", "UInt16"}, "Int32"},
{{"Int32", "UInt32"}, "Int64"},
{{"Int32", "UInt64"}, "Int128"},
{{"Int32", "bUInt256"}, "Error"},
{{"Int32", "UInt256"}, "Error"},
{{"Int32", "Int8"}, "Int32"},
{{"Int32", "Int16"}, "Int32"},
{{"Int32", "Int32"}, "Int32"},
{{"Int32", "Int64"}, "Int64"},
{{"Int32", "Int128"}, "Int128"},
{{"Int32", "bInt256"}, "bInt256"},
{{"Int32", "Int256"}, "Int256"},
{{"Int32", "Float32"}, "Float64"},
{{"Int32", "Float64"}, "Float64"},
{{"Int64", "UInt8"}, "Int64"},
{{"Int64", "UInt16"}, "Int64"},
{{"Int64", "UInt32"}, "Int64"},
{{"Int64", "UInt64"}, "Int128"},
{{"Int64", "bUInt256"}, "Error"},
{{"Int64", "UInt256"}, "Error"},
{{"Int64", "Int8"}, "Int64"},
{{"Int64", "Int16"}, "Int64"},
{{"Int64", "Int32"}, "Int64"},
{{"Int64", "Int64"}, "Int64"},
{{"Int64", "Int128"}, "Int128"},
{{"Int64", "bInt256"}, "bInt256"},
{{"Int64", "Int256"}, "Int256"},
{{"Int64", "Float32"}, "Error"},
{{"Int64", "Float64"}, "Error"},
{{"Int128", "UInt8"}, "Int128"},
{{"Int128", "UInt16"}, "Int128"},
{{"Int128", "UInt32"}, "Int128"},
{{"Int128", "UInt64"}, "Int128"},
{{"Int128", "bUInt256"}, "Error"},
{{"Int128", "UInt256"}, "Error"},
{{"Int128", "Int8"}, "Int128"},
{{"Int128", "Int16"}, "Int128"},
{{"Int128", "Int32"}, "Int128"},
{{"Int128", "Int64"}, "Int128"},
{{"Int128", "Int128"}, "Int128"},
{{"Int128", "bInt256"}, "bInt256"},
{{"Int128", "Int256"}, "Int256"},
{{"Int128", "Float32"}, "Error"},
{{"Int128", "Float64"}, "Error"},
{{"bInt256", "UInt8"}, "bInt256"},
{{"bInt256", "UInt16"}, "bInt256"},
{{"bInt256", "UInt32"}, "bInt256"},
{{"bInt256", "UInt64"}, "bInt256"},
{{"bInt256", "bUInt256"}, "Error"},
{{"bInt256", "Int8"}, "bInt256"},
{{"bInt256", "Int16"}, "bInt256"},
{{"bInt256", "Int32"}, "bInt256"},
{{"bInt256", "Int64"}, "bInt256"},
{{"bInt256", "Int128"}, "bInt256"},
{{"bInt256", "bInt256"}, "bInt256"},
{{"bInt256", "Float32"}, "Error"},
{{"bInt256", "Float64"}, "Error"},
{{"Int256", "UInt8"}, "Int256"},
{{"Int256", "UInt16"}, "Int256"},
{{"Int256", "UInt32"}, "Int256"},
{{"Int256", "UInt64"}, "Int256"},
{{"Int256", "UInt256"}, "Error"},
{{"Int256", "Int8"}, "Int256"},
{{"Int256", "Int16"}, "Int256"},
{{"Int256", "Int32"}, "Int256"},
{{"Int256", "Int64"}, "Int256"},
{{"Int256", "Int128"}, "Int256"},
{{"Int256", "Int256"}, "Int256"},
{{"Int256", "Float32"}, "Error"},
{{"Int256", "Float64"}, "Error"},
{{"Float32", "UInt8"}, "Float32"},
{{"Float32", "UInt16"}, "Float32"},
{{"Float32", "UInt32"}, "Float64"},
{{"Float32", "UInt64"}, "Error"},
{{"Float32", "bUInt256"}, "Error"},
{{"Float32", "UInt256"}, "Error"},
{{"Float32", "Int8"}, "Float32"},
{{"Float32", "Int16"}, "Float32"},
{{"Float32", "Int32"}, "Float64"},
{{"Float32", "Int64"}, "Error"},
{{"Float32", "Int128"}, "Error"},
{{"Float32", "bInt256"}, "Error"},
{{"Float32", "Int256"}, "Error"},
{{"Float32", "Float32"}, "Float32"},
{{"Float32", "Float64"}, "Float64"},
{{"Float64", "UInt8"}, "Float64"},
{{"Float64", "UInt16"}, "Float64"},
{{"Float64", "UInt32"}, "Float64"},
{{"Float64", "UInt64"}, "Error"},
{{"Float64", "bUInt256"}, "Error"},
{{"Float64", "UInt256"}, "Error"},
{{"Float64", "Int8"}, "Float64"},
{{"Float64", "Int16"}, "Float64"},
{{"Float64", "Int32"}, "Float64"},
{{"Float64", "Int64"}, "Error"},
{{"Float64", "Int128"}, "Error"},
{{"Float64", "bInt256"}, "Error"},
{{"Float64", "Int256"}, "Error"},
{{"Float64", "Float32"}, "Float64"},
{{"Float64", "Float64"}, "Float64"}
};
@ -184,13 +184,13 @@ static std::string getTypeString(DB::UInt8) { return "UInt8"; }
static std::string getTypeString(DB::UInt16) { return "UInt16"; }
static std::string getTypeString(DB::UInt32) { return "UInt32"; }
static std::string getTypeString(DB::UInt64) { return "UInt64"; }
static std::string getTypeString(DB::bUInt256) { return "bUInt256"; }
static std::string getTypeString(DB::UInt256) { return "UInt256"; }
static std::string getTypeString(DB::Int8) { return "Int8"; }
static std::string getTypeString(DB::Int16) { return "Int16"; }
static std::string getTypeString(DB::Int32) { return "Int32"; }
static std::string getTypeString(DB::Int64) { return "Int64"; }
static std::string getTypeString(DB::Int128) { return "Int128"; }
static std::string getTypeString(DB::bInt256) { return "bInt256"; }
static std::string getTypeString(DB::Int256) { return "Int256"; }
static std::string getTypeString(DB::Float32) { return "Float32"; }
static std::string getTypeString(DB::Float64) { return "Float64"; }
static std::string getTypeString(DB::NumberTraits::Error) { return "Error"; }
@ -225,13 +225,13 @@ void ifLeftType()
ifRightType<T0, DB::UInt16>();
ifRightType<T0, DB::UInt32>();
ifRightType<T0, DB::UInt64>();
ifRightType<T0, DB::bUInt256>();
ifRightType<T0, DB::UInt256>();
ifRightType<T0, DB::Int8>();
ifRightType<T0, DB::Int16>();
ifRightType<T0, DB::Int32>();
ifRightType<T0, DB::Int64>();
ifRightType<T0, DB::Int128>();
ifRightType<T0, DB::bInt256>();
ifRightType<T0, DB::Int256>();
ifRightType<T0, DB::Float32>();
ifRightType<T0, DB::Float64>();
}
@ -268,13 +268,13 @@ TEST(NumberTraits, FunctionIf)
ifLeftType<DB::UInt16>();
ifLeftType<DB::UInt32>();
ifLeftType<DB::UInt64>();
ifLeftType<DB::bUInt256>();
ifLeftType<DB::UInt256>();
ifLeftType<DB::Int8>();
ifLeftType<DB::Int16>();
ifLeftType<DB::Int32>();
ifLeftType<DB::Int64>();
ifLeftType<DB::Int128>();
ifLeftType<DB::bInt256>();
ifLeftType<DB::Int256>();
ifLeftType<DB::Float32>();
ifLeftType<DB::Float64>();
}

View File

@ -801,15 +801,15 @@ readBinary(T & x, ReadBuffer & buf) { readPODBinary(x, buf); }
inline void readBinary(String & x, ReadBuffer & buf) { readStringBinary(x, buf); }
inline void readBinary(Int128 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
inline void readBinary(UInt128 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
inline void readBinary(UInt256 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
inline void readBinary(DummyUInt256 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
inline void readBinary(Decimal32 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
inline void readBinary(Decimal64 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
inline void readBinary(Decimal128 & x, ReadBuffer & buf) { readPODBinary(x, buf); }
inline void readBinary(Decimal256 & x, ReadBuffer & buf) { readBigIntBinary(x.value, buf); }
inline void readBinary(LocalDate & x, ReadBuffer & buf) { readPODBinary(x, buf); }
inline void readBinary(bUInt256 & x, ReadBuffer & buf) { readBigIntBinary(x, buf); }
inline void readBinary(bInt256 & x, ReadBuffer & buf) { readBigIntBinary(x, buf); }
inline void readBinary(UInt256 & x, ReadBuffer & buf) { readBigIntBinary(x, buf); }
inline void readBinary(Int256 & x, ReadBuffer & buf) { readBigIntBinary(x, buf); }
template <typename T>
inline std::enable_if_t<is_arithmetic_v<T> && (sizeof(T) <= 8), void>
@ -943,8 +943,8 @@ inline void readCSV(UUID & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
*/
throw Exception("UInt128 cannot be read as a text", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
inline void readCSV(bUInt256 & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
inline void readCSV(bInt256 & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
inline void readCSV(UInt256 & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
inline void readCSV(Int256 & x, ReadBuffer & buf) { readCSVSimple(x, buf); }
template <typename T>
void readBinary(std::vector<T> & x, ReadBuffer & buf)

View File

@ -801,7 +801,7 @@ inline void writeBinary(const String & x, WriteBuffer & buf) { writeStringBinary
inline void writeBinary(const StringRef & x, WriteBuffer & buf) { writeStringBinary(x, buf); }
inline void writeBinary(const Int128 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const UInt128 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const UInt256 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const DummyUInt256 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const Decimal32 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const Decimal64 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const Decimal128 & x, WriteBuffer & buf) { writePODBinary(x, buf); }
@ -809,8 +809,8 @@ inline void writeBinary(const Decimal256 & x, WriteBuffer & buf) { writeBigIntBi
inline void writeBinary(const LocalDate & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const LocalDateTime & x, WriteBuffer & buf) { writePODBinary(x, buf); }
inline void writeBinary(const bUInt256 & x, WriteBuffer & buf) { writeBigIntBinary(x, buf); }
inline void writeBinary(const bInt256 & x, WriteBuffer & buf) { writeBigIntBinary(x, buf); }
inline void writeBinary(const UInt256 & x, WriteBuffer & buf) { writeBigIntBinary(x, buf); }
inline void writeBinary(const Int256 & x, WriteBuffer & buf) { writeBigIntBinary(x, buf); }
/// Methods for outputting the value in text form for a tab-separated format.
template <typename T>
@ -835,13 +835,13 @@ inline void writeText(const LocalDate & x, WriteBuffer & buf) { writeDateText(x,
inline void writeText(const LocalDateTime & x, WriteBuffer & buf) { writeDateTimeText(x, buf); }
inline void writeText(const UUID & x, WriteBuffer & buf) { writeUUIDText(x, buf); }
inline void writeText(const UInt128 & x, WriteBuffer & buf) { writeText(UUID(x), buf); }
inline void writeText(const bUInt256 & x, WriteBuffer & buf) { writeText(x.str(), buf); }
inline void writeText(const bInt256 & x, WriteBuffer & buf) { writeText(x.str(), buf); }
inline void writeText(const UInt256 & x, WriteBuffer & buf) { writeText(bigintToString(x), buf); }
inline void writeText(const Int256 & x, WriteBuffer & buf) { writeText(bigintToString(x), buf); }
template <typename T>
String decimalFractional(const T & x, UInt32 scale)
{
if constexpr (std::is_same_v<T, bInt256>)
if constexpr (std::is_same_v<T, Int256>)
{
static constexpr Int128 max_int128 = (Int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll;
@ -877,7 +877,7 @@ void writeText(Decimal<T> x, UInt32 scale, WriteBuffer & ostr)
writeChar('-', ostr); /// avoid crop leading minus when whole part is zero
}
if constexpr (std::is_same_v<T, bInt256>)
if constexpr (std::is_same_v<T, Int256>)
writeText(part, ostr);
else
writeIntText(part, ostr);
@ -919,14 +919,14 @@ inline void writeQuoted(const UUID & x, WriteBuffer & buf)
writeChar('\'', buf);
}
inline void writeQuoted(const bUInt256 & x, WriteBuffer & buf)
inline void writeQuoted(const UInt256 & x, WriteBuffer & buf)
{
writeChar('\'', buf);
writeText(x, buf);
writeChar('\'', buf);
}
inline void writeQuoted(const bInt256 & x, WriteBuffer & buf)
inline void writeQuoted(const Int256 & x, WriteBuffer & buf)
{
writeChar('\'', buf);
writeText(x, buf);

View File

@ -304,7 +304,7 @@ AggregatedDataVariants::Type Aggregator::chooseAggregationMethod()
/// into a fixed 16- or 32-byte blob.
if (std::tuple_size<KeysNullMap<UInt128>>::value + keys_bytes <= 16)
return AggregatedDataVariants::Type::nullable_keys128;
if (std::tuple_size<KeysNullMap<UInt256>>::value + keys_bytes <= 32)
if (std::tuple_size<KeysNullMap<DummyUInt256>>::value + keys_bytes <= 32)
return AggregatedDataVariants::Type::nullable_keys256;
}

View File

@ -79,7 +79,7 @@ using AggregatedDataWithShortStringKey = StringHashMap<AggregateDataPtr>;
using AggregatedDataWithStringKey = HashMapWithSavedHash<StringRef, AggregateDataPtr>;
using AggregatedDataWithKeys128 = HashMap<UInt128, AggregateDataPtr, UInt128HashCRC32>;
using AggregatedDataWithKeys256 = HashMap<UInt256, AggregateDataPtr, UInt256HashCRC32>;
using AggregatedDataWithKeys256 = HashMap<DummyUInt256, AggregateDataPtr, UInt256HashCRC32>;
using AggregatedDataWithUInt32KeyTwoLevel = TwoLevelHashMap<UInt32, AggregateDataPtr, HashCRC32<UInt32>>;
using AggregatedDataWithUInt64KeyTwoLevel = TwoLevelHashMap<UInt64, AggregateDataPtr, HashCRC32<UInt64>>;
@ -89,7 +89,7 @@ using AggregatedDataWithShortStringKeyTwoLevel = TwoLevelStringHashMap<Aggregate
using AggregatedDataWithStringKeyTwoLevel = TwoLevelHashMapWithSavedHash<StringRef, AggregateDataPtr>;
using AggregatedDataWithKeys128TwoLevel = TwoLevelHashMap<UInt128, AggregateDataPtr, UInt128HashCRC32>;
using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap<UInt256, AggregateDataPtr, UInt256HashCRC32>;
using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap<DummyUInt256, AggregateDataPtr, UInt256HashCRC32>;
/** Variants with better hash function, using more than 32 bits for hash.
* Using for merging phase of external aggregation, where number of keys may be far greater than 4 billion,
@ -101,7 +101,7 @@ using AggregatedDataWithKeys256TwoLevel = TwoLevelHashMap<UInt256, AggregateData
using AggregatedDataWithUInt64KeyHash64 = HashMap<UInt64, AggregateDataPtr, DefaultHash<UInt64>>;
using AggregatedDataWithStringKeyHash64 = HashMapWithSavedHash<StringRef, AggregateDataPtr, StringRefHash64>;
using AggregatedDataWithKeys128Hash64 = HashMap<UInt128, AggregateDataPtr, UInt128Hash>;
using AggregatedDataWithKeys256Hash64 = HashMap<UInt256, AggregateDataPtr, UInt256Hash>;
using AggregatedDataWithKeys256Hash64 = HashMap<DummyUInt256, AggregateDataPtr, UInt256Hash>;
template <typename Base>
struct AggregationDataWithNullKey : public Base

View File

@ -327,7 +327,7 @@ template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<HashJoin:
};
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<HashJoin::Type::keys256, Value, Mapped>
{
using Type = ColumnsHashing::HashMethodKeysFixed<Value, UInt256, Mapped, false, false, false>;
using Type = ColumnsHashing::HashMethodKeysFixed<Value, DummyUInt256, Mapped, false, false, false>;
};
template <typename Value, typename Mapped> struct KeyGetterForTypeImpl<HashJoin::Type::hashed, Value, Mapped>
{

View File

@ -245,7 +245,7 @@ public:
std::unique_ptr<HashMapWithSavedHash<StringRef, Mapped>> key_string;
std::unique_ptr<HashMapWithSavedHash<StringRef, Mapped>> key_fixed_string;
std::unique_ptr<HashMap<UInt128, Mapped, UInt128HashCRC32>> keys128;
std::unique_ptr<HashMap<UInt256, Mapped, UInt256HashCRC32>> keys256;
std::unique_ptr<HashMap<DummyUInt256, Mapped, UInt256HashCRC32>> keys256;
std::unique_ptr<HashMap<UInt128, Mapped, UInt128TrivialHash>> hashed;
void create(Type which)

View File

@ -123,7 +123,7 @@ typename SetVariantsTemplate<Variant>::Type SetVariantsTemplate<Variant>::choose
throw Exception{"Aggregator: keys sizes overflow", ErrorCodes::LOGICAL_ERROR};
if ((std::tuple_size<KeysNullMap<UInt128>>::value + keys_bytes) <= 16)
return Type::nullable_keys128;
if ((std::tuple_size<KeysNullMap<UInt256>>::value + keys_bytes) <= 32)
if ((std::tuple_size<KeysNullMap<DummyUInt256>>::value + keys_bytes) <= 32)
return Type::nullable_keys256;
}

View File

@ -202,12 +202,12 @@ struct NonClearableSet
std::unique_ptr<SetMethodString<HashSetWithSavedHash<StringRef>>> key_string;
std::unique_ptr<SetMethodFixedString<HashSetWithSavedHash<StringRef>>> key_fixed_string;
std::unique_ptr<SetMethodKeysFixed<HashSet<UInt128, UInt128HashCRC32>>> keys128;
std::unique_ptr<SetMethodKeysFixed<HashSet<UInt256, UInt256HashCRC32>>> keys256;
std::unique_ptr<SetMethodKeysFixed<HashSet<DummyUInt256, UInt256HashCRC32>>> keys256;
std::unique_ptr<SetMethodHashed<HashSet<UInt128, UInt128TrivialHash>>> hashed;
/// Support for nullable keys (for DISTINCT implementation).
std::unique_ptr<SetMethodKeysFixed<HashSet<UInt128, UInt128HashCRC32>, true>> nullable_keys128;
std::unique_ptr<SetMethodKeysFixed<HashSet<UInt256, UInt256HashCRC32>, true>> nullable_keys256;
std::unique_ptr<SetMethodKeysFixed<HashSet<DummyUInt256, UInt256HashCRC32>, true>> nullable_keys256;
/** Unlike Aggregator, `concat` method is not used here.
* This is done because `hashed` method, although slower, but in this case, uses less RAM.
* since when you use it, the key values themselves are not stored.
@ -224,12 +224,12 @@ struct ClearableSet
std::unique_ptr<SetMethodString<ClearableHashSetWithSavedHash<StringRef>>> key_string;
std::unique_ptr<SetMethodFixedString<ClearableHashSetWithSavedHash<StringRef>>> key_fixed_string;
std::unique_ptr<SetMethodKeysFixed<ClearableHashSet<UInt128, UInt128HashCRC32>>> keys128;
std::unique_ptr<SetMethodKeysFixed<ClearableHashSet<UInt256, UInt256HashCRC32>>> keys256;
std::unique_ptr<SetMethodKeysFixed<ClearableHashSet<DummyUInt256, UInt256HashCRC32>>> keys256;
std::unique_ptr<SetMethodHashed<ClearableHashSet<UInt128, UInt128TrivialHash>>> hashed;
/// Support for nullable keys (for DISTINCT implementation).
std::unique_ptr<SetMethodKeysFixed<ClearableHashSet<UInt128, UInt128HashCRC32>, true>> nullable_keys128;
std::unique_ptr<SetMethodKeysFixed<ClearableHashSet<UInt256, UInt256HashCRC32>, true>> nullable_keys256;
std::unique_ptr<SetMethodKeysFixed<ClearableHashSet<DummyUInt256, UInt256HashCRC32>, true>> nullable_keys256;
/** Unlike Aggregator, `concat` method is not used here.
* This is done because `hashed` method, although slower, but in this case, uses less RAM.
* since when you use it, the key values themselves are not stored.

View File

@ -21,6 +21,7 @@ MergeListElement::MergeListElement(const std::string & database_, const std::str
, result_data_version{future_part.part_info.getDataVersion()}
, num_parts{future_part.parts.size()}
, thread_id{getThreadId()}
, merge_type{toString(future_part.merge_type)}
{
for (const auto & source_part : future_part.parts)
{
@ -70,6 +71,7 @@ MergeInfo MergeListElement::getInfo() const
res.columns_written = columns_written.load(std::memory_order_relaxed);
res.memory_usage = memory_tracker.get();
res.thread_id = thread_id;
res.merge_type = merge_type;
for (const auto & source_part_name : source_part_names)
res.source_part_names.emplace_back(source_part_name);

View File

@ -45,6 +45,7 @@ struct MergeInfo
UInt64 columns_written;
UInt64 memory_usage;
UInt64 thread_id;
std::string merge_type;
};
struct FutureMergedMutatedPart;
@ -88,6 +89,7 @@ struct MergeListElement : boost::noncopyable
UInt64 thread_id;
const std::string merge_type;
MergeListElement(const std::string & database, const std::string & table, const FutureMergedMutatedPart & future_part);

View File

@ -41,10 +41,16 @@ public:
const void * data;
/// Minimal time, when we need to delete some data from this part.
time_t min_ttl;
time_t min_delete_ttl;
/// Maximum time, when we will need to drop this part altogether because all rows in it are expired.
time_t max_ttl;
time_t max_delete_ttl;
/// Minimal time, when we need to recompress this part.
time_t min_recompress_ttl;
/// Maximum time, when we need to recompress this part.
time_t max_recompress_ttl;
};
/// Parts are belong to partitions. Only parts within same partition could be merged.

View File

@ -3064,10 +3064,10 @@ CompressionCodecPtr MergeTreeData::getCompressionCodecForPart(size_t part_size_c
auto metadata_snapshot = getInMemoryMetadataPtr();
const auto & recompression_ttl_entries = metadata_snapshot->getRecompressionTTLs();
std::cerr << "RECOMPRESSION ENTRIES SIZE:" << recompression_ttl_entries.size() << std::endl;
//std::cerr << "RECOMPRESSION ENTRIES SIZE:" << recompression_ttl_entries.size() << std::endl;
for (auto ttl_entry_it = recompression_ttl_entries.begin(); ttl_entry_it != recompression_ttl_entries.end(); ++ttl_entry_it)
{
std::cerr << "RECOMPRESSION TTL SIZE:" << ttl_infos.recompression_ttl.size() << std::endl;
//std::cerr << "RECOMPRESSION TTL SIZE:" << ttl_infos.recompression_ttl.size() << std::endl;
auto ttl_info_it = ttl_infos.recompression_ttl.find(ttl_entry_it->result_column);
/// Prefer TTL rule which went into action last.
if (ttl_info_it != ttl_infos.recompression_ttl.end()
@ -3081,14 +3081,14 @@ CompressionCodecPtr MergeTreeData::getCompressionCodecForPart(size_t part_size_c
if (max_max_ttl)
{
std::cerr << "BEST ENTRY FOUND, MAX MAX:" << max_max_ttl << std::endl;
std::cerr << "RECOMPRESSION IS NULLPTR:" << (best_entry_it->recompression_codec == nullptr) << std::endl;
//std::cerr << "BEST ENTRY FOUND, MAX MAX:" << max_max_ttl << std::endl;
//std::cerr << "RECOMPRESSION IS NULLPTR:" << (best_entry_it->recompression_codec == nullptr) << std::endl;
return CompressionCodecFactory::instance().get(best_entry_it->recompression_codec, {});
}
else
{
std::cerr << "NOT FOUND NEW RECOMPRESSION\n";
}
//else
//{
// std::cerr << "NOT FOUND NEW RECOMPRESSION\n";
//}
return global_context.chooseCompressionCodec(
part_size_compressed,

View File

@ -158,15 +158,15 @@ MergeTreeDataMergerMutator::MergeTreeDataMergerMutator(MergeTreeData & data_, si
}
UInt64 MergeTreeDataMergerMutator::getMaxSourcePartsSizeForMerge()
UInt64 MergeTreeDataMergerMutator::getMaxSourcePartsSizeForMerge(bool with_ttl) const
{
size_t busy_threads_in_pool = CurrentMetrics::values[CurrentMetrics::BackgroundPoolTask].load(std::memory_order_relaxed);
return getMaxSourcePartsSizeForMerge(background_pool_size, busy_threads_in_pool == 0 ? 0 : busy_threads_in_pool - 1); /// 1 is current thread
return getMaxSourcePartsSizeForMerge(background_pool_size, busy_threads_in_pool == 0 ? 0 : busy_threads_in_pool - 1, with_ttl); /// 1 is current thread
}
UInt64 MergeTreeDataMergerMutator::getMaxSourcePartsSizeForMerge(size_t pool_size, size_t pool_used)
UInt64 MergeTreeDataMergerMutator::getMaxSourcePartsSizeForMerge(size_t pool_size, size_t pool_used, bool with_ttl) const
{
if (pool_used > pool_size)
throw Exception("Logical error: invalid arguments passed to getMaxSourcePartsSize: pool_used > pool_size", ErrorCodes::LOGICAL_ERROR);
@ -178,20 +178,27 @@ UInt64 MergeTreeDataMergerMutator::getMaxSourcePartsSizeForMerge(size_t pool_siz
/// One entry is probably the entry where this function is executed.
/// This will protect from bad settings.
size_t lowering_setting;
if (with_ttl)
lowering_setting = data_settings->number_of_free_entries_in_pool_to_lower_max_size_of_merge_with_ttl;
else
lowering_setting = data_settings->number_of_free_entries_in_pool_to_lower_max_size_of_merge;
UInt64 max_size = 0;
if (pool_used <= 1 || free_entries >= data_settings->number_of_free_entries_in_pool_to_lower_max_size_of_merge)
if (pool_used <= 1 || free_entries >= lowering_setting)
max_size = data_settings->max_bytes_to_merge_at_max_space_in_pool;
else
max_size = interpolateExponential(
data_settings->max_bytes_to_merge_at_min_space_in_pool,
data_settings->max_bytes_to_merge_at_max_space_in_pool,
static_cast<double>(free_entries) / data_settings->number_of_free_entries_in_pool_to_lower_max_size_of_merge);
static_cast<double>(free_entries) / lowering_setting);
return std::min(max_size, static_cast<UInt64>(data.getStoragePolicy()->getMaxUnreservedFreeSpace() / DISK_USAGE_COEFFICIENT_TO_SELECT));
}
UInt64 MergeTreeDataMergerMutator::getMaxSourcePartSizeForMutation()
UInt64 MergeTreeDataMergerMutator::getMaxSourcePartSizeForMutation() const
{
const auto data_settings = data.getSettings();
size_t busy_threads_in_pool = CurrentMetrics::values[CurrentMetrics::BackgroundPoolTask].load(std::memory_order_relaxed);
@ -213,6 +220,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge(
bool aggressive,
size_t max_total_size_to_merge,
const AllowedMergingPredicate & can_merge_callback,
size_t max_total_size_to_merge_with_ttl,
String * out_disable_reason)
{
MergeTreeData::DataPartsVector data_parts = data.getDataPartsVector();
@ -260,8 +268,10 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge(
part_info.age = current_time - part->modification_time;
part_info.level = part->info.level;
part_info.data = &part;
part_info.min_ttl = part->ttl_infos.part_min_ttl;
part_info.max_ttl = part->ttl_infos.part_max_ttl;
part_info.min_delete_ttl = part->ttl_infos.part_min_ttl;
part_info.max_delete_ttl = part->ttl_infos.part_max_ttl;
part_info.min_recompress_ttl = part->ttl_infos.getMinRecompressionTTL();
part_info.max_recompress_ttl = part->ttl_infos.getMaxRecompressionTTL();
partitions.back().emplace_back(part_info);
@ -279,13 +289,26 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge(
if (!ttl_merges_blocker.isCancelled())
{
TTLMergeSelector merge_selector(
TTLDeleteMergeSelector delete_ttl_selector(
next_ttl_merge_times_by_partition,
current_time,
data_settings->merge_with_ttl_timeout,
data_settings->ttl_only_drop_parts);
parts_to_merge = merge_selector.select(partitions, max_total_size_to_merge);
parts_to_merge = delete_ttl_selector.select(partitions, max_total_size_to_merge_with_ttl);
if (!parts_to_merge.empty())
future_part.merge_type = MergeType::TTL_DELETE;
else
{
TTLRecompressMergeSelector recompress_ttl_selector(
next_ttl_merge_times_by_partition,
current_time,
data_settings->merge_with_ttl_timeout);
parts_to_merge = recompress_ttl_selector.select(partitions, max_total_size_to_merge_with_ttl);
if (!parts_to_merge.empty())
future_part.merge_type = MergeType::TTL_RECOMPRESS;
}
}
if (parts_to_merge.empty())
@ -307,6 +330,7 @@ bool MergeTreeDataMergerMutator::selectPartsToMerge(
*out_disable_reason = "There is no need to merge parts according to merge selector algorithm";
return false;
}
future_part.merge_type = MergeType::NORMAL;
}
MergeTreeData::DataPartsVector parts;
@ -386,6 +410,12 @@ bool MergeTreeDataMergerMutator::selectAllPartsToMergeWithinPartition(
LOG_DEBUG(log, "Selected {} parts from {} to {}", parts.size(), parts.front()->name, parts.back()->name);
future_part.assign(std::move(parts));
if (final)
future_part.merge_type = MergeType::FINAL;
else
future_part.merge_type = MergeType::NORMAL;
available_disk_space -= required_disk_space;
return true;
}
@ -635,6 +665,9 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor
new_data_part->partition.assign(future_part.getPartition());
new_data_part->is_temp = true;
if (future_part.merge_type == MergeType::TTL_DELETE && ttl_merges_blocker.isCancelled())
throw Exception("Cancelled merging parts with expired TTL", ErrorCodes::ABORTED);
bool need_remove_expired_values = false;
for (const auto & part : parts)
new_data_part->ttl_infos.update(part->ttl_infos);

View File

@ -6,6 +6,7 @@
#include <functional>
#include <Common/ActionBlocker.h>
#include <Storages/MergeTree/TTLMergeSelector.h>
#include <Storages/MergeTree/MergeType.h>
namespace DB
@ -22,6 +23,7 @@ struct FutureMergedMutatedPart
MergeTreeDataPartType type;
MergeTreePartInfo part_info;
MergeTreeData::DataPartsVector parts;
MergeType merge_type = MergeType::NORMAL;
const MergeTreePartition & getPartition() const { return parts.front()->partition; }
@ -57,17 +59,17 @@ public:
/** Get maximum total size of parts to do merge, at current moment of time.
* It depends on number of free threads in background_pool and amount of free space in disk.
*/
UInt64 getMaxSourcePartsSizeForMerge();
UInt64 getMaxSourcePartsSizeForMerge(bool with_ttl) const;
/** For explicitly passed size of pool and number of used tasks.
* This method could be used to calculate threshold depending on number of tasks in replication queue.
*/
UInt64 getMaxSourcePartsSizeForMerge(size_t pool_size, size_t pool_used);
UInt64 getMaxSourcePartsSizeForMerge(size_t pool_size, size_t pool_used, bool with_ttl) const;
/** Get maximum total size of parts to do mutation, at current moment of time.
* It depends only on amount of free space in disk.
*/
UInt64 getMaxSourcePartSizeForMutation();
UInt64 getMaxSourcePartSizeForMutation() const;
/** Selects which parts to merge. Uses a lot of heuristics.
*
@ -81,6 +83,7 @@ public:
bool aggressive,
size_t max_total_size_to_merge,
const AllowedMergingPredicate & can_merge,
size_t max_total_size_to_merge_with_ttl,
String * out_disable_reason = nullptr);
/** Select all the parts in the specified partition for merge, if possible.
@ -248,7 +251,7 @@ private:
time_t disk_space_warning_time = 0;
/// Stores the next TTL merge due time for each partition (used only by TTLMergeSelector)
TTLMergeSelector::PartitionIdToTTLs next_ttl_merge_times_by_partition;
ITTLMergeSelector::PartitionIdToTTLs next_ttl_merge_times_by_partition;
/// Performing TTL merges independently for each partition guarantees that
/// there is only a limited number of TTL merges and no partition stores data, that is too stale

View File

@ -2,6 +2,7 @@
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <Common/quoteString.h>
#include <algorithm>
#include <common/JSON.h>
@ -17,10 +18,7 @@ void MergeTreeDataPartTTLInfos::update(const MergeTreeDataPartTTLInfos & other_i
}
for (const auto & [name, ttl_info] : other_infos.recompression_ttl)
{
recompression_ttl[name].update(ttl_info);
updatePartMinMaxTTL(ttl_info.min, ttl_info.max);
}
for (const auto & [expression, ttl_info] : other_infos.moves_ttl)
{
@ -161,9 +159,31 @@ void MergeTreeDataPartTTLInfos::write(WriteBuffer & out) const
writeString("}", out);
}
writeString("]", out);
}
writeString("}", out);
}
time_t MergeTreeDataPartTTLInfos::getMinRecompressionTTL() const
{
time_t min = std::numeric_limits<time_t>::max();
for (const auto & [name, info] : recompression_ttl)
{
if (info.min != 0)
min = std::min(info.min, min);
}
if (min == std::numeric_limits<time_t>::max())
return 0;
return min;
}
time_t MergeTreeDataPartTTLInfos::getMaxRecompressionTTL() const
{
time_t max = 0;
for (const auto & [name, info] : recompression_ttl)
max = std::max(info.max, max);
return max;
}
}

View File

@ -48,6 +48,9 @@ struct MergeTreeDataPartTTLInfos
/// Order is important as it would be serialized and hashed for checksums
std::map<String, MergeTreeDataPartTTLInfo> recompression_ttl;
time_t getMinRecompressionTTL() const;
time_t getMaxRecompressionTTL() const;
void read(ReadBuffer & in);
void write(WriteBuffer & out) const;
void update(const MergeTreeDataPartTTLInfos & other_infos);

View File

@ -33,10 +33,10 @@ struct Settings;
M(UInt64, max_bytes_to_merge_at_min_space_in_pool, 1024 * 1024, "Maximum in total size of parts to merge, when there are minimum free threads in background pool (or entries in replication queue).", 0) \
M(UInt64, max_replicated_merges_in_queue, 16, "How many tasks of merging and mutating parts are allowed simultaneously in ReplicatedMergeTree queue.", 0) \
M(UInt64, max_replicated_mutations_in_queue, 8, "How many tasks of mutating parts are allowed simultaneously in ReplicatedMergeTree queue.", 0) \
M(UInt64, max_replicated_recompressions_in_queue, 1, "How many tasks of recompressiong parts are allowed simultaneously in ReplicatedMergeTree queue.", 0) \
M(UInt64, max_replicated_merges_with_ttl_in_queue, 1, "How many tasks of mutating parts are allowed simultaneously in ReplicatedMergeTree queue.", 0) \
M(UInt64, number_of_free_entries_in_pool_to_lower_max_size_of_merge, 8, "When there is less than specified number of free entries in pool (or replicated queue), start to lower maximum size of merge to process (or to put in queue). This is to allow small merges to process - not filling the pool with long running merges.", 0) \
M(UInt64, number_of_free_entries_in_pool_to_execute_mutation, 10, "When there is less than specified number of free entries in pool, do not execute part mutations. This is to leave free threads for regular merges and avoid \"Too many parts\"", 0) \
M(UInt64, number_of_free_entries_in_pool_to_execute_ttl_recompression, 10, "When there is less than specified number of free entries in pool, do not execute part recompression according to TTL. This is to leave free threads for regular merges and avoid \"Too many parts\"", 0) \
M(UInt64, number_of_free_entries_in_pool_to_lower_max_size_of_merge_with_ttl, 14, "When there is less than specified number of free entries in pool (or replicated queue), start to lower maximum size of merge to process (or to put in queue). This is to allow small merges to process - not filling the pool with long running merges.", 0) \
M(Seconds, old_parts_lifetime, 8 * 60, "How many seconds to keep obsolete parts.", 0) \
M(Seconds, temporary_directories_lifetime, 86400, "How many seconds to keep tmp_-directories.", 0) \
M(Seconds, lock_acquire_timeout_for_background_operations, DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC, "For background operations like merges, mutations etc. How many seconds before failing to acquire table locks.", 0) \
@ -85,7 +85,7 @@ struct Settings;
M(UInt64, min_merge_bytes_to_use_direct_io, 10ULL * 1024 * 1024 * 1024, "Minimal amount of bytes to enable O_DIRECT in merge (0 - disabled).", 0) \
M(UInt64, index_granularity_bytes, 10 * 1024 * 1024, "Approximate amount of bytes in single granule (0 - disabled).", 0) \
M(UInt64, min_index_granularity_bytes, 1024, "Minimum amount of bytes in single granule.", 1024) \
M(Int64, merge_with_ttl_timeout, 3600 * 24, "Minimal time in seconds, when merge with TTL can be repeated.", 0) \
M(Int64, merge_with_ttl_timeout, 0, "Minimal time in seconds, when merge with TTL can be repeated.", 0) \
M(Bool, ttl_only_drop_parts, false, "Only drop altogether the expired parts and not partially prune them.", 0) \
M(Bool, write_final_mark, 1, "Write final mark after end of column (0 - disabled, do nothing if index_granularity_bytes=0)", 0) \
M(Bool, enable_mixed_granularity_parts, 1, "Enable parts with adaptive and non adaptive granularity", 0) \

View File

@ -0,0 +1,29 @@
#include <Storages/MergeTree/MergeType.h>
#include <Common/Exception.h>
namespace DB
{
namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
}
String toString(MergeType merge_type)
{
switch (merge_type)
{
case MergeType::NORMAL:
return "NORMAL";
case MergeType::FINAL:
return "FINAL";
case MergeType::TTL_DELETE:
return "TTL_DELETE";
case MergeType::TTL_RECOMPRESS:
return "TTL_RECOMPRESS";
}
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unknown MergeType {}", static_cast<UInt64>(merge_type));
}
}

View File

@ -0,0 +1,18 @@
#pragma once
#include <Core/Types.h>
namespace DB
{
enum class MergeType
{
NORMAL,
FINAL,
TTL_DELETE,
TTL_RECOMPRESS,
};
String toString(MergeType merge_type);
}

View File

@ -36,9 +36,8 @@ void ReplicatedMergeTreeLogEntryData::writeText(WriteBuffer & out) const
out << s << '\n';
out << "into\n" << new_part_name;
out << "\ndeduplicate: " << deduplicate;
/// For backward compatibility write only if enabled
if (recompress)
out << "\nrecompress: " << recompress;
if (merge_type != MergeType::NORMAL)
out <<"\nmerge_type: " << static_cast<UInt64>(merge_type);
break;
case DROP_RANGE:
@ -157,8 +156,12 @@ void ReplicatedMergeTreeLogEntryData::readText(ReadBuffer & in)
in >> "\n";
if (in.eof())
trailing_newline_found = true;
else if (checkString("recompress\n", in))
in >> recompress;
else if (checkString("merge_type: ", in))
{
UInt64 value;
in >> value;
merge_type = static_cast<MergeType>(value);
}
}
}
else if (type_str == "drop" || type_str == "detach")

View File

@ -5,6 +5,7 @@
#include <Core/Types.h>
#include <IO/WriteHelpers.h>
#include <Storages/MergeTree/MergeTreeDataPartType.h>
#include <Storages/MergeTree/MergeType.h>
#include <mutex>
#include <condition_variable>
@ -79,7 +80,7 @@ struct ReplicatedMergeTreeLogEntryData
Strings source_parts;
bool deduplicate = false; /// Do deduplicate on merge
bool recompress = false; /// Recompress parts on merge
MergeType merge_type = MergeType::NORMAL;
String column_name;
String index_name;

View File

@ -1061,7 +1061,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry(
return false;
}
UInt64 max_source_parts_size = entry.type == LogEntry::MERGE_PARTS ? merger_mutator.getMaxSourcePartsSizeForMerge()
UInt64 max_source_parts_size = entry.type == LogEntry::MERGE_PARTS ? merger_mutator.getMaxSourcePartsSizeForMerge(entry.merge_type == MergeType::TTL_DELETE)
: merger_mutator.getMaxSourcePartSizeForMutation();
/** If there are enough free threads in background pool to do large merges (maximal size of merge is allowed),
* then ignore value returned by getMaxSourcePartsSizeForMerge() and execute merge of any size,
@ -1312,21 +1312,26 @@ bool ReplicatedMergeTreeQueue::processEntry(
}
std::pair<size_t, size_t> ReplicatedMergeTreeQueue::countMergesAndPartMutations() const
ReplicatedMergeTreeQueue::OperationsInQueue ReplicatedMergeTreeQueue::countMergesAndPartMutations() const
{
std::lock_guard lock(state_mutex);
size_t count_merges = 0;
size_t count_mutations = 0;
size_t count_merges_with_ttl = 0;
for (const auto & entry : queue)
{
if (entry->type == ReplicatedMergeTreeLogEntry::MERGE_PARTS)
{
++count_merges;
if (entry->merge_type == MergeType::TTL_DELETE)
++count_merges_with_ttl;
}
else if (entry->type == ReplicatedMergeTreeLogEntry::MUTATE_PART)
++count_mutations;
}
return std::make_pair(count_merges, count_mutations);
return OperationsInQueue{count_merges, count_mutations, count_merges_with_ttl};
}

View File

@ -46,6 +46,13 @@ private:
}
};
struct OperationsInQueue
{
size_t merges = 0;
size_t mutations = 0;
size_t merges_with_ttl = 0;
};
/// To calculate min_unprocessed_insert_time, max_processed_insert_time, for which the replica lag is calculated.
using InsertsByTime = std::set<LogEntryPtr, ByTime>;
@ -325,7 +332,7 @@ public:
bool processEntry(std::function<zkutil::ZooKeeperPtr()> get_zookeeper, LogEntryPtr & entry, const std::function<bool(LogEntryPtr &)> func);
/// Count the number of merges and mutations of single parts in the queue.
std::pair<size_t, size_t> countMergesAndPartMutations() const;
OperationsInQueue countMergesAndPartMutations() const;
/// Count the total number of active mutations.
size_t countMutations() const;

View File

@ -8,14 +8,14 @@
namespace DB
{
const String & getPartitionIdForPart(const TTLMergeSelector::Part & part_info)
const String & getPartitionIdForPart(const ITTLMergeSelector::Part & part_info)
{
const MergeTreeData::DataPartPtr & part = *static_cast<const MergeTreeData::DataPartPtr *>(part_info.data);
return part->info.partition_id;
}
IMergeSelector::PartsInPartition TTLMergeSelector::select(
IMergeSelector::PartsInPartition ITTLMergeSelector::select(
const Partitions & partitions,
const size_t max_total_size_to_merge)
{
@ -37,7 +37,7 @@ IMergeSelector::PartsInPartition TTLMergeSelector::select(
for (Iterator part_it = mergeable_parts_in_partition.cbegin(); part_it != mergeable_parts_in_partition.cend(); ++part_it)
{
time_t ttl = only_drop_parts ? part_it->max_ttl : part_it->min_ttl;
time_t ttl = getTTLForPart(*part_it);
if (ttl && (partition_to_merge_index == -1 || ttl < partition_to_merge_min_ttl))
{
@ -57,7 +57,7 @@ IMergeSelector::PartsInPartition TTLMergeSelector::select(
while (true)
{
time_t ttl = only_drop_parts ? best_begin->max_ttl : best_begin->min_ttl;
time_t ttl = getTTLForPart(*best_begin);
if (!ttl || ttl > current_time
|| (max_total_size_to_merge && total_size > max_total_size_to_merge))
@ -75,7 +75,7 @@ IMergeSelector::PartsInPartition TTLMergeSelector::select(
while (best_end != best_partition.end())
{
time_t ttl = only_drop_parts ? best_end->max_ttl : best_end->min_ttl;
time_t ttl = getTTLForPart(*best_end);
if (!ttl || ttl > current_time
|| (max_total_size_to_merge && total_size > max_total_size_to_merge))
@ -91,4 +91,14 @@ IMergeSelector::PartsInPartition TTLMergeSelector::select(
return PartsInPartition(best_begin, best_end);
}
time_t TTLDeleteMergeSelector::getTTLForPart(const IMergeSelector::Part & part) const
{
return only_drop_parts ? part.max_delete_ttl : part.min_delete_ttl;
}
time_t TTLRecompressMergeSelector::getTTLForPart(const IMergeSelector::Part & part) const
{
return part.min_recompress_ttl;
}
}

View File

@ -10,31 +10,57 @@ namespace DB
{
/** Merge selector, which is used to remove values with expired ttl.
* It selects parts to merge by greedy algorithm:
* It selects parts to merge by greedy algorithm:
* 1. Finds part with the most earliest expired ttl and includes it to result.
* 2. Tries to find the longest range of parts with expired ttl, that includes part from step 1.
* Finally, merge selector updates TTL merge timer for the selected partition
*/
class TTLMergeSelector : public IMergeSelector
class ITTLMergeSelector : public IMergeSelector
{
public:
using PartitionIdToTTLs = std::map<String, time_t>;
explicit TTLMergeSelector(PartitionIdToTTLs & merge_due_times_, time_t current_time_, Int64 merge_cooldown_time_, bool only_drop_parts_)
ITTLMergeSelector(PartitionIdToTTLs & merge_due_times_, time_t current_time_, Int64 merge_cooldown_time_)
: merge_due_times(merge_due_times_),
current_time(current_time_),
merge_cooldown_time(merge_cooldown_time_),
only_drop_parts(only_drop_parts_) {}
merge_cooldown_time(merge_cooldown_time_)
{
}
PartsInPartition select(
const Partitions & partitions,
const size_t max_total_size_to_merge) override;
virtual time_t getTTLForPart(const IMergeSelector::Part & part) const = 0;
private:
PartitionIdToTTLs & merge_due_times;
time_t current_time;
Int64 merge_cooldown_time;
};
class TTLDeleteMergeSelector : public ITTLMergeSelector
{
public:
using PartitionIdToTTLs = std::map<String, time_t>;
TTLDeleteMergeSelector(PartitionIdToTTLs & merge_due_times_, time_t current_time_, Int64 merge_cooldown_time_, bool only_drop_parts_)
: ITTLMergeSelector(merge_due_times_, current_time_, merge_cooldown_time_)
, only_drop_parts(only_drop_parts_) {}
time_t getTTLForPart(const IMergeSelector::Part & part) const override;
private:
bool only_drop_parts;
};
class TTLRecompressMergeSelector : public ITTLMergeSelector
{
public:
using ITTLMergeSelector::ITTLMergeSelector;
time_t getTTLForPart(const IMergeSelector::Part & part) const override;
};
}

View File

@ -650,9 +650,14 @@ bool StorageMergeTree::merge(
if (partition_id.empty())
{
UInt64 max_source_parts_size = merger_mutator.getMaxSourcePartsSizeForMerge();
UInt64 max_source_parts_size = merger_mutator.getMaxSourcePartsSizeForMerge(false);
UInt64 max_source_parts_size_with_ttl = 0;
if (!aggressive)
max_source_parts_size_with_ttl = merger_mutator.getMaxSourcePartsSizeForMerge(true);
if (max_source_parts_size > 0)
selected = merger_mutator.selectPartsToMerge(future_part, aggressive, max_source_parts_size, can_merge, out_disable_reason);
selected = merger_mutator.selectPartsToMerge(future_part, aggressive, max_source_parts_size, can_merge, max_source_parts_size_with_ttl, out_disable_reason);
else if (out_disable_reason)
*out_disable_reason = "Current value of max_source_parts_size is zero";
}
@ -724,6 +729,7 @@ bool StorageMergeTree::merge(
try
{
std::cerr << "FUTURE PART MERGE TYPE:" << toString(future_part.merge_type) << std::endl;
new_part = merger_mutator.mergePartsToTemporaryPart(
future_part, metadata_snapshot, *merge_entry, table_lock_holder, time(nullptr),
merging_tagger->reserved_space, deduplicate);

View File

@ -2514,31 +2514,38 @@ void StorageReplicatedMergeTree::mergeSelectingTask()
/// and in the same time, many small parts could be created and won't be merged.
auto merges_and_mutations_queued = queue.countMergesAndPartMutations();
size_t merges_and_mutations_sum = merges_and_mutations_queued.first + merges_and_mutations_queued.second;
size_t merges_and_mutations_sum = merges_and_mutations_queued.merges + merges_and_mutations_queued.mutations;
if (merges_and_mutations_sum >= storage_settings_ptr->max_replicated_merges_in_queue)
{
LOG_TRACE(log, "Number of queued merges ({}) and part mutations ({})"
" is greater than max_replicated_merges_in_queue ({}), so won't select new parts to merge or mutate.",
merges_and_mutations_queued.first,
merges_and_mutations_queued.second,
merges_and_mutations_queued.merges,
merges_and_mutations_queued.mutations,
storage_settings_ptr->max_replicated_merges_in_queue);
}
else
{
UInt64 max_source_parts_size_for_merge = merger_mutator.getMaxSourcePartsSizeForMerge(
storage_settings_ptr->max_replicated_merges_in_queue, merges_and_mutations_sum);
storage_settings_ptr->max_replicated_merges_in_queue, merges_and_mutations_sum, false);
UInt64 max_source_parts_size_for_merge_with_ttl = 0;
if (merges_and_mutations_queued.merges_with_ttl < storage_settings_ptr->max_replicated_merges_with_ttl_in_queue)
max_source_parts_size_for_merge_with_ttl = merger_mutator.getMaxSourcePartsSizeForMerge(
storage_settings_ptr->max_replicated_merges_in_queue, merges_and_mutations_sum, true);
UInt64 max_source_part_size_for_mutation = merger_mutator.getMaxSourcePartSizeForMutation();
FutureMergedMutatedPart future_merged_part;
if (max_source_parts_size_for_merge > 0 &&
merger_mutator.selectPartsToMerge(future_merged_part, false, max_source_parts_size_for_merge, merge_pred, nullptr))
merger_mutator.selectPartsToMerge(future_merged_part, false,
max_source_parts_size_for_merge, merge_pred, max_source_parts_size_for_merge_with_ttl, nullptr))
{
create_result = createLogEntryToMergeParts(zookeeper, future_merged_part.parts,
future_merged_part.name, future_merged_part.type, deduplicate, nullptr, merge_pred.getVersion());
future_merged_part.name, future_merged_part.type, deduplicate, nullptr, merge_pred.getVersion(), future_merged_part.merge_type);
}
/// If there are many mutations in queue, it may happen, that we cannot enqueue enough merges to merge all new parts
else if (max_source_part_size_for_mutation > 0 && queue.countMutations() > 0
&& merges_and_mutations_queued.second < storage_settings_ptr->max_replicated_mutations_in_queue)
&& merges_and_mutations_queued.mutations < storage_settings_ptr->max_replicated_mutations_in_queue)
{
/// Choose a part to mutate.
DataPartsVector data_parts = getDataPartsVector();
@ -2617,7 +2624,8 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c
const MergeTreeDataPartType & merged_part_type,
bool deduplicate,
ReplicatedMergeTreeLogEntryData * out_log_entry,
int32_t log_version)
int32_t log_version,
MergeType merge_type)
{
std::vector<std::future<Coordination::ExistsResponse>> exists_futures;
exists_futures.reserve(parts.size());
@ -2649,6 +2657,7 @@ StorageReplicatedMergeTree::CreateMergeEntryResult StorageReplicatedMergeTree::c
entry.source_replica = replica_name;
entry.new_part_name = merged_name;
entry.new_part_type = merged_part_type;
entry.merge_type = merge_type;
entry.deduplicate = deduplicate;
entry.create_time = time(nullptr);
@ -3584,7 +3593,7 @@ bool StorageReplicatedMergeTree::optimize(
CreateMergeEntryResult create_result = createLogEntryToMergeParts(
zookeeper, future_merged_part.parts,
future_merged_part.name, future_merged_part.type, deduplicate,
&merge_entry, can_merge.getVersion());
&merge_entry, can_merge.getVersion(), future_merged_part.merge_type);
if (create_result == CreateMergeEntryResult::MissingPart)
return handle_noop("Can't create merge queue node in ZooKeeper, because some parts are missing");
@ -3614,7 +3623,7 @@ bool StorageReplicatedMergeTree::optimize(
if (!partition)
{
selected = merger_mutator.selectPartsToMerge(
future_merged_part, true, storage_settings_ptr->max_bytes_to_merge_at_max_space_in_pool, can_merge, &disable_reason);
future_merged_part, true, storage_settings_ptr->max_bytes_to_merge_at_max_space_in_pool, can_merge, 0, &disable_reason);
}
else
{
@ -3639,7 +3648,7 @@ bool StorageReplicatedMergeTree::optimize(
CreateMergeEntryResult create_result = createLogEntryToMergeParts(
zookeeper, future_merged_part.parts,
future_merged_part.name, future_merged_part.type, deduplicate,
&merge_entry, can_merge.getVersion());
&merge_entry, can_merge.getVersion(), future_merged_part.merge_type);
if (create_result == CreateMergeEntryResult::MissingPart)
return handle_noop("Can't create merge queue node in ZooKeeper, because some parts are missing");

View File

@ -450,7 +450,8 @@ private:
const MergeTreeDataPartType & merged_part_type,
bool deduplicate,
ReplicatedMergeTreeLogEntryData * out_log_entry,
int32_t log_version);
int32_t log_version,
MergeType merge_type);
CreateMergeEntryResult createLogEntryToMutatePart(
const IMergeTreeDataPart & part,

View File

@ -30,6 +30,7 @@ NamesAndTypesList StorageSystemMerges::getNamesAndTypes()
{"columns_written", std::make_shared<DataTypeUInt64>()},
{"memory_usage", std::make_shared<DataTypeUInt64>()},
{"thread_id", std::make_shared<DataTypeUInt64>()},
{"merge_type", std::make_shared<DataTypeString>()},
};
}
@ -65,6 +66,7 @@ void StorageSystemMerges::fillData(MutableColumns & res_columns, const Context &
res_columns[i++]->insert(merge.columns_written);
res_columns[i++]->insert(merge.memory_usage);
res_columns[i++]->insert(merge.thread_id);
res_columns[i++]->insert(merge.merge_type);
}
}

View File

@ -337,7 +337,7 @@ TTLTableDescription TTLTableDescription::getTTLForTableFromAST(
}
else if (ttl.mode == TTLMode::RECOMPRESS)
{
std::cerr << "GOT RECOMPRESSIOn TTL\n";
//std::cerr << "GOT RECOMPRESSIOn TTL\n";
result.recompression_ttl.emplace_back(std::move(ttl));
}
else

View File

@ -155,7 +155,7 @@ if __name__ == "__main__":
elif image == "yandex/clickhouse-postgresql-java-client":
env_tags += "-e {}={} ".format("DOCKER_POSTGRESQL_JAVA_CLIENT_TAG", tag)
else:
raise Exception("Unknown image {}".format(image))
logging.info("Unknown image {}".format(image))
# create named volume which will be used inside to store images and other docker related files,
# to avoid redownloading it every time