2011-09-25 05:07:47 +00:00
|
|
|
#pragma once
|
|
|
|
|
2017-12-22 22:02:52 +00:00
|
|
|
#include <type_traits>
|
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <IO/WriteHelpers.h>
|
|
|
|
#include <IO/ReadHelpers.h>
|
2011-09-25 05:07:47 +00:00
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <DataTypes/DataTypesNumber.h>
|
2018-09-11 18:42:06 +00:00
|
|
|
#include <DataTypes/DataTypesDecimal.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Columns/ColumnVector.h>
|
2011-09-25 05:07:47 +00:00
|
|
|
|
2017-12-20 07:36:30 +00:00
|
|
|
#include <AggregateFunctions/IAggregateFunction.h>
|
2011-09-25 05:07:47 +00:00
|
|
|
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
|
|
|
template <typename T>
|
2013-02-08 19:34:44 +00:00
|
|
|
struct AggregateFunctionSumData
|
2011-09-25 05:07:47 +00:00
|
|
|
{
|
2017-08-22 17:29:02 +00:00
|
|
|
T sum{};
|
2017-12-22 22:02:52 +00:00
|
|
|
|
|
|
|
void add(T value)
|
|
|
|
{
|
|
|
|
sum += value;
|
|
|
|
}
|
|
|
|
|
2020-05-18 03:50:53 +00:00
|
|
|
/// Vectorized version
|
|
|
|
template <typename Value>
|
|
|
|
void addMany(const Value * __restrict ptr, size_t count)
|
|
|
|
{
|
|
|
|
/// Compiler cannot unroll this loop, do it manually.
|
|
|
|
|
|
|
|
/// Something around the number of SSE registers * the number of elements fit in register.
|
|
|
|
constexpr size_t unroll_count = 128 / sizeof(T);
|
|
|
|
T partial_sums[unroll_count]{};
|
|
|
|
|
|
|
|
const auto * end = ptr + count;
|
|
|
|
const auto * unrolled_end = ptr + (count / unroll_count * unroll_count);
|
|
|
|
|
|
|
|
while (ptr < unrolled_end)
|
|
|
|
{
|
|
|
|
for (size_t i = 0; i < unroll_count; ++i)
|
|
|
|
partial_sums[i] += ptr[i];
|
|
|
|
ptr += unroll_count;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (size_t i = 0; i < unroll_count; ++i)
|
|
|
|
sum += partial_sums[i];
|
|
|
|
|
|
|
|
while (ptr < end)
|
|
|
|
{
|
|
|
|
sum += *ptr;
|
|
|
|
++ptr;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-12-22 22:02:52 +00:00
|
|
|
void merge(const AggregateFunctionSumData & rhs)
|
|
|
|
{
|
|
|
|
sum += rhs.sum;
|
|
|
|
}
|
|
|
|
|
|
|
|
void write(WriteBuffer & buf) const
|
|
|
|
{
|
|
|
|
writeBinary(sum, buf);
|
|
|
|
}
|
|
|
|
|
|
|
|
void read(ReadBuffer & buf)
|
|
|
|
{
|
|
|
|
readBinary(sum, buf);
|
|
|
|
}
|
|
|
|
|
|
|
|
T get() const
|
|
|
|
{
|
|
|
|
return sum;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
template <typename T>
|
|
|
|
struct AggregateFunctionSumKahanData
|
|
|
|
{
|
|
|
|
static_assert(std::is_floating_point_v<T>,
|
|
|
|
"It doesn't make sense to use Kahan Summation algorithm for non floating point types");
|
|
|
|
|
|
|
|
T sum{};
|
|
|
|
T compensation{};
|
|
|
|
|
2020-05-18 03:50:53 +00:00
|
|
|
template <typename Value>
|
|
|
|
ALWAYS_INLINE void addImpl(Value value, T & out_sum, T & out_compensation)
|
|
|
|
{
|
|
|
|
auto compensated_value = value - out_compensation;
|
|
|
|
auto new_sum = out_sum + compensated_value;
|
|
|
|
out_compensation = (new_sum - out_sum) - compensated_value;
|
|
|
|
out_sum = new_sum;
|
|
|
|
}
|
|
|
|
|
2017-12-22 22:02:52 +00:00
|
|
|
void add(T value)
|
|
|
|
{
|
2020-05-18 03:50:53 +00:00
|
|
|
addImpl(value, sum, compensation);
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Vectorized version
|
|
|
|
template <typename Value>
|
|
|
|
void addMany(const Value * __restrict ptr, size_t count)
|
|
|
|
{
|
|
|
|
constexpr size_t unroll_count = 4; // 128 / sizeof(T);
|
|
|
|
T partial_sums[unroll_count]{};
|
|
|
|
T partial_compensations[unroll_count]{};
|
|
|
|
|
|
|
|
const auto * end = ptr + count;
|
|
|
|
const auto * unrolled_end = ptr + (count / unroll_count * unroll_count);
|
|
|
|
|
|
|
|
while (ptr < unrolled_end)
|
|
|
|
{
|
|
|
|
for (size_t i = 0; i < unroll_count; ++i)
|
|
|
|
addImpl(ptr[i], partial_sums[i], partial_compensations[i]);
|
|
|
|
ptr += unroll_count;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (size_t i = 0; i < unroll_count; ++i)
|
|
|
|
mergeImpl(sum, compensation, partial_sums[i], partial_compensations[i]);
|
|
|
|
|
|
|
|
while (ptr < end)
|
|
|
|
{
|
|
|
|
addImpl(*ptr, sum, compensation);
|
|
|
|
++ptr;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void ALWAYS_INLINE mergeImpl(T & to_sum, T & to_compensation, T from_sum, T from_compensation)
|
|
|
|
{
|
|
|
|
auto raw_sum = to_sum + from_sum;
|
|
|
|
auto rhs_compensated = raw_sum - to_sum;
|
|
|
|
auto compensations = ((from_sum - rhs_compensated) + (to_sum - (raw_sum - rhs_compensated))) + compensation + from_compensation;
|
|
|
|
to_sum = raw_sum + compensations;
|
|
|
|
to_compensation = compensations - (to_sum - raw_sum);
|
2017-12-22 22:02:52 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void merge(const AggregateFunctionSumKahanData & rhs)
|
|
|
|
{
|
2020-05-18 03:50:53 +00:00
|
|
|
mergeImpl(sum, compensation, rhs.sum, rhs.compensation);
|
2017-12-22 22:02:52 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void write(WriteBuffer & buf) const
|
|
|
|
{
|
|
|
|
writeBinary(sum, buf);
|
|
|
|
writeBinary(compensation, buf);
|
|
|
|
}
|
|
|
|
|
|
|
|
void read(ReadBuffer & buf)
|
|
|
|
{
|
|
|
|
readBinary(sum, buf);
|
|
|
|
readBinary(compensation, buf);
|
|
|
|
}
|
|
|
|
|
|
|
|
T get() const
|
|
|
|
{
|
|
|
|
return sum;
|
|
|
|
}
|
2013-02-08 19:34:44 +00:00
|
|
|
};
|
|
|
|
|
2015-11-11 02:04:23 +00:00
|
|
|
|
2020-03-12 19:13:38 +00:00
|
|
|
enum AggregateFunctionSumType
|
|
|
|
{
|
|
|
|
AggregateFunctionTypeSum,
|
|
|
|
AggregateFunctionTypeSumWithOverflow,
|
|
|
|
AggregateFunctionTypeSumKahan,
|
|
|
|
};
|
2017-03-09 00:56:38 +00:00
|
|
|
/// Counts the sum of the numbers.
|
2020-03-12 19:13:38 +00:00
|
|
|
template <typename T, typename TResult, typename Data, AggregateFunctionSumType Type>
|
|
|
|
class AggregateFunctionSum final : public IAggregateFunctionDataHelper<Data, AggregateFunctionSum<T, TResult, Data, Type>>
|
2013-02-08 19:34:44 +00:00
|
|
|
{
|
2011-09-25 05:07:47 +00:00
|
|
|
public:
|
2018-09-12 17:50:51 +00:00
|
|
|
using ResultDataType = std::conditional_t<IsDecimalNumber<T>, DataTypeDecimal<TResult>, DataTypeNumber<TResult>>;
|
|
|
|
using ColVecType = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;
|
|
|
|
using ColVecResult = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<TResult>, ColumnVector<TResult>>;
|
|
|
|
|
2020-03-12 19:13:38 +00:00
|
|
|
String getName() const override
|
|
|
|
{
|
|
|
|
if constexpr (Type == AggregateFunctionTypeSum)
|
|
|
|
return "sum";
|
|
|
|
else if constexpr (Type == AggregateFunctionTypeSumWithOverflow)
|
|
|
|
return "sumWithOverflow";
|
|
|
|
else if constexpr (Type == AggregateFunctionTypeSumKahan)
|
|
|
|
return "sumKahan";
|
|
|
|
__builtin_unreachable();
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-02-13 11:50:41 +00:00
|
|
|
AggregateFunctionSum(const DataTypes & argument_types_)
|
2020-03-12 19:13:38 +00:00
|
|
|
: IAggregateFunctionDataHelper<Data, AggregateFunctionSum<T, TResult, Data, Type>>(argument_types_, {})
|
2019-02-11 19:26:32 +00:00
|
|
|
, scale(0)
|
2018-09-12 17:50:51 +00:00
|
|
|
{}
|
|
|
|
|
2019-02-13 11:50:41 +00:00
|
|
|
AggregateFunctionSum(const IDataType & data_type, const DataTypes & argument_types_)
|
2020-03-12 19:13:38 +00:00
|
|
|
: IAggregateFunctionDataHelper<Data, AggregateFunctionSum<T, TResult, Data, Type>>(argument_types_, {})
|
2019-02-11 19:26:32 +00:00
|
|
|
, scale(getDecimalScale(data_type))
|
2018-09-12 17:50:51 +00:00
|
|
|
{}
|
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
DataTypePtr getReturnType() const override
|
|
|
|
{
|
2018-09-12 17:50:51 +00:00
|
|
|
if constexpr (IsDecimalNumber<T>)
|
|
|
|
return std::make_shared<ResultDataType>(ResultDataType::maxPrecision(), scale);
|
|
|
|
else
|
|
|
|
return std::make_shared<ResultDataType>();
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
2017-12-20 07:36:30 +00:00
|
|
|
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2018-09-12 17:50:51 +00:00
|
|
|
const auto & column = static_cast<const ColVecType &>(*columns[0]);
|
|
|
|
this->data(place).add(column.getData()[row_num]);
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
2020-05-18 03:50:53 +00:00
|
|
|
/// Vectorized version when there is no GROUP BY keys.
|
|
|
|
void addBatchSinglePlace(size_t batch_size, AggregateDataPtr place, const IColumn ** columns, Arena *) const override
|
|
|
|
{
|
|
|
|
const auto & column = static_cast<const ColVecType &>(*columns[0]);
|
|
|
|
this->data(place).addMany(column.getData().data(), batch_size);
|
|
|
|
}
|
|
|
|
|
2017-12-01 21:51:50 +00:00
|
|
|
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2017-12-22 22:02:52 +00:00
|
|
|
this->data(place).merge(this->data(rhs));
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
|
|
|
|
{
|
2017-12-22 22:02:52 +00:00
|
|
|
this->data(place).write(buf);
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
|
|
|
|
{
|
2017-12-22 22:02:52 +00:00
|
|
|
this->data(place).read(buf);
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
2020-05-14 07:59:14 +00:00
|
|
|
void insertResultInto(AggregateDataPtr place, IColumn & to) const override
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2018-09-12 17:50:51 +00:00
|
|
|
auto & column = static_cast<ColVecResult &>(to);
|
2018-09-11 18:42:06 +00:00
|
|
|
column.getData().push_back(this->data(place).get());
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
UInt32 scale;
|
|
|
|
};
|
|
|
|
|
2011-09-25 05:07:47 +00:00
|
|
|
}
|