2011-09-26 04:00:46 +00:00
|
|
|
#pragma once
|
|
|
|
|
|
|
|
#include <city.h>
|
2014-02-02 09:08:06 +00:00
|
|
|
#include <type_traits>
|
2011-09-26 04:00:46 +00:00
|
|
|
|
2018-06-04 14:17:47 +00:00
|
|
|
#include <ext/bit_cast.h>
|
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <IO/WriteHelpers.h>
|
|
|
|
#include <IO/ReadHelpers.h>
|
2011-09-26 04:00:46 +00:00
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <DataTypes/DataTypesNumber.h>
|
|
|
|
#include <DataTypes/DataTypeTuple.h>
|
2011-09-26 04:00:46 +00:00
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Interpreters/AggregationCommon.h>
|
2019-02-10 17:40:52 +00:00
|
|
|
|
2020-08-19 11:52:17 +00:00
|
|
|
#include <Common/HashTable/Hash.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Common/HashTable/HashSet.h>
|
|
|
|
#include <Common/HyperLogLogWithSmallSetOptimization.h>
|
|
|
|
#include <Common/CombinedCardinalityEstimator.h>
|
2017-07-13 20:58:19 +00:00
|
|
|
#include <Common/typeid_cast.h>
|
2019-08-21 02:28:04 +00:00
|
|
|
#include <Common/assert_cast.h>
|
2013-06-30 11:38:46 +00:00
|
|
|
|
2019-02-10 17:40:52 +00:00
|
|
|
#include <AggregateFunctions/UniquesHashSet.h>
|
2017-12-20 07:36:30 +00:00
|
|
|
#include <AggregateFunctions/IAggregateFunction.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <AggregateFunctions/UniqVariadicHash.h>
|
2011-09-26 04:00:46 +00:00
|
|
|
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2015-08-04 13:37:56 +00:00
|
|
|
/// uniq
|
2011-09-26 04:00:46 +00:00
|
|
|
|
2013-08-21 13:26:42 +00:00
|
|
|
struct AggregateFunctionUniqUniquesHashSetData
|
2013-02-08 19:34:44 +00:00
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
using Set = UniquesHashSet<DefaultHash<UInt64>>;
|
|
|
|
Set set;
|
2015-02-22 07:23:37 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
static String getName() { return "uniq"; }
|
2013-02-08 19:34:44 +00:00
|
|
|
};
|
|
|
|
|
2017-03-09 00:56:38 +00:00
|
|
|
/// For a function that takes multiple arguments. Such a function pre-hashes them in advance, so TrivialHash is used here.
|
2015-10-29 04:02:22 +00:00
|
|
|
struct AggregateFunctionUniqUniquesHashSetDataForVariadic
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
using Set = UniquesHashSet<TrivialHash>;
|
|
|
|
Set set;
|
2015-10-29 04:02:22 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
static String getName() { return "uniq"; }
|
2015-10-29 04:02:22 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
2015-08-04 13:37:56 +00:00
|
|
|
/// uniqHLL12
|
2014-02-02 09:08:06 +00:00
|
|
|
|
2015-02-22 07:23:37 +00:00
|
|
|
template <typename T>
|
2013-08-21 13:26:42 +00:00
|
|
|
struct AggregateFunctionUniqHLL12Data
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
using Set = HyperLogLogWithSmallSetOptimization<T, 16, 12>;
|
|
|
|
Set set;
|
2015-02-22 07:23:37 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
static String getName() { return "uniqHLL12"; }
|
2015-02-22 07:23:37 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
template <>
|
|
|
|
struct AggregateFunctionUniqHLL12Data<String>
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
using Set = HyperLogLogWithSmallSetOptimization<UInt64, 16, 12>;
|
|
|
|
Set set;
|
2015-02-22 07:23:37 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
static String getName() { return "uniqHLL12"; }
|
2013-08-21 13:26:42 +00:00
|
|
|
};
|
|
|
|
|
2017-11-09 23:37:57 +00:00
|
|
|
template <>
|
|
|
|
struct AggregateFunctionUniqHLL12Data<UInt128>
|
|
|
|
{
|
|
|
|
using Set = HyperLogLogWithSmallSetOptimization<UInt64, 16, 12>;
|
|
|
|
Set set;
|
|
|
|
|
|
|
|
static String getName() { return "uniqHLL12"; }
|
|
|
|
};
|
|
|
|
|
2015-10-29 04:13:18 +00:00
|
|
|
struct AggregateFunctionUniqHLL12DataForVariadic
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
using Set = HyperLogLogWithSmallSetOptimization<UInt64, 16, 12, TrivialHash>;
|
|
|
|
Set set;
|
2015-10-29 04:13:18 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
static String getName() { return "uniqHLL12"; }
|
2015-10-29 04:13:18 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
2015-08-04 13:37:56 +00:00
|
|
|
/// uniqExact
|
2013-02-08 19:34:44 +00:00
|
|
|
|
2014-02-02 09:08:06 +00:00
|
|
|
template <typename T>
|
|
|
|
struct AggregateFunctionUniqExactData
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
using Key = T;
|
2014-02-02 09:08:06 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
/// When creating, the hash table must be small.
|
2017-05-11 22:21:06 +00:00
|
|
|
using Set = HashSet<
|
2017-04-01 07:20:54 +00:00
|
|
|
Key,
|
|
|
|
HashCRC32<Key>,
|
|
|
|
HashTableGrower<4>,
|
2017-05-11 22:21:06 +00:00
|
|
|
HashTableAllocatorWithStackMemory<sizeof(Key) * (1 << 4)>>;
|
2014-02-02 09:08:06 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
Set set;
|
2014-02-02 09:08:06 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
static String getName() { return "uniqExact"; }
|
2014-02-02 09:08:06 +00:00
|
|
|
};
|
|
|
|
|
2017-03-09 04:18:41 +00:00
|
|
|
/// For rows, we put the SipHash values (128 bits) into the hash table.
|
2014-02-02 09:08:06 +00:00
|
|
|
template <>
|
|
|
|
struct AggregateFunctionUniqExactData<String>
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
using Key = UInt128;
|
2014-02-02 09:08:06 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
/// When creating, the hash table must be small.
|
2017-05-11 22:21:06 +00:00
|
|
|
using Set = HashSet<
|
2017-04-01 07:20:54 +00:00
|
|
|
Key,
|
|
|
|
UInt128TrivialHash,
|
|
|
|
HashTableGrower<3>,
|
2017-05-11 22:21:06 +00:00
|
|
|
HashTableAllocatorWithStackMemory<sizeof(Key) * (1 << 3)>>;
|
2014-02-02 09:08:06 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
Set set;
|
2014-02-02 09:08:06 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
static String getName() { return "uniqExact"; }
|
2014-02-02 09:08:06 +00:00
|
|
|
};
|
|
|
|
|
2018-03-17 18:02:19 +00:00
|
|
|
|
2015-08-31 13:52:17 +00:00
|
|
|
namespace detail
|
2015-08-04 13:37:56 +00:00
|
|
|
{
|
2015-07-29 17:06:39 +00:00
|
|
|
|
2017-03-09 00:56:38 +00:00
|
|
|
/** Hash function for uniq.
|
2015-08-04 13:37:56 +00:00
|
|
|
*/
|
|
|
|
template <typename T> struct AggregateFunctionUniqTraits
|
|
|
|
{
|
2020-08-19 11:52:17 +00:00
|
|
|
static UInt64 hash(T x)
|
2017-11-09 23:37:57 +00:00
|
|
|
{
|
2020-08-19 11:52:17 +00:00
|
|
|
if constexpr (std::is_same_v<T, UInt128>)
|
|
|
|
{
|
|
|
|
return sipHash64(x);
|
|
|
|
}
|
|
|
|
else if constexpr (std::is_same_v<T, Float32> || std::is_same_v<T, Float64>)
|
|
|
|
{
|
|
|
|
return ext::bit_cast<UInt64>(x);
|
|
|
|
}
|
|
|
|
else if constexpr (sizeof(T) <= sizeof(UInt64))
|
|
|
|
return x;
|
|
|
|
else
|
|
|
|
return DefaultHash64<T>(x);
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2015-08-04 13:37:56 +00:00
|
|
|
};
|
2015-08-04 12:33:08 +00:00
|
|
|
|
2017-12-02 03:42:21 +00:00
|
|
|
|
2017-03-09 00:56:38 +00:00
|
|
|
/** The structure for the delegation work to add one element to the `uniq` aggregate functions.
|
|
|
|
* Used for partial specialization to add strings.
|
2015-08-31 13:52:17 +00:00
|
|
|
*/
|
|
|
|
template <typename T, typename Data>
|
2017-12-02 03:42:21 +00:00
|
|
|
struct OneAdder
|
2015-08-04 13:37:56 +00:00
|
|
|
{
|
2017-12-20 07:36:30 +00:00
|
|
|
static void ALWAYS_INLINE add(Data & data, const IColumn & column, size_t row_num)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2017-12-25 04:01:46 +00:00
|
|
|
if constexpr (std::is_same_v<Data, AggregateFunctionUniqUniquesHashSetData>
|
|
|
|
|| std::is_same_v<Data, AggregateFunctionUniqHLL12Data<T>>)
|
2017-12-02 03:42:21 +00:00
|
|
|
{
|
2017-12-25 04:01:46 +00:00
|
|
|
if constexpr (!std::is_same_v<T, String>)
|
2017-12-02 03:42:21 +00:00
|
|
|
{
|
2019-09-30 15:45:53 +00:00
|
|
|
const auto & value = assert_cast<const ColumnVector<T> &>(column).getElement(row_num);
|
2017-12-02 03:42:21 +00:00
|
|
|
data.set.insert(AggregateFunctionUniqTraits<T>::hash(value));
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
StringRef value = column.getDataAt(row_num);
|
|
|
|
data.set.insert(CityHash_v1_0_2::CityHash64(value.data, value.size));
|
|
|
|
}
|
|
|
|
}
|
2017-12-25 04:01:46 +00:00
|
|
|
else if constexpr (std::is_same_v<Data, AggregateFunctionUniqExactData<T>>)
|
2017-12-02 03:42:21 +00:00
|
|
|
{
|
2017-12-25 04:01:46 +00:00
|
|
|
if constexpr (!std::is_same_v<T, String>)
|
2017-12-02 03:42:21 +00:00
|
|
|
{
|
2019-08-21 02:28:04 +00:00
|
|
|
data.set.insert(assert_cast<const ColumnVector<T> &>(column).getData()[row_num]);
|
2017-12-02 03:42:21 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
StringRef value = column.getDataAt(row_num);
|
|
|
|
|
|
|
|
UInt128 key;
|
|
|
|
SipHash hash;
|
|
|
|
hash.update(value.data, value.size);
|
|
|
|
hash.get128(key.low, key.high);
|
|
|
|
|
|
|
|
data.set.insert(key);
|
|
|
|
}
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2015-08-04 13:37:56 +00:00
|
|
|
};
|
2015-07-20 14:22:08 +00:00
|
|
|
|
2014-02-02 09:08:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-03-09 04:18:41 +00:00
|
|
|
/// Calculates the number of different values approximately or exactly.
|
2013-08-21 13:26:42 +00:00
|
|
|
template <typename T, typename Data>
|
2017-12-20 07:36:30 +00:00
|
|
|
class AggregateFunctionUniq final : public IAggregateFunctionDataHelper<Data, AggregateFunctionUniq<T, Data>>
|
2011-09-26 04:00:46 +00:00
|
|
|
{
|
|
|
|
public:
|
2019-02-13 11:50:41 +00:00
|
|
|
AggregateFunctionUniq(const DataTypes & argument_types_)
|
|
|
|
: IAggregateFunctionDataHelper<Data, AggregateFunctionUniq<T, Data>>(argument_types_, {}) {}
|
2019-02-11 19:26:32 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
String getName() const override { return Data::getName(); }
|
|
|
|
|
|
|
|
DataTypePtr getReturnType() const override
|
|
|
|
{
|
|
|
|
return std::make_shared<DataTypeUInt64>();
|
|
|
|
}
|
|
|
|
|
2019-11-11 08:36:19 +00:00
|
|
|
/// ALWAYS_INLINE is required to have better code layout for uniqHLL12 function
|
|
|
|
void ALWAYS_INLINE add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2017-12-20 07:36:30 +00:00
|
|
|
detail::OneAdder<T, Data>::add(this->data(place), *columns[0], row_num);
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
2017-12-01 21:51:50 +00:00
|
|
|
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
|
|
|
this->data(place).set.merge(this->data(rhs).set);
|
|
|
|
}
|
|
|
|
|
|
|
|
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
|
|
|
|
{
|
|
|
|
this->data(place).set.write(buf);
|
|
|
|
}
|
|
|
|
|
|
|
|
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
|
|
|
|
{
|
|
|
|
this->data(place).set.read(buf);
|
|
|
|
}
|
|
|
|
|
2020-06-17 19:36:27 +00:00
|
|
|
void insertResultInto(AggregateDataPtr place, IColumn & to, Arena *) const override
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2019-08-21 02:28:04 +00:00
|
|
|
assert_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).set.size());
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2013-08-21 13:26:42 +00:00
|
|
|
};
|
2013-06-25 14:16:16 +00:00
|
|
|
|
2013-04-26 18:57:08 +00:00
|
|
|
|
2017-03-09 00:56:38 +00:00
|
|
|
/** For multiple arguments. To compute, hashes them.
|
|
|
|
* You can pass multiple arguments as is; You can also pass one argument - a tuple.
|
2018-03-17 18:02:19 +00:00
|
|
|
* But (for the possibility of efficient implementation), you can not pass several arguments, among which there are tuples.
|
2015-10-29 02:13:37 +00:00
|
|
|
*/
|
2018-07-16 03:12:01 +00:00
|
|
|
template <typename Data, bool is_exact, bool argument_is_tuple>
|
|
|
|
class AggregateFunctionUniqVariadic final : public IAggregateFunctionDataHelper<Data, AggregateFunctionUniqVariadic<Data, is_exact, argument_is_tuple>>
|
2015-10-29 02:13:37 +00:00
|
|
|
{
|
|
|
|
private:
|
2017-04-01 07:20:54 +00:00
|
|
|
size_t num_args = 0;
|
2015-10-29 02:13:37 +00:00
|
|
|
|
|
|
|
public:
|
2017-12-20 20:25:22 +00:00
|
|
|
AggregateFunctionUniqVariadic(const DataTypes & arguments)
|
2019-02-12 09:31:20 +00:00
|
|
|
: IAggregateFunctionDataHelper<Data, AggregateFunctionUniqVariadic<Data, is_exact, argument_is_tuple>>(arguments, {})
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
|
|
|
if (argument_is_tuple)
|
|
|
|
num_args = typeid_cast<const DataTypeTuple &>(*arguments[0]).getElements().size();
|
|
|
|
else
|
|
|
|
num_args = arguments.size();
|
|
|
|
}
|
|
|
|
|
2017-12-20 07:36:30 +00:00
|
|
|
String getName() const override { return Data::getName(); }
|
|
|
|
|
|
|
|
DataTypePtr getReturnType() const override
|
|
|
|
{
|
|
|
|
return std::make_shared<DataTypeUInt64>();
|
|
|
|
}
|
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
|
|
|
|
{
|
2020-08-03 00:01:23 +00:00
|
|
|
this->data(place).set.insert(typename Data::Set::value_type(
|
|
|
|
UniqVariadicHash<is_exact, argument_is_tuple>::apply(num_args, columns, row_num)));
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
2017-12-01 21:51:50 +00:00
|
|
|
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs, Arena *) const override
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
|
|
|
this->data(place).set.merge(this->data(rhs).set);
|
|
|
|
}
|
|
|
|
|
|
|
|
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const override
|
|
|
|
{
|
|
|
|
this->data(place).set.write(buf);
|
|
|
|
}
|
|
|
|
|
|
|
|
void deserialize(AggregateDataPtr place, ReadBuffer & buf, Arena *) const override
|
|
|
|
{
|
|
|
|
this->data(place).set.read(buf);
|
|
|
|
}
|
|
|
|
|
2020-06-17 19:36:27 +00:00
|
|
|
void insertResultInto(AggregateDataPtr place, IColumn & to, Arena *) const override
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2019-08-21 02:28:04 +00:00
|
|
|
assert_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).set.size());
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2015-10-29 02:13:37 +00:00
|
|
|
};
|
|
|
|
|
2011-09-26 04:00:46 +00:00
|
|
|
}
|