2011-09-26 04:00:46 +00:00
|
|
|
|
#pragma once
|
|
|
|
|
|
|
|
|
|
#include <city.h>
|
|
|
|
|
|
|
|
|
|
#include <stats/UniquesHashSet.h>
|
2013-08-21 13:26:42 +00:00
|
|
|
|
#include <statdaemons/HyperLogLogCounter.h>
|
2011-09-26 04:00:46 +00:00
|
|
|
|
|
|
|
|
|
#include <DB/IO/WriteHelpers.h>
|
|
|
|
|
#include <DB/IO/ReadHelpers.h>
|
2013-04-26 18:57:08 +00:00
|
|
|
|
#include <DB/IO/WriteBufferFromString.h>
|
2011-09-26 04:00:46 +00:00
|
|
|
|
|
|
|
|
|
#include <DB/DataTypes/DataTypesNumberFixed.h>
|
2013-04-26 18:57:08 +00:00
|
|
|
|
#include <DB/DataTypes/DataTypeString.h>
|
2011-09-26 04:00:46 +00:00
|
|
|
|
|
2013-06-30 11:38:46 +00:00
|
|
|
|
#include <DB/Columns/ColumnString.h>
|
|
|
|
|
|
2011-09-26 04:00:46 +00:00
|
|
|
|
#include <DB/AggregateFunctions/IUnaryAggregateFunction.h>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
|
{
|
|
|
|
|
|
|
|
|
|
|
2013-06-25 14:16:16 +00:00
|
|
|
|
template <typename T> struct AggregateFunctionUniqTraits
|
2011-09-26 04:00:46 +00:00
|
|
|
|
{
|
2013-06-25 14:16:16 +00:00
|
|
|
|
static UInt64 hash(T x) { return x; }
|
2011-09-26 04:00:46 +00:00
|
|
|
|
};
|
|
|
|
|
|
2013-06-25 14:16:16 +00:00
|
|
|
|
template <> struct AggregateFunctionUniqTraits<Float32>
|
2011-09-26 04:00:46 +00:00
|
|
|
|
{
|
2013-06-25 14:16:16 +00:00
|
|
|
|
static UInt64 hash(Float32 x)
|
|
|
|
|
{
|
|
|
|
|
UInt64 res = 0;
|
|
|
|
|
memcpy(reinterpret_cast<char *>(&res), reinterpret_cast<char *>(&x), sizeof(x));
|
|
|
|
|
return res;
|
|
|
|
|
}
|
2011-09-26 04:00:46 +00:00
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
template <> struct AggregateFunctionUniqTraits<Float64>
|
|
|
|
|
{
|
|
|
|
|
static UInt64 hash(Float64 x)
|
|
|
|
|
{
|
|
|
|
|
UInt64 res = 0;
|
|
|
|
|
memcpy(reinterpret_cast<char *>(&res), reinterpret_cast<char *>(&x), sizeof(x));
|
|
|
|
|
return res;
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
2013-08-21 13:26:42 +00:00
|
|
|
|
struct AggregateFunctionUniqUniquesHashSetData
|
2013-02-08 19:34:44 +00:00
|
|
|
|
{
|
2013-08-21 13:26:42 +00:00
|
|
|
|
typedef UniquesHashSet Set;
|
|
|
|
|
Set set;
|
|
|
|
|
|
|
|
|
|
static String getName() { return "uniq"; }
|
2013-02-08 19:34:44 +00:00
|
|
|
|
};
|
|
|
|
|
|
2013-08-21 13:26:42 +00:00
|
|
|
|
struct AggregateFunctionUniqHLL12Data
|
|
|
|
|
{
|
2013-08-22 10:58:52 +00:00
|
|
|
|
typedef HLL12 Set;
|
2013-08-21 13:26:42 +00:00
|
|
|
|
Set set;
|
|
|
|
|
|
|
|
|
|
static String getName() { return "uniqHLL12"; }
|
|
|
|
|
};
|
|
|
|
|
|
2013-02-08 19:34:44 +00:00
|
|
|
|
|
2011-09-26 04:00:46 +00:00
|
|
|
|
/// Приближённо вычисляет количество различных значений.
|
2013-08-21 13:26:42 +00:00
|
|
|
|
template <typename T, typename Data>
|
|
|
|
|
class AggregateFunctionUniq : public IUnaryAggregateFunction<Data>
|
2011-09-26 04:00:46 +00:00
|
|
|
|
{
|
|
|
|
|
public:
|
|
|
|
|
AggregateFunctionUniq() {}
|
|
|
|
|
|
2013-08-21 13:26:42 +00:00
|
|
|
|
String getName() const { return Data::getName(); }
|
2011-09-26 04:00:46 +00:00
|
|
|
|
|
|
|
|
|
DataTypePtr getReturnType() const
|
|
|
|
|
{
|
2012-12-14 20:24:26 +00:00
|
|
|
|
return new DataTypeUInt64;
|
2011-09-26 04:00:46 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void setArgument(const DataTypePtr & argument)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
|
2013-06-25 08:06:40 +00:00
|
|
|
|
void addOne(AggregateDataPtr place, const IColumn & column, size_t row_num) const
|
2011-09-26 04:00:46 +00:00
|
|
|
|
{
|
2013-08-21 13:26:42 +00:00
|
|
|
|
OneAdder<T, Data>::addOne(*this, place, column, row_num);
|
2011-09-26 04:00:46 +00:00
|
|
|
|
}
|
|
|
|
|
|
2013-02-08 19:34:44 +00:00
|
|
|
|
void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs) const
|
2011-09-26 04:00:46 +00:00
|
|
|
|
{
|
2013-08-21 13:26:42 +00:00
|
|
|
|
this->data(place).set.merge(this->data(rhs).set);
|
2011-09-26 04:00:46 +00:00
|
|
|
|
}
|
|
|
|
|
|
2013-02-08 19:34:44 +00:00
|
|
|
|
void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const
|
2011-09-26 04:00:46 +00:00
|
|
|
|
{
|
2013-08-21 13:26:42 +00:00
|
|
|
|
this->data(place).set.write(buf);
|
2011-09-26 04:00:46 +00:00
|
|
|
|
}
|
|
|
|
|
|
2013-02-08 19:34:44 +00:00
|
|
|
|
void deserializeMerge(AggregateDataPtr place, ReadBuffer & buf) const
|
2011-09-26 04:00:46 +00:00
|
|
|
|
{
|
2013-08-21 13:26:42 +00:00
|
|
|
|
typename Data::Set tmp_set;
|
2011-09-26 04:00:46 +00:00
|
|
|
|
tmp_set.read(buf);
|
2013-08-21 13:26:42 +00:00
|
|
|
|
this->data(place).set.merge(tmp_set);
|
2011-09-26 04:00:46 +00:00
|
|
|
|
}
|
|
|
|
|
|
2013-06-30 11:38:46 +00:00
|
|
|
|
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const
|
2011-09-26 04:00:46 +00:00
|
|
|
|
{
|
2013-08-21 13:26:42 +00:00
|
|
|
|
static_cast<ColumnUInt64 &>(to).getData().push_back(this->data(place).set.size());
|
2011-09-26 04:00:46 +00:00
|
|
|
|
}
|
2013-08-21 13:26:42 +00:00
|
|
|
|
|
|
|
|
|
private:
|
2013-08-22 11:06:58 +00:00
|
|
|
|
/// Структура для делегации работы по добавлению одного элемента
|
|
|
|
|
/// в аггрегатную функцию uniq. Используется для частичной специализации
|
|
|
|
|
/// для добавления строк.
|
|
|
|
|
template<typename T0, typename Data0>
|
|
|
|
|
struct OneAdder
|
2013-08-21 13:26:42 +00:00
|
|
|
|
{
|
2013-08-22 11:06:58 +00:00
|
|
|
|
static void addOne(const AggregateFunctionUniq<T0, Data0> & aggregate_function,
|
|
|
|
|
AggregateDataPtr place, const IColumn & column, size_t row_num)
|
|
|
|
|
{
|
|
|
|
|
aggregate_function.data(place).set.insert(
|
|
|
|
|
AggregateFunctionUniqTraits<T0>::hash(static_cast<const ColumnVector<T0> &>(column).getData()[row_num]));
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
template<typename Data0>
|
|
|
|
|
struct OneAdder<String, Data0>
|
2013-08-21 13:26:42 +00:00
|
|
|
|
{
|
2013-08-22 11:06:58 +00:00
|
|
|
|
static void addOne(const AggregateFunctionUniq<String, Data0> & aggregate_function,
|
|
|
|
|
AggregateDataPtr place, const IColumn & column, size_t row_num)
|
|
|
|
|
{
|
|
|
|
|
/// Имейте ввиду, что вычисление приближённое.
|
|
|
|
|
StringRef value = column.getDataAt(row_num);
|
|
|
|
|
aggregate_function.data(place).set.insert(CityHash64(value.data, value.size));
|
|
|
|
|
}
|
|
|
|
|
};
|
2013-08-21 13:26:42 +00:00
|
|
|
|
};
|
2013-06-25 14:16:16 +00:00
|
|
|
|
|
2013-04-26 18:57:08 +00:00
|
|
|
|
|
|
|
|
|
/** То же самое, но выводит состояние вычислений в строке в текстовом виде.
|
|
|
|
|
* Используется, если какой-то внешней программе (сейчас это ███████████)
|
|
|
|
|
* надо получить это состояние и потом использовать по-своему.
|
|
|
|
|
*/
|
2013-08-21 13:26:42 +00:00
|
|
|
|
template <typename T, typename Data>
|
|
|
|
|
class AggregateFunctionUniqState : public AggregateFunctionUniq<T, Data>
|
2013-04-26 18:57:08 +00:00
|
|
|
|
{
|
|
|
|
|
public:
|
2013-08-21 13:26:42 +00:00
|
|
|
|
String getName() const { return Data::getName() + "State"; }
|
2013-04-26 18:57:08 +00:00
|
|
|
|
|
|
|
|
|
DataTypePtr getReturnType() const
|
|
|
|
|
{
|
|
|
|
|
return new DataTypeString;
|
|
|
|
|
}
|
|
|
|
|
|
2013-06-30 11:38:46 +00:00
|
|
|
|
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const
|
2013-04-26 18:57:08 +00:00
|
|
|
|
{
|
2013-06-30 11:38:46 +00:00
|
|
|
|
String res;
|
2013-06-30 11:40:30 +00:00
|
|
|
|
{
|
|
|
|
|
WriteBufferFromString wb(res);
|
|
|
|
|
this->data(place).set.writeText(wb);
|
|
|
|
|
}
|
2013-06-30 11:38:46 +00:00
|
|
|
|
|
|
|
|
|
static_cast<ColumnString &>(to).insertDataWithTerminatingZero(res.data(), res.size() + 1);
|
2013-04-26 18:57:08 +00:00
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
2013-06-13 20:30:47 +00:00
|
|
|
|
|
2011-09-26 04:00:46 +00:00
|
|
|
|
}
|