mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 23:21:59 +00:00
Rename quantileDDSketch to quantileDD
This commit is contained in:
Alexey Milovidov
parent
b4a5b6060e
commit
798f655276
@ -88,7 +88,7 @@ ClickHouse-specific aggregate functions:
|
||||
- [quantileTDigestWeighted](/docs/en/sql-reference/aggregate-functions/reference/quantiletdigestweighted.md)
|
||||
- [quantileBFloat16](/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md#quantilebfloat16)
|
||||
- [quantileBFloat16Weighted](/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md#quantilebfloat16weighted)
|
||||
- [quantileDDSketch](/docs/en/sql-reference/aggregate-functions/reference/quantileddsketch.md#quantileddsketch)
|
||||
- [quantileDD](/docs/en/sql-reference/aggregate-functions/reference/quantileddsketch.md#quantileddsketch)
|
||||
- [simpleLinearRegression](/docs/en/sql-reference/aggregate-functions/reference/simplelinearregression.md)
|
||||
- [stochasticLinearRegression](/docs/en/sql-reference/aggregate-functions/reference/stochasticlinearregression.md)
|
||||
- [stochasticLogisticRegression](/docs/en/sql-reference/aggregate-functions/reference/stochasticlogisticregression.md)
|
||||
@ -105,4 +105,3 @@ ClickHouse-specific aggregate functions:
|
||||
- [sparkBar](./sparkbar.md)
|
||||
- [sumCount](./sumcount.md)
|
||||
- [largestTriangleThreeBuckets](./largestTriangleThreeBuckets.md)
|
||||
|
||||
|
||||
@ -18,7 +18,7 @@ Functions:
|
||||
- `medianTDigest` — Alias for [quantileTDigest](../../../sql-reference/aggregate-functions/reference/quantiletdigest.md#quantiletdigest).
|
||||
- `medianTDigestWeighted` — Alias for [quantileTDigestWeighted](../../../sql-reference/aggregate-functions/reference/quantiletdigestweighted.md#quantiletdigestweighted).
|
||||
- `medianBFloat16` — Alias for [quantileBFloat16](../../../sql-reference/aggregate-functions/reference/quantilebfloat16.md#quantilebfloat16).
|
||||
- `medianDDSketch` — Alias for [quantileDDSketch](../../../sql-reference/aggregate-functions/reference/quantileddsketch.md#quantileddsketch).
|
||||
- `medianDD` — Alias for [quantileDD](../../../sql-reference/aggregate-functions/reference/quantileddsketch.md#quantileddsketch).
|
||||
|
||||
**Example**
|
||||
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
---
|
||||
slug: /en/sql-reference/aggregate-functions/reference/quantileddsketch
|
||||
sidebar_position: 211
|
||||
title: quantileDDSketch
|
||||
title: quantileDD
|
||||
---
|
||||
|
||||
Computes an approximate [quantile](https://en.wikipedia.org/wiki/Quantile) of a sample with relative-error guarantees. It works by building a [DDSketch](https://www.vldb.org/pvldb/vol12/p2195-masson.pdf).
|
||||
Computes an approximate [quantile](https://en.wikipedia.org/wiki/Quantile) of a sample with relative-error guarantees. It works by building a [DD](https://www.vldb.org/pvldb/vol12/p2195-masson.pdf).
|
||||
|
||||
**Syntax**
|
||||
|
||||
@ -44,13 +44,13 @@ Input table has an integer and a float columns:
|
||||
Query to calculate 0.75-quantile (third quartile):
|
||||
|
||||
``` sql
|
||||
SELECT quantileDDSketch(0.01, 0.75)(a), quantileDDSketch(0.01, 0.75)(b) FROM example_table;
|
||||
SELECT quantileDD(0.01, 0.75)(a), quantileDD(0.01, 0.75)(b) FROM example_table;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─quantileDDSketch(0.01, 0.75)(a)─┬─quantileDDSketch(0.01, 0.75)(b)─┐
|
||||
┌─quantileDD(0.01, 0.75)(a)─┬─quantileDD(0.01, 0.75)(b)─┐
|
||||
│ 2.974233423476717 │ 1.01 │
|
||||
└─────────────────────────────────┴─────────────────────────────────┘
|
||||
```
|
||||
|
||||
@ -9,7 +9,7 @@ sidebar_position: 201
|
||||
|
||||
Syntax: `quantiles(level1, level2, …)(x)`
|
||||
|
||||
All the quantile functions also have corresponding quantiles functions: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantileInterpolatedWeighted`, `quantilesTDigest`, `quantilesBFloat16`, `quantilesDDSketch`. These functions calculate all the quantiles of the listed levels in one pass, and return an array of the resulting values.
|
||||
All the quantile functions also have corresponding quantiles functions: `quantiles`, `quantilesDeterministic`, `quantilesTiming`, `quantilesTimingWeighted`, `quantilesExact`, `quantilesExactWeighted`, `quantileInterpolatedWeighted`, `quantilesTDigest`, `quantilesBFloat16`, `quantilesDD`. These functions calculate all the quantiles of the listed levels in one pass, and return an array of the resulting values.
|
||||
|
||||
## quantilesExactExclusive
|
||||
|
||||
|
||||
@ -31,7 +31,7 @@ namespace ErrorCodes
|
||||
|
||||
template <typename> class QuantileTiming;
|
||||
template <typename> class QuantileGK;
|
||||
template <typename> class QuantileDDSketch;
|
||||
template <typename> class QuantileDD;
|
||||
|
||||
/** Generic aggregate function for calculation of quantiles.
|
||||
* It depends on quantile calculation data structure. Look at Quantile*.h for various implementations.
|
||||
@ -64,7 +64,7 @@ private:
|
||||
using ColVecType = ColumnVectorOrDecimal<Value>;
|
||||
|
||||
static constexpr bool returns_float = !(std::is_same_v<FloatReturnType, void>);
|
||||
static constexpr bool is_quantile_ddsketch = std::is_same_v<Data, QuantileDDSketch<Value>>;
|
||||
static constexpr bool is_quantile_ddsketch = std::is_same_v<Data, QuantileDD<Value>>;
|
||||
static_assert(!is_decimal<Value> || !returns_float);
|
||||
|
||||
QuantileLevels<Float64> levels;
|
||||
@ -334,7 +334,7 @@ struct NameQuantilesBFloat16Weighted { static constexpr auto name = "quantilesBF
|
||||
struct NameQuantileGK { static constexpr auto name = "quantileGK"; };
|
||||
struct NameQuantilesGK { static constexpr auto name = "quantilesGK"; };
|
||||
|
||||
struct NameQuantileDDSketch { static constexpr auto name = "quantileDDSketch"; };
|
||||
struct NameQuantilesDDSketch { static constexpr auto name = "quantilesDDSketch"; };
|
||||
struct NameQuantileDD { static constexpr auto name = "quantileDD"; };
|
||||
struct NameQuantilesDD { static constexpr auto name = "quantilesDD"; };
|
||||
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#include <AggregateFunctions/AggregateFunctionQuantile.h>
|
||||
#include <AggregateFunctions/QuantileDDSketch.h>
|
||||
#include <AggregateFunctions/QuantileDD.h>
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
#include <AggregateFunctions/Helpers.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
@ -19,8 +19,8 @@ namespace ErrorCodes
|
||||
namespace
|
||||
{
|
||||
|
||||
template <typename Value, bool float_return> using FuncQuantileDDSketch = AggregateFunctionQuantile<Value, QuantileDDSketch<Value>, NameQuantileDDSketch, false, std::conditional_t<float_return, Float64, void>, false, true>;
|
||||
template <typename Value, bool float_return> using FuncQuantilesDDSketch = AggregateFunctionQuantile<Value, QuantileDDSketch<Value>, NameQuantilesDDSketch, false, std::conditional_t<float_return, Float64, void>, true, true>;
|
||||
template <typename Value, bool float_return> using FuncQuantileDD = AggregateFunctionQuantile<Value, QuantileDD<Value>, NameQuantileDD, false, std::conditional_t<float_return, Float64, void>, false, true>;
|
||||
template <typename Value, bool float_return> using FuncQuantilesDD = AggregateFunctionQuantile<Value, QuantileDD<Value>, NameQuantilesDD, false, std::conditional_t<float_return, Float64, void>, true, true>;
|
||||
|
||||
|
||||
template <template <typename, bool> class Function>
|
||||
@ -46,16 +46,16 @@ AggregateFunctionPtr createAggregateFunctionQuantile(
|
||||
|
||||
}
|
||||
|
||||
void registerAggregateFunctionsQuantileDDSketch(AggregateFunctionFactory & factory)
|
||||
void registerAggregateFunctionsQuantileDD(AggregateFunctionFactory & factory)
|
||||
{
|
||||
/// For aggregate functions returning array we cannot return NULL on empty set.
|
||||
AggregateFunctionProperties properties = { .returns_default_when_only_null = true };
|
||||
|
||||
factory.registerFunction(NameQuantileDDSketch::name, createAggregateFunctionQuantile<FuncQuantileDDSketch>);
|
||||
factory.registerFunction(NameQuantilesDDSketch::name, { createAggregateFunctionQuantile<FuncQuantilesDDSketch>, properties });
|
||||
factory.registerFunction(NameQuantileDD::name, createAggregateFunctionQuantile<FuncQuantileDD>);
|
||||
factory.registerFunction(NameQuantilesDD::name, { createAggregateFunctionQuantile<FuncQuantilesDD>, properties });
|
||||
|
||||
/// 'median' is an alias for 'quantile'
|
||||
factory.registerAlias("medianDDSketch", NameQuantileDDSketch::name);
|
||||
factory.registerAlias("medianDD", NameQuantileDD::name);
|
||||
}
|
||||
|
||||
}
|
||||
@ -33,14 +33,14 @@ namespace DB
|
||||
*/
|
||||
|
||||
template <typename Value>
|
||||
class QuantileDDSketch
|
||||
class QuantileDD
|
||||
{
|
||||
public:
|
||||
using Weight = UInt64;
|
||||
|
||||
QuantileDDSketch() = default;
|
||||
QuantileDD() = default;
|
||||
|
||||
explicit QuantileDDSketch(Float64 relative_accuracy) : data(relative_accuracy) { }
|
||||
explicit QuantileDD(Float64 relative_accuracy) : data(relative_accuracy) { }
|
||||
|
||||
void add(const Value & x)
|
||||
{
|
||||
@ -53,7 +53,7 @@ public:
|
||||
data.add(x, w);
|
||||
}
|
||||
|
||||
void merge(const QuantileDDSketch &other)
|
||||
void merge(const QuantileDD &other)
|
||||
{
|
||||
data.merge(other.data);
|
||||
}
|
||||
@ -31,7 +31,7 @@ void registerAggregateFunctionsQuantileTimingWeighted(AggregateFunctionFactory &
|
||||
void registerAggregateFunctionsQuantileTDigest(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsQuantileTDigestWeighted(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsQuantileBFloat16(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsQuantileDDSketch(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsQuantileDD(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsQuantileBFloat16Weighted(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsQuantileApprox(AggregateFunctionFactory &);
|
||||
void registerAggregateFunctionsSequenceMatch(AggregateFunctionFactory &);
|
||||
@ -128,7 +128,7 @@ void registerAggregateFunctions()
|
||||
registerAggregateFunctionsQuantileTDigest(factory);
|
||||
registerAggregateFunctionsQuantileTDigestWeighted(factory);
|
||||
registerAggregateFunctionsQuantileBFloat16(factory);
|
||||
registerAggregateFunctionsQuantileDDSketch(factory);
|
||||
registerAggregateFunctionsQuantileDD(factory);
|
||||
registerAggregateFunctionsQuantileBFloat16Weighted(factory);
|
||||
registerAggregateFunctionsQuantileApprox(factory);
|
||||
registerAggregateFunctionsSequenceMatch(factory);
|
||||
|
||||
@ -2,13 +2,13 @@ SELECT quantiles(0.5)(x) FROM (SELECT number AS x FROM system.numbers LIMIT 1001
|
||||
SELECT quantilesExact(0.5)(x) FROM (SELECT number AS x FROM system.numbers LIMIT 1001);
|
||||
SELECT quantilesTDigest(0.5)(x) FROM (SELECT number AS x FROM system.numbers LIMIT 1001);
|
||||
SELECT quantilesDeterministic(0.5)(x, x) FROM (SELECT number AS x FROM system.numbers LIMIT 1001);
|
||||
SELECT arrayMap(a -> round(a, 2), quantilesDDSketch(0.01, 0.5)(x)) FROM (SELECT number AS x FROM system.numbers LIMIT 1001);
|
||||
SELECT arrayMap(a -> round(a, 2), quantilesDD(0.01, 0.5)(x)) FROM (SELECT number AS x FROM system.numbers LIMIT 1001);
|
||||
|
||||
SELECT quantiles(0, 0.001, 0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 0.999, 1)(x) FROM (SELECT number AS x FROM system.numbers LIMIT 1001);
|
||||
SELECT quantilesExact(0, 0.001, 0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 0.999, 1)(x) FROM (SELECT number AS x FROM system.numbers LIMIT 1001);
|
||||
SELECT quantilesTDigest(0, 0.001, 0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 0.999, 1)(x) FROM (SELECT number AS x FROM system.numbers LIMIT 1001);
|
||||
SELECT quantilesDeterministic(0, 0.001, 0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 0.999, 1)(x, x) FROM (SELECT number AS x FROM system.numbers LIMIT 1001);
|
||||
SELECT arrayMap(a -> round(a, 2), quantilesDDSketch(0.01, 0, 0.001, 0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 0.999, 1)(x)) FROM (SELECT number AS x FROM system.numbers LIMIT 1001);
|
||||
SELECT arrayMap(a -> round(a, 2), quantilesDD(0.01, 0, 0.001, 0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 0.999, 1)(x)) FROM (SELECT number AS x FROM system.numbers LIMIT 1001);
|
||||
|
||||
-- The result slightly differs but it's ok since `quantilesDeterministic` is an approximate function.
|
||||
SET max_bytes_before_external_group_by = 0;
|
||||
|
||||
@ -1,23 +1,23 @@
|
||||
SELECT '1'; -- simple test
|
||||
SELECT round(quantileDDSketch(0.01, 0.5)(number), 2) FROM numbers(200);
|
||||
SELECT round(quantileDDSketch(0.0001, 0.69)(number), 2) FROM numbers(500);
|
||||
SELECT round(quantileDDSketch(0.003, 0.42)(number), 2) FROM numbers(200);
|
||||
SELECT round(quantileDDSketch(0.02, 0.99)(number), 2) FROM numbers(500);
|
||||
SELECT round(quantileDD(0.01, 0.5)(number), 2) FROM numbers(200);
|
||||
SELECT round(quantileDD(0.0001, 0.69)(number), 2) FROM numbers(500);
|
||||
SELECT round(quantileDD(0.003, 0.42)(number), 2) FROM numbers(200);
|
||||
SELECT round(quantileDD(0.02, 0.99)(number), 2) FROM numbers(500);
|
||||
|
||||
SELECT '2'; -- median is close to 0
|
||||
SELECT round(quantileDDSketch(0.01, 0.5)(number), 2)
|
||||
SELECT round(quantileDD(0.01, 0.5)(number), 2)
|
||||
FROM
|
||||
(
|
||||
SELECT arrayJoin([toInt64(number), number - 10]) AS number
|
||||
FROM numbers(0, 10)
|
||||
);
|
||||
SELECT round(quantileDDSketch(0.01, 0.5)(number - 10), 2) FROM numbers(21);
|
||||
SELECT round(quantileDD(0.01, 0.5)(number - 10), 2) FROM numbers(21);
|
||||
|
||||
SELECT '3'; -- all values are negative
|
||||
SELECT round(quantileDDSketch(0.01, 0.99)(-number), 2) FROM numbers(1, 500);
|
||||
SELECT round(quantileDD(0.01, 0.99)(-number), 2) FROM numbers(1, 500);
|
||||
|
||||
SELECT '4'; -- min and max values of integer types (-2^63, 2^63-1)
|
||||
SELECT round(quantileDDSketch(0.01, 0.5)(number), 2)
|
||||
SELECT round(quantileDD(0.01, 0.5)(number), 2)
|
||||
FROM
|
||||
(
|
||||
SELECT arrayJoin([toInt64(number), number - 9223372036854775808, toInt64(number + 9223372036854775798)]) AS number
|
||||
@ -25,7 +25,7 @@ FROM
|
||||
);
|
||||
|
||||
SELECT '5'; -- min and max values of floating point types
|
||||
SELECT round(quantileDDSketch(0.01, 0.42)(number), 2)
|
||||
SELECT round(quantileDD(0.01, 0.42)(number), 2)
|
||||
FROM
|
||||
(
|
||||
SELECT arrayJoin([toFloat32(number), number - 3.4028235e+38, toFloat32(number + 3.4028235e+38)]) AS number
|
||||
@ -33,7 +33,7 @@ FROM
|
||||
);
|
||||
|
||||
SELECT '6'; -- denormalized floats
|
||||
SELECT round(quantileDDSketch(0.01, 0.69)(number), 2)
|
||||
SELECT round(quantileDD(0.01, 0.69)(number), 2)
|
||||
FROM
|
||||
(
|
||||
SELECT arrayJoin([toFloat32(number), number - 1.1754944e-38, toFloat32(number + 1.1754944e-38)]) AS number
|
||||
@ -41,7 +41,7 @@ FROM
|
||||
);
|
||||
|
||||
SELECT '7'; -- NaNs
|
||||
SELECT round(quantileDDSketch(0.01, 0.5)(number), 2)
|
||||
SELECT round(quantileDD(0.01, 0.5)(number), 2)
|
||||
FROM
|
||||
(
|
||||
SELECT arrayJoin([toFloat32(number), NaN * number]) AS number
|
||||
@ -50,7 +50,7 @@ FROM
|
||||
|
||||
SELECT '8'; -- sparse sketch
|
||||
|
||||
SELECT round(quantileDDSketch(0.01, 0.75)(number), 2)
|
||||
SELECT round(quantileDD(0.01, 0.75)(number), 2)
|
||||
FROM
|
||||
(
|
||||
SELECT number * 1e7 AS number
|
||||
@ -63,11 +63,11 @@ DROP TABLE IF EXISTS `02919_ddsketch_quantile`;
|
||||
|
||||
CREATE TABLE `02919_ddsketch_quantile`
|
||||
ENGINE = Log AS
|
||||
SELECT quantilesDDSketchState(0.001, 0.9)(number) AS sketch
|
||||
SELECT quantilesDDState(0.001, 0.9)(number) AS sketch
|
||||
FROM numbers(1000);
|
||||
|
||||
INSERT INTO `02919_ddsketch_quantile` SELECT quantilesDDSketchState(0.001, 0.9)(number + 1000)
|
||||
INSERT INTO `02919_ddsketch_quantile` SELECT quantilesDDState(0.001, 0.9)(number + 1000)
|
||||
FROM numbers(1000);
|
||||
|
||||
SELECT arrayMap(a -> round(a, 2), (quantilesDDSketchMerge(0.001, 0.9)(sketch)))
|
||||
SELECT arrayMap(a -> round(a, 2), (quantilesDDMerge(0.001, 0.9)(sketch)))
|
||||
FROM `02919_ddsketch_quantile`;
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
SELECT CounterID AS k, round(quantileDDSketch(0.01, 0.5)(ResolutionWidth), 2) FROM test.hits GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;
|
||||
SELECT CounterID AS k, arrayMap(a -> round(a, 2), quantilesDDSketch(0.01, 0.1, 0.5, 0.9, 0.99, 0.999)(ResolutionWidth)) FROM test.hits GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;
|
||||
SELECT CounterID AS k, round(quantileDD(0.01, 0.5)(ResolutionWidth), 2) FROM test.hits GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;
|
||||
SELECT CounterID AS k, arrayMap(a -> round(a, 2), quantilesDD(0.01, 0.1, 0.5, 0.9, 0.99, 0.999)(ResolutionWidth)) FROM test.hits GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;
|
||||
|
||||
SELECT CounterID AS k, round(quantileDDSketch(0.01, 0.5)(ResolutionWidth), 2) FROM remote('127.0.0.{1,2}', test.hits) GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;
|
||||
SELECT CounterID AS k, arrayMap(a -> round(a, 2), quantilesDDSketch(0.01, 0.1, 0.5, 0.9, 0.99, 0.999)(ResolutionWidth)) FROM remote('127.0.0.{1,2}', test.hits) GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;
|
||||
SELECT CounterID AS k, round(quantileDD(0.01, 0.5)(ResolutionWidth), 2) FROM remote('127.0.0.{1,2}', test.hits) GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;
|
||||
SELECT CounterID AS k, arrayMap(a -> round(a, 2), quantilesDD(0.01, 0.1, 0.5, 0.9, 0.99, 0.999)(ResolutionWidth)) FROM remote('127.0.0.{1,2}', test.hits) GROUP BY k ORDER BY count() DESC, CounterID LIMIT 10;
|
||||
|
||||
@ -214,7 +214,7 @@ DatabaseOrdinaryThreadsActive
|
||||
DateTime
|
||||
DateTimes
|
||||
DbCL
|
||||
DDSketch
|
||||
DD
|
||||
Decrypted
|
||||
Deduplicate
|
||||
Deduplication
|
||||
@ -2136,7 +2136,7 @@ quantiletdigestweighted
|
||||
quantiletiming
|
||||
quantiletimingweighted
|
||||
quantileddsketch
|
||||
quantileDDSketch
|
||||
quantileDD
|
||||
quartile
|
||||
queryID
|
||||
queryString
|
||||
|
||||
Loading…
Reference in New Issue
Block a user