thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 01:22:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #19394 from ClickHouse/quantile-weighted-overflow

Browse Source 
Fix UBSan report in quantileExactWeighted
This commit is contained in:
alexey-milovidov 2021-01-22 17:18:01 +03:00 committed by GitHub
commit 9add38f47b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 17 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/AggregateFunctions/QuantileExactWeighted.h
View File

@ -84,8 +84,15 @@ struct QuantileExactWeighted
std::unique_ptr<Pair[]> array_holder(new Pair[size]);
Pair * array = array_holder.get();
/// Note: 64-bit integer weight can overflow.
/// We do some implementation specific behaviour (return approximate or garbage results).
/// Float64 is used as accumulator here to get approximate results.
/// But weight can be already overflowed in computations in 'add' and 'merge' methods.
/// It will be reasonable to change the type of weight to Float64 in the map,
/// but we don't do that for compatibility of serialized data.
size_t i = 0;
UInt64 sum_weight = 0;
Float64 sum_weight = 0;
for (const auto & pair : map)
{
sum_weight += pair.getMapped();
@ -95,8 +102,8 @@ struct QuantileExactWeighted
std::sort(array, array + size, [](const Pair & a, const Pair & b) { return a.first < b.first; });
UInt64 threshold = std::ceil(sum_weight * level);
UInt64 accumulated = 0;
Float64 threshold = std::ceil(sum_weight * level);
Float64 accumulated = 0;
const Pair * it = array;
const Pair * end = array + size;
@ -135,7 +142,7 @@ struct QuantileExactWeighted
Pair * array = array_holder.get();
size_t i = 0;
UInt64 sum_weight = 0;
Float64 sum_weight = 0;
for (const auto & pair : map)
{
sum_weight += pair.getMapped();
@ -145,13 +152,13 @@ struct QuantileExactWeighted
std::sort(array, array + size, [](const Pair & a, const Pair & b) { return a.first < b.first; });
UInt64 accumulated = 0;
Float64 accumulated = 0;
const Pair * it = array;
const Pair * end = array + size;
size_t level_index = 0;
UInt64 threshold = std::ceil(sum_weight * levels[indices[level_index]]);
Float64 threshold = std::ceil(sum_weight * levels[indices[level_index]]);
while (it < end)
{

2
tests/queries/0_stateless/01663_quantile_weighted_overflow.reference Normal file
View File

@ -0,0 +1,2 @@
5
\N [0,9,9,0,0,0,9,0,0,4,9,9,0,0,9,9]

2
tests/queries/0_stateless/01663_quantile_weighted_overflow.sql Normal file
View File

@ -0,0 +1,2 @@
SELECT quantileExactWeighted(1)(number, 9223372036854775807) FROM numbers(6);
SELECT quantileExactWeighted((NULL, NULL, NULL, NULL))((NULL), number, (NULL, 0, 255), 9223372036854775807), quantilesExactWeighted(0, 1., 0.9998999834060669, 0.00009999999747378752, 0., 0., 0.9998999834060669, 0., 0.00009999999747378752, 0.5, 1., 0.9998999834060669, 0.00009999999747378752, 0., 1., 1)(number, 9223372036854775807) FROM (SELECT (NULL, NULL, NULL, 0.9998999834060669), number FROM system.numbers LIMIT 10);