thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix UBSan report in quantileExactWeighted

Browse Source
This commit is contained in:
Alexey Milovidov 2021-01-22 03:05:45 +03:00
parent b0fca03d79
commit 14adc2d5f0
3 changed files with 17 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/AggregateFunctions/QuantileExactWeighted.h
View File

@ -84,8 +84,15 @@ struct QuantileExactWeighted
std::unique_ptr<Pair[]> array_holder(new Pair[size]); std::unique_ptr<Pair[]> array_holder(new Pair[size]);
Pair * array = array_holder.get(); Pair * array = array_holder.get();
/// Note: 64-bit integer weight can overflow.
/// We do some implementation specific behaviour (return approximate or garbage results).
/// Float64 is used as accumulator here to get approximate results.
/// But weight can be already overflowed in computations in 'add' and 'merge' methods.
/// It will be reasonable to change the type of weight to Float64 in the map,
/// but we don't do that for compatibility of serialized data.
size_t i = 0; size_t i = 0;
UInt64 sum_weight = 0; Float64 sum_weight = 0;
for (const auto & pair : map) for (const auto & pair : map)
{ {
sum_weight += pair.getMapped(); sum_weight += pair.getMapped();
@ -95,8 +102,8 @@ struct QuantileExactWeighted
std::sort(array, array + size, [](const Pair & a, const Pair & b) { return a.first < b.first; }); std::sort(array, array + size, [](const Pair & a, const Pair & b) { return a.first < b.first; });
UInt64 threshold = std::ceil(sum_weight * level); Float64 threshold = std::ceil(sum_weight * level);
UInt64 accumulated = 0; Float64 accumulated = 0;
const Pair * it = array; const Pair * it = array;
const Pair * end = array + size; const Pair * end = array + size;
@ -135,7 +142,7 @@ struct QuantileExactWeighted
Pair * array = array_holder.get(); Pair * array = array_holder.get();
size_t i = 0; size_t i = 0;
UInt64 sum_weight = 0; Float64 sum_weight = 0;
for (const auto & pair : map) for (const auto & pair : map)
{ {
sum_weight += pair.getMapped(); sum_weight += pair.getMapped();
@ -145,13 +152,13 @@ struct QuantileExactWeighted
std::sort(array, array + size, [](const Pair & a, const Pair & b) { return a.first < b.first; }); std::sort(array, array + size, [](const Pair & a, const Pair & b) { return a.first < b.first; });
UInt64 accumulated = 0; Float64 accumulated = 0;
const Pair * it = array; const Pair * it = array;
const Pair * end = array + size; const Pair * end = array + size;
size_t level_index = 0; size_t level_index = 0;
UInt64 threshold = std::ceil(sum_weight * levels[indices[level_index]]); Float64 threshold = std::ceil(sum_weight * levels[indices[level_index]]);
while (it < end) while (it < end)
{ {

2
tests/queries/0_stateless/01663_quantile_weighted_overflow.reference Normal file
View File

@ -0,0 +1,2 @@
5
\N [0,9,9,0,0,0,9,0,0,4,9,9,0,0,9,9]

2
tests/queries/0_stateless/01663_quantile_weighted_overflow.sql Normal file
View File

@ -0,0 +1,2 @@
SELECT quantileExactWeighted(1)(number, 9223372036854775807) FROM numbers(6);
SELECT quantileExactWeighted((NULL, NULL, NULL, NULL))((NULL), number, (NULL, 0, 255), 9223372036854775807), quantilesExactWeighted(0, 1., 0.9998999834060669, 0.00009999999747378752, 0., 0., 0.9998999834060669, 0., 0.00009999999747378752, 0.5, 1., 0.9998999834060669, 0.00009999999747378752, 0., 1., 1)(number, 9223372036854775807) FROM (SELECT (NULL, NULL, NULL, 0.9998999834060669), number FROM system.numbers LIMIT 10);