thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-24 16:42:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix undefined behavior in quantiles function

Browse Source
This commit is contained in:
Alexey Milovidov 2022-12-09 04:17:54 +01:00
parent f75b203071
commit 6f69200662
7 changed files with 67 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/AggregateFunctions/AggregateFunctionQuantile.h
View File

@ -170,7 +170,7 @@ public:
{
auto & data_to = assert_cast<ColumnVector<FloatReturnType> &>(arr_to.getData()).getData();
size_t old_size = data_to.size();
data_to.resize(data_to.size() + size);
data_to.resize(old_size + size);
data.getManyFloat(levels.levels.data(), levels.permutation.data(), size, data_to.data() + old_size);
}
@ -178,7 +178,7 @@ public:
{
auto & data_to = static_cast<ColVecType &>(arr_to.getData()).getData();
size_t old_size = data_to.size();
data_to.resize(data_to.size() + size);
data_to.resize(old_size + size);
data.getMany(levels.levels.data(), levels.permutation.data(), size, data_to.data() + old_size);
}

17
src/AggregateFunctions/QuantileReservoirSampler.h
View File

@ -65,11 +65,22 @@ struct QuantileReservoirSampler
/// indices - an array of index levels such that the corresponding elements will go in ascending order.
void getMany(const Float64 * levels, const size_t * indices, size_t size, Value * result)
{
bool is_empty = data.empty();
for (size_t i = 0; i < size; ++i)
if constexpr (is_decimal<Value>)
result[indices[i]] = Value(static_cast<typename Value::NativeType>(data.quantileInterpolated(levels[indices[i]])));
{
if (is_empty)
{
result[i] = Value{};
}
else
result[indices[i]] = Value(data.quantileInterpolated(levels[indices[i]]));
{
if constexpr (is_decimal<Value>)
result[indices[i]] = Value(static_cast<typename Value::NativeType>(data.quantileInterpolated(levels[indices[i]])));
else
result[indices[i]] = Value(data.quantileInterpolated(levels[indices[i]]));
}
}
}
/// The same, but in the case of an empty state, NaN is returned.

17
src/AggregateFunctions/QuantileReservoirSamplerDeterministic.h
View File

@ -65,11 +65,22 @@ struct QuantileReservoirSamplerDeterministic
/// indices - an array of index levels such that the corresponding elements will go in ascending order.
void getMany(const Float64 * levels, const size_t * indices, size_t size, Value * result)
{
bool is_empty = data.empty();
for (size_t i = 0; i < size; ++i)
if constexpr (is_decimal<Value>)
result[indices[i]] = static_cast<typename Value::NativeType>(data.quantileInterpolated(levels[indices[i]]));
{
if (is_empty)
{
result[i] = Value{};
}
else
result[indices[i]] = static_cast<Value>(data.quantileInterpolated(levels[indices[i]]));
{
if constexpr (is_decimal<Value>)
result[indices[i]] = static_cast<typename Value::NativeType>(data.quantileInterpolated(levels[indices[i]]));
else
result[indices[i]] = static_cast<Value>(data.quantileInterpolated(levels[indices[i]]));
}
}
}
/// The same, but in the case of an empty state, NaN is returned.

5
src/AggregateFunctions/ReservoirSampler.h
View File

@ -103,6 +103,11 @@ public:
return total_values;
}
bool empty() const
{
return samples.empty();
}
T quantileNearest(double level)
{
if (samples.empty())

5
src/AggregateFunctions/ReservoirSamplerDeterministic.h
View File

@ -95,6 +95,11 @@ public:
return total_values;
}
bool empty() const
{
return samples.empty();
}
T quantileNearest(double level)
{
if (samples.empty())

15
tests/queries/0_stateless/02499_quantile_nan_ubsan_msan.reference Normal file
View File

@ -0,0 +1,15 @@
['1970-01-01 01:00:00']
['1970-01-01 01:00:00']
['1970-01-01 01:00:00']
['1970-01-01 01:00:00']
['1970-01-01 01:00:00','1970-01-01 01:00:00','1970-01-01 01:00:00','1970-01-01 01:00:00','1970-01-01 01:00:00','1970-01-01 01:00:00','1970-01-01 01:00:00','1970-01-01 01:00:00','1970-01-01 01:00:00']
['1970-01-01 01:00:00','1970-01-01 01:00:00','1970-01-01 01:00:00','1970-01-01 01:00:00','1970-01-01 01:00:00','1970-01-01 01:00:00','1970-01-01 01:00:00','1970-01-01 01:00:00','1970-01-01 01:00:00']
['1970-01-01 01:00:00']
['1970-01-01 01:00:00','1970-01-01 01:00:00','1970-01-01 01:00:00','1970-01-01 01:00:00']
[18446744073709552000]
['1970-01-01 01:00:00']
['1970-01-01 01:00:00','1970-01-01 01:00:00','1970-01-01 01:00:00','1970-01-01 01:00:00']
[1.157920892373162e77]
[nan]

12
tests/queries/0_stateless/02499_quantile_nan_ubsan_msan.sql Normal file
View File

@ -0,0 +1,12 @@
SELECT quantiles(0.5)(now()) WHERE 0;
SELECT quantiles(0.5)(now()) WHERE 0 WITH TOTALS;
SELECT arrayReduce('quantiles(0.5)', []::Array(DateTime));
SELECT quantiles(0.5, 1.1754943508222875e-38, 0.0001, -0., 0.0001, -0., 0.0001, 0., 0.5)(now()) WHERE 0 WITH TOTALS;
SELECT DISTINCT arrayReduce('quantiles(0.5)', materialize([]::Array(DateTime))) FROM numbers(1000) LIMIT 10;
SELECT DISTINCT arrayReduce('quantiles(0, 0.5, 0.9, 1)', materialize([]::Array(DateTime))) FROM numbers(1000) LIMIT 10;
SELECT DISTINCT arrayReduce('quantiles(0.5)', [0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFE]) FROM numbers(1000) LIMIT 10;
SELECT DISTINCT arrayReduce('quantilesDeterministic(0.5)', materialize([]::Array(DateTime)), []::Array(UInt64)) FROM numbers(1000) LIMIT 10;
SELECT DISTINCT arrayReduce('quantilesDeterministic(0, 0.5, 0.9, 1)', materialize([]::Array(DateTime)), []::Array(UInt64)) FROM numbers(1000) LIMIT 10;
SELECT DISTINCT arrayReduce('quantiles(0.5)', [CAST(-1, 'UInt256'), CAST(-2, 'UInt256')]) FROM numbers(1000) LIMIT 10;
SELECT DISTINCT arrayReduce('quantiles(0.5)', []::Array(Float64)) FROM numbers(1000) LIMIT 10;