thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #44067 from ClickHouse/fix-quantiles-msan

Browse Source 
Fix undefined behavior in `quantiles` function
This commit is contained in:
Alexey Milovidov 2022-12-12 04:19:00 +03:00 committed by GitHub
commit bca18298a1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 93 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/AggregateFunctions/AggregateFunctionQuantile.h
View File

@ -170,7 +170,7 @@ public:
{
auto & data_to = assert_cast<ColumnVector<FloatReturnType> &>(arr_to.getData()).getData();
size_t old_size = data_to.size();
data_to.resize(data_to.size() + size);
data_to.resize(old_size + size);
data.getManyFloat(levels.levels.data(), levels.permutation.data(), size, data_to.data() + old_size);
}
@ -178,7 +178,7 @@ public:
{
auto & data_to = static_cast<ColVecType &>(arr_to.getData()).getData();
size_t old_size = data_to.size();
data_to.resize(data_to.size() + size);
data_to.resize(old_size + size);
data.getMany(levels.levels.data(), levels.permutation.data(), size, data_to.data() + old_size);
}

20
src/AggregateFunctions/QuantileReservoirSampler.h
View File

@ -55,6 +55,9 @@ struct QuantileReservoirSampler
/// Get the value of the `level` quantile. The level must be between 0 and 1.
Value get(Float64 level)
{
if (data.empty())
return {};
if constexpr (is_decimal<Value>)
return Value(static_cast<typename Value::NativeType>(data.quantileInterpolated(level)));
else
@ -65,11 +68,22 @@ struct QuantileReservoirSampler
/// indices - an array of index levels such that the corresponding elements will go in ascending order.
void getMany(const Float64 * levels, const size_t * indices, size_t size, Value * result)
{
bool is_empty = data.empty();
for (size_t i = 0; i < size; ++i)
if constexpr (is_decimal<Value>)
result[indices[i]] = Value(static_cast<typename Value::NativeType>(data.quantileInterpolated(levels[indices[i]])));
{
if (is_empty)
{
result[i] = Value{};
}
else
result[indices[i]] = Value(data.quantileInterpolated(levels[indices[i]]));
{
if constexpr (is_decimal<Value>)
result[indices[i]] = Value(static_cast<typename Value::NativeType>(data.quantileInterpolated(levels[indices[i]])));
else
result[indices[i]] = Value(data.quantileInterpolated(levels[indices[i]]));
}
}
}
/// The same, but in the case of an empty state, NaN is returned.

20
src/AggregateFunctions/QuantileReservoirSamplerDeterministic.h
View File

@ -55,6 +55,9 @@ struct QuantileReservoirSamplerDeterministic
/// Get the value of the `level` quantile. The level must be between 0 and 1.
Value get(Float64 level)
{
if (data.empty())
return {};
if constexpr (is_decimal<Value>)
return static_cast<typename Value::NativeType>(data.quantileInterpolated(level));
else
@ -65,11 +68,22 @@ struct QuantileReservoirSamplerDeterministic
/// indices - an array of index levels such that the corresponding elements will go in ascending order.
void getMany(const Float64 * levels, const size_t * indices, size_t size, Value * result)
{
bool is_empty = data.empty();
for (size_t i = 0; i < size; ++i)
if constexpr (is_decimal<Value>)
result[indices[i]] = static_cast<typename Value::NativeType>(data.quantileInterpolated(levels[indices[i]]));
{
if (is_empty)
{
result[i] = Value{};
}
else
result[indices[i]] = static_cast<Value>(data.quantileInterpolated(levels[indices[i]]));
{
if constexpr (is_decimal<Value>)
result[indices[i]] = static_cast<typename Value::NativeType>(data.quantileInterpolated(levels[indices[i]]));
else
result[indices[i]] = static_cast<Value>(data.quantileInterpolated(levels[indices[i]]));
}
}
}
/// The same, but in the case of an empty state, NaN is returned.

5
src/AggregateFunctions/ReservoirSampler.h
View File

@ -103,6 +103,11 @@ public:
return total_values;
}
bool empty() const
{
return samples.empty();
}
T quantileNearest(double level)
{
if (samples.empty())

5
src/AggregateFunctions/ReservoirSamplerDeterministic.h
View File

@ -95,6 +95,11 @@ public:
return total_values;
}
bool empty() const
{
return samples.empty();
}
T quantileNearest(double level)
{
if (samples.empty())

25
tests/queries/0_stateless/02499_quantile_nan_ubsan_msan.reference Normal file
View File

@ -0,0 +1,25 @@
['1970-01-01 00:00:00']
['1970-01-01 00:00:00']
['1970-01-01 00:00:00']
['1970-01-01 00:00:00']
['1970-01-01 00:00:00','1970-01-01 00:00:00','1970-01-01 00:00:00','1970-01-01 00:00:00','1970-01-01 00:00:00','1970-01-01 00:00:00','1970-01-01 00:00:00','1970-01-01 00:00:00','1970-01-01 00:00:00']
['1970-01-01 00:00:00','1970-01-01 00:00:00','1970-01-01 00:00:00','1970-01-01 00:00:00','1970-01-01 00:00:00','1970-01-01 00:00:00','1970-01-01 00:00:00','1970-01-01 00:00:00','1970-01-01 00:00:00']
['1970-01-01 00:00:00']
['1970-01-01 00:00:00','1970-01-01 00:00:00','1970-01-01 00:00:00','1970-01-01 00:00:00']
[18446744073709552000]
['1970-01-01 00:00:00']
['1970-01-01 00:00:00','1970-01-01 00:00:00','1970-01-01 00:00:00','1970-01-01 00:00:00']
[1.157920892373162e77]
[nan]
1970-01-01 00:00:00
1970-01-01 00:00:00
1970-01-01 00:00:00
1970-01-01 00:00:00
1970-01-01 00:00:00
18446744073709552000
1970-01-01 00:00:00
1.157920892373162e77
nan

22
tests/queries/0_stateless/02499_quantile_nan_ubsan_msan.sql Normal file
View File

@ -0,0 +1,22 @@
SELECT quantiles(0.5)(now()::DateTime('UTC')) WHERE 0;
SELECT quantiles(0.5)(now()::DateTime('UTC')) WHERE 0 WITH TOTALS;
SELECT arrayReduce('quantiles(0.5)', []::Array(DateTime('UTC')));
SELECT quantiles(0.5, 1.1754943508222875e-38, 0.0001, -0., 0.0001, -0., 0.0001, 0., 0.5)(now()::DateTime('UTC')) WHERE 0 WITH TOTALS;
SELECT DISTINCT arrayReduce('quantiles(0.5)', materialize([]::Array(DateTime('UTC')))) FROM numbers(1000) LIMIT 10;
SELECT DISTINCT arrayReduce('quantiles(0, 0.5, 0.9, 1)', materialize([]::Array(DateTime('UTC')))) FROM numbers(1000) LIMIT 10;
SELECT DISTINCT arrayReduce('quantiles(0.5)', [0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFE]) FROM numbers(1000) LIMIT 10;
SELECT DISTINCT arrayReduce('quantilesDeterministic(0.5)', materialize([]::Array(DateTime('UTC'))), []::Array(UInt64)) FROM numbers(1000) LIMIT 10;
SELECT DISTINCT arrayReduce('quantilesDeterministic(0, 0.5, 0.9, 1)', materialize([]::Array(DateTime('UTC'))), []::Array(UInt64)) FROM numbers(1000) LIMIT 10;
SELECT DISTINCT arrayReduce('quantiles(0.5)', [CAST(-1, 'UInt256'), CAST(-2, 'UInt256')]) FROM numbers(1000) LIMIT 10;
SELECT DISTINCT arrayReduce('quantiles(0.5)', []::Array(Float64)) FROM numbers(1000) LIMIT 10;
SELECT quantile(0.5)(now()::DateTime('UTC')) WHERE 0;
SELECT quantile(0.5)(now()::DateTime('UTC')) WHERE 0 WITH TOTALS;
SELECT arrayReduce('quantile(0.5)', []::Array(DateTime('UTC')));
SELECT DISTINCT arrayReduce('quantile(0.5)', materialize([]::Array(DateTime('UTC')))) FROM numbers(1000) LIMIT 10;
SELECT DISTINCT arrayReduce('quantile(0.5)', [0xFFFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFFE]) FROM numbers(1000) LIMIT 10;
SELECT DISTINCT arrayReduce('quantileDeterministic(0.5)', materialize([]::Array(DateTime('UTC'))), []::Array(UInt64)) FROM numbers(1000) LIMIT 10;
SELECT DISTINCT arrayReduce('quantile(0.5)', [CAST(-1, 'UInt256'), CAST(-2, 'UInt256')]) FROM numbers(1000) LIMIT 10;
SELECT DISTINCT arrayReduce('quantile(0.5)', []::Array(Float64)) FROM numbers(1000) LIMIT 10;