mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-26 19:50:51 +00:00
Avoid too slow queries with arrays
This commit is contained in:
parent
d0eeedd322
commit
57b8d3f89b
@ -31,8 +31,17 @@ namespace ErrorCodes
|
|||||||
extern const int PARAMETER_OUT_OF_BOUND;
|
extern const int PARAMETER_OUT_OF_BOUND;
|
||||||
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
|
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
|
||||||
extern const int LOGICAL_ERROR;
|
extern const int LOGICAL_ERROR;
|
||||||
|
extern const int TOO_LARGE_ARRAY_SIZE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Obtaining array as Field can be slow for large arrays and consume vast amount of memory.
|
||||||
|
* Just don't allow to do it.
|
||||||
|
* You can increase the limit if the following query:
|
||||||
|
* SELECT range(10000000)
|
||||||
|
* will take less than 500ms on your machine.
|
||||||
|
*/
|
||||||
|
static constexpr size_t max_array_size_as_field = 1000000;
|
||||||
|
|
||||||
|
|
||||||
ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column)
|
ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && offsets_column)
|
||||||
: data(std::move(nested_column)), offsets(std::move(offsets_column))
|
: data(std::move(nested_column)), offsets(std::move(offsets_column))
|
||||||
@ -117,6 +126,11 @@ Field ColumnArray::operator[](size_t n) const
|
|||||||
{
|
{
|
||||||
size_t offset = offsetAt(n);
|
size_t offset = offsetAt(n);
|
||||||
size_t size = sizeAt(n);
|
size_t size = sizeAt(n);
|
||||||
|
|
||||||
|
if (size > max_array_size_as_field)
|
||||||
|
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Array of size {} is too large to be manipulated as single field, maximum size {}",
|
||||||
|
size, max_array_size_as_field);
|
||||||
|
|
||||||
Array res(size);
|
Array res(size);
|
||||||
|
|
||||||
for (size_t i = 0; i < size; ++i)
|
for (size_t i = 0; i < size; ++i)
|
||||||
@ -130,6 +144,11 @@ void ColumnArray::get(size_t n, Field & res) const
|
|||||||
{
|
{
|
||||||
size_t offset = offsetAt(n);
|
size_t offset = offsetAt(n);
|
||||||
size_t size = sizeAt(n);
|
size_t size = sizeAt(n);
|
||||||
|
|
||||||
|
if (size > max_array_size_as_field)
|
||||||
|
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Array of size {} is too large to be manipulated as single field, maximum size {}",
|
||||||
|
size, max_array_size_as_field);
|
||||||
|
|
||||||
res = Array(size);
|
res = Array(size);
|
||||||
Array & res_arr = DB::get<Array &>(res);
|
Array & res_arr = DB::get<Array &>(res);
|
||||||
|
|
||||||
|
@ -102,7 +102,8 @@ private:
|
|||||||
}
|
}
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
bool executeConstStartStep(Block & block, const IColumn * end_arg, const T start, const T step, const size_t input_rows_count, const size_t result) const
|
bool executeConstStartStep(
|
||||||
|
Block & block, const IColumn * end_arg, const T start, const T step, const size_t input_rows_count, const size_t result) const
|
||||||
{
|
{
|
||||||
auto end_column = checkAndGetColumn<ColumnVector<T>>(end_arg);
|
auto end_column = checkAndGetColumn<ColumnVector<T>>(end_arg);
|
||||||
if (!end_column)
|
if (!end_column)
|
||||||
@ -155,7 +156,8 @@ private:
|
|||||||
}
|
}
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
bool executeConstStep(Block & block, const IColumn * start_arg, const IColumn * end_arg, const T step, const size_t input_rows_count, const size_t result) const
|
bool executeConstStep(
|
||||||
|
Block & block, const IColumn * start_arg, const IColumn * end_arg, const T step, const size_t input_rows_count, const size_t result) const
|
||||||
{
|
{
|
||||||
auto start_column = checkAndGetColumn<ColumnVector<T>>(start_arg);
|
auto start_column = checkAndGetColumn<ColumnVector<T>>(start_arg);
|
||||||
auto end_column = checkAndGetColumn<ColumnVector<T>>(end_arg);
|
auto end_column = checkAndGetColumn<ColumnVector<T>>(end_arg);
|
||||||
@ -210,7 +212,8 @@ private:
|
|||||||
}
|
}
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
bool executeConstStart(Block & block, const IColumn * end_arg, const IColumn * step_arg, const T start, const size_t input_rows_count, const size_t result) const
|
bool executeConstStart(
|
||||||
|
Block & block, const IColumn * end_arg, const IColumn * step_arg, const T start, const size_t input_rows_count, const size_t result) const
|
||||||
{
|
{
|
||||||
auto end_column = checkAndGetColumn<ColumnVector<T>>(end_arg);
|
auto end_column = checkAndGetColumn<ColumnVector<T>>(end_arg);
|
||||||
auto step_column = checkAndGetColumn<ColumnVector<T>>(step_arg);
|
auto step_column = checkAndGetColumn<ColumnVector<T>>(step_arg);
|
||||||
@ -265,7 +268,9 @@ private:
|
|||||||
}
|
}
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
bool executeGeneric(Block & block, const IColumn * start_col, const IColumn * end_col, const IColumn * step_col, const size_t input_rows_count, const size_t result) const
|
bool executeGeneric(
|
||||||
|
Block & block, const IColumn * start_col, const IColumn * end_col, const IColumn * step_col,
|
||||||
|
const size_t input_rows_count, const size_t result) const
|
||||||
{
|
{
|
||||||
auto start_column = checkAndGetColumn<ColumnVector<T>>(start_col);
|
auto start_column = checkAndGetColumn<ColumnVector<T>>(start_col);
|
||||||
auto end_column = checkAndGetColumn<ColumnVector<T>>(end_col);
|
auto end_column = checkAndGetColumn<ColumnVector<T>>(end_col);
|
||||||
|
Loading…
Reference in New Issue
Block a user