ClickHouse/src/Interpreters/sortBlock.cpp

280 lines
8.5 KiB
C++
Raw Normal View History

#include <Interpreters/sortBlock.h>
2012-07-17 20:05:36 +00:00
#include <Columns/ColumnString.h>
2019-12-05 15:31:45 +00:00
#include <Columns/ColumnConst.h>
2017-07-13 20:58:19 +00:00
#include <Common/typeid_cast.h>
2019-12-06 22:22:12 +00:00
#include <Functions/FunctionHelpers.h>
#include <pdqsort.h>
2012-07-17 20:05:36 +00:00
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_COLLATION;
2020-05-15 00:01:14 +00:00
extern const int OPENCL_ERROR;
}
2019-12-06 22:22:12 +00:00
static bool isCollationRequired(const SortColumnDescription & description)
2018-10-04 10:24:51 +00:00
{
2019-12-06 22:22:12 +00:00
return description.collator != nullptr;
2018-10-04 10:24:51 +00:00
}
ColumnsWithSortDescriptions getColumnsWithSortDescription(const Block & block, const SortDescription & description)
{
size_t size = description.size();
ColumnsWithSortDescriptions res;
res.reserve(size);
for (size_t i = 0; i < size; ++i)
{
const IColumn * column = !description[i].column_name.empty()
? block.getByName(description[i].column_name).column.get()
: block.safeGetByPosition(description[i].column_number).column.get();
2019-12-06 22:22:12 +00:00
res.emplace_back(ColumnWithSortDescription{column, description[i], isColumnConst(*column)});
}
return res;
}
2013-05-28 16:56:05 +00:00
2018-10-04 14:55:02 +00:00
struct PartialSortingLess
{
const ColumnsWithSortDescriptions & columns;
explicit PartialSortingLess(const ColumnsWithSortDescriptions & columns_) : columns(columns_) {}
bool operator() (size_t a, size_t b) const
{
2020-03-09 00:28:05 +00:00
for (const auto & elem : columns)
2018-10-04 14:55:02 +00:00
{
2019-12-06 22:22:12 +00:00
int res;
2020-03-09 00:28:05 +00:00
if (elem.column_const)
2019-12-06 22:22:12 +00:00
res = 0;
else
2020-03-09 00:28:05 +00:00
res = elem.description.direction * elem.column->compareAt(a, b, *elem.column, elem.description.nulls_direction);
2018-10-04 14:55:02 +00:00
if (res < 0)
return true;
else if (res > 0)
return false;
}
return false;
}
};
2013-05-28 16:56:05 +00:00
struct PartialSortingLessWithCollation
{
const ColumnsWithSortDescriptions & columns;
2019-12-06 22:22:12 +00:00
explicit PartialSortingLessWithCollation(const ColumnsWithSortDescriptions & columns_)
: columns(columns_)
{
}
bool operator() (size_t a, size_t b) const
{
2020-03-09 00:28:05 +00:00
for (const auto & elem : columns)
{
int res;
2019-12-06 22:22:12 +00:00
2020-03-09 00:28:05 +00:00
if (elem.column_const)
2019-12-09 01:15:42 +00:00
{
2019-12-06 22:22:12 +00:00
res = 0;
2019-12-09 01:15:42 +00:00
}
2020-03-09 00:28:05 +00:00
else if (isCollationRequired(elem.description))
{
2020-03-09 00:28:05 +00:00
const ColumnString & column_string = assert_cast<const ColumnString &>(*elem.column);
res = column_string.compareAtWithCollation(a, b, *elem.column, *elem.description.collator);
}
else
2020-03-09 00:28:05 +00:00
res = elem.column->compareAt(a, b, *elem.column, elem.description.nulls_direction);
res *= elem.description.direction;
if (res < 0)
return true;
else if (res > 0)
return false;
}
return false;
}
2012-07-17 20:05:36 +00:00
};
2019-02-10 15:17:45 +00:00
void sortBlock(Block & block, const SortDescription & description, UInt64 limit)
2012-07-17 20:05:36 +00:00
{
if (!block)
return;
2019-12-06 22:22:12 +00:00
/// If only one column to sort by
if (description.size() == 1)
{
2019-12-07 13:13:14 +00:00
IColumn::Permutation perm;
bool reverse = description[0].direction == -1;
const IColumn * column = !description[0].column_name.empty()
? block.getByName(description[0].column_name).column.get()
: block.safeGetByPosition(description[0].column_number).column.get();
2019-12-07 13:13:14 +00:00
bool is_column_const = false;
2019-12-06 22:22:12 +00:00
if (isCollationRequired(description[0]))
{
2019-12-06 22:22:12 +00:00
/// it it's real string column, than we need sort
if (const ColumnString * column_string = checkAndGetColumn<ColumnString>(column))
column_string->getPermutationWithCollation(*description[0].collator, reverse, limit, perm);
2019-12-07 13:13:14 +00:00
else if (checkAndGetColumnConstData<ColumnString>(column))
is_column_const = true;
else
2019-12-06 22:22:12 +00:00
throw Exception("Collations could be specified only for String columns.", ErrorCodes::BAD_COLLATION);
}
2019-12-06 22:22:12 +00:00
else if (!isColumnConst(*column))
2020-05-15 00:01:14 +00:00
{
int nan_direction_hint = description[0].nulls_direction;
/// If in Settings `special_sort` option has been set as `bitonic_sort`,
/// then via `nan_direction_hint` variable a flag which specifies bitonic sort as preferred
/// will be passed to `getPermutation` method with value 42.
if (description[0].special_sort == SpecialSort::OPENCL_BITONIC)
{
nan_direction_hint = 42;
#ifndef BITONIC_SORT_PREFERRED
throw DB::Exception("Bitonic sort specified as preferred, but OpenCL not available", DB::ErrorCodes::OPENCL_ERROR);
#endif
}
column->getPermutation(reverse, limit, nan_direction_hint, perm);
}
2019-12-07 13:13:14 +00:00
else
2020-01-29 10:08:39 +00:00
/// we don't need to do anything with const column
2019-12-07 13:13:14 +00:00
is_column_const = true;
2020-01-29 10:08:39 +00:00
size_t columns = block.columns();
for (size_t i = 0; i < columns; ++i)
2019-12-07 13:13:14 +00:00
{
if (!is_column_const)
block.getByPosition(i).column = block.getByPosition(i).column->permute(perm, limit);
}
}
else
{
2019-12-07 13:13:14 +00:00
size_t size = block.rows();
IColumn::Permutation perm(size);
for (size_t i = 0; i < size; ++i)
perm[i] = i;
if (limit >= size)
limit = 0;
bool need_collation = false;
ColumnsWithSortDescriptions columns_with_sort_desc = getColumnsWithSortDescription(block, description);
2018-08-27 18:16:32 +00:00
for (size_t i = 0, num_sort_columns = description.size(); i < num_sort_columns; ++i)
{
2019-12-06 22:22:12 +00:00
const IColumn * column = columns_with_sort_desc[i].column;
if (isCollationRequired(description[i]))
{
2019-12-06 22:22:12 +00:00
if (!checkAndGetColumn<ColumnString>(column) && !checkAndGetColumnConstData<ColumnString>(column))
throw Exception("Collations could be specified only for String columns.", ErrorCodes::BAD_COLLATION);
need_collation = true;
}
}
if (need_collation)
{
PartialSortingLessWithCollation less_with_collation(columns_with_sort_desc);
if (limit)
std::partial_sort(perm.begin(), perm.begin() + limit, perm.end(), less_with_collation);
else
pdqsort(perm.begin(), perm.end(), less_with_collation);
}
else
{
PartialSortingLess less(columns_with_sort_desc);
if (limit)
std::partial_sort(perm.begin(), perm.begin() + limit, perm.end(), less);
else
pdqsort(perm.begin(), perm.end(), less);
}
size_t columns = block.columns();
for (size_t i = 0; i < columns; ++i)
2019-12-06 22:22:12 +00:00
{
2018-06-30 21:35:01 +00:00
block.getByPosition(i).column = block.getByPosition(i).column->permute(perm, limit);
2019-12-06 22:22:12 +00:00
}
}
2012-07-17 20:05:36 +00:00
}
2013-11-29 22:10:15 +00:00
void stableGetPermutation(const Block & block, const SortDescription & description, IColumn::Permutation & out_permutation)
2013-11-29 22:10:15 +00:00
{
if (!block)
return;
2013-11-29 22:10:15 +00:00
size_t size = block.rows();
out_permutation.resize(size);
for (size_t i = 0; i < size; ++i)
out_permutation[i] = i;
2013-11-29 22:10:15 +00:00
ColumnsWithSortDescriptions columns_with_sort_desc = getColumnsWithSortDescription(block, description);
2013-11-29 22:10:15 +00:00
std::stable_sort(out_permutation.begin(), out_permutation.end(), PartialSortingLess(columns_with_sort_desc));
}
bool isAlreadySorted(const Block & block, const SortDescription & description)
{
if (!block)
return true;
size_t rows = block.rows();
ColumnsWithSortDescriptions columns_with_sort_desc = getColumnsWithSortDescription(block, description);
PartialSortingLess less(columns_with_sort_desc);
2017-04-02 17:37:49 +00:00
/** If the rows are not too few, then let's make a quick attempt to verify that the block is not sorted.
* Constants - at random.
*/
static constexpr size_t num_rows_to_try = 10;
if (rows > num_rows_to_try * 5)
{
for (size_t i = 1; i < num_rows_to_try; ++i)
{
size_t prev_position = rows * (i - 1) / num_rows_to_try;
size_t curr_position = rows * i / num_rows_to_try;
if (less(curr_position, prev_position))
return false;
}
}
for (size_t i = 1; i < rows; ++i)
if (less(i, i - 1))
return false;
return true;
}
void stableSortBlock(Block & block, const SortDescription & description)
{
if (!block)
return;
IColumn::Permutation perm;
stableGetPermutation(block, description, perm);
2013-11-29 22:10:15 +00:00
size_t columns = block.columns();
for (size_t i = 0; i < columns; ++i)
block.safeGetByPosition(i).column = block.safeGetByPosition(i).column->permute(perm, 0);
2013-11-29 22:10:15 +00:00
}
2012-07-17 20:05:36 +00:00
}