2019-03-29 01:02:05 +00:00
|
|
|
#pragma once
|
|
|
|
|
|
|
|
#include <Columns/ColumnArray.h>
|
|
|
|
#include <Columns/ColumnConst.h>
|
|
|
|
#include <Columns/ColumnString.h>
|
|
|
|
#include <Columns/ColumnVector.h>
|
|
|
|
#include <Columns/ColumnsNumber.h>
|
|
|
|
#include <DataTypes/DataTypeArray.h>
|
|
|
|
#include <DataTypes/DataTypeString.h>
|
|
|
|
#include <DataTypes/DataTypesNumber.h>
|
|
|
|
#include <Functions/FunctionHelpers.h>
|
2021-05-17 07:30:42 +00:00
|
|
|
#include <Functions/IFunction.h>
|
2019-03-29 01:02:05 +00:00
|
|
|
#include <IO/WriteHelpers.h>
|
|
|
|
#include <Interpreters/Context.h>
|
|
|
|
|
|
|
|
#include <optional>
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
|
|
|
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
|
|
|
extern const int ILLEGAL_COLUMN;
|
|
|
|
}
|
|
|
|
|
2022-07-14 11:00:51 +00:00
|
|
|
/**
|
|
|
|
* multiFuzzyMatchAny(haystack, [pattern_1, pattern_2, ..., pattern_n])
|
|
|
|
* multiFuzzyMatchAnyIndex(haystack, [pattern_1, pattern_2, ..., pattern_n])
|
|
|
|
* multiFuzzyMatchAllIndices(haystack, [pattern_1, pattern_2, ..., pattern_n])
|
|
|
|
*
|
|
|
|
*/
|
2019-03-29 01:02:05 +00:00
|
|
|
|
2022-06-24 14:42:39 +00:00
|
|
|
template <typename Impl>
|
2019-03-29 01:27:17 +00:00
|
|
|
class FunctionsMultiStringFuzzySearch : public IFunction
|
2019-03-29 01:02:05 +00:00
|
|
|
{
|
|
|
|
public:
|
2021-09-21 16:43:46 +00:00
|
|
|
static constexpr auto name = Impl::name;
|
2022-06-25 15:53:11 +00:00
|
|
|
|
2021-06-01 12:20:52 +00:00
|
|
|
static FunctionPtr create(ContextPtr context)
|
2019-03-29 01:02:05 +00:00
|
|
|
{
|
2022-06-24 13:20:03 +00:00
|
|
|
const auto & settings = context->getSettingsRef();
|
2023-02-08 13:07:27 +00:00
|
|
|
return std::make_shared<FunctionsMultiStringFuzzySearch>(settings.allow_hyperscan, settings.max_hyperscan_regexp_length, settings.max_hyperscan_regexp_total_length, settings.reject_expensive_hyperscan_regexps);
|
2021-08-07 05:07:41 +00:00
|
|
|
}
|
|
|
|
|
2023-02-08 13:07:27 +00:00
|
|
|
FunctionsMultiStringFuzzySearch(bool allow_hyperscan_, size_t max_hyperscan_regexp_length_, size_t max_hyperscan_regexp_total_length_, bool reject_expensive_hyperscan_regexps_)
|
2022-06-25 15:53:11 +00:00
|
|
|
: allow_hyperscan(allow_hyperscan_)
|
|
|
|
, max_hyperscan_regexp_length(max_hyperscan_regexp_length_)
|
|
|
|
, max_hyperscan_regexp_total_length(max_hyperscan_regexp_total_length_)
|
2023-02-08 13:07:27 +00:00
|
|
|
, reject_expensive_hyperscan_regexps(reject_expensive_hyperscan_regexps_)
|
2022-06-24 13:33:34 +00:00
|
|
|
{}
|
2019-03-29 01:02:05 +00:00
|
|
|
|
|
|
|
String getName() const override { return name; }
|
|
|
|
size_t getNumberOfArguments() const override { return 3; }
|
|
|
|
bool useDefaultImplementationForConstants() const override { return true; }
|
2021-06-22 16:21:23 +00:00
|
|
|
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
|
2022-06-26 16:45:16 +00:00
|
|
|
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
|
2019-03-29 01:02:05 +00:00
|
|
|
|
|
|
|
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
|
|
|
{
|
|
|
|
if (!isString(arguments[0]))
|
2022-06-24 13:33:34 +00:00
|
|
|
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[0]->getName(), getName());
|
2019-03-29 01:02:05 +00:00
|
|
|
|
|
|
|
if (!isUnsignedInteger(arguments[1]))
|
2022-06-24 13:33:34 +00:00
|
|
|
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[1]->getName(), getName());
|
2019-03-29 01:02:05 +00:00
|
|
|
|
|
|
|
const DataTypeArray * array_type = checkAndGetDataType<DataTypeArray>(arguments[2].get());
|
|
|
|
if (!array_type || !checkAndGetDataType<DataTypeString>(array_type->getNestedType().get()))
|
2022-06-24 13:33:34 +00:00
|
|
|
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}", arguments[2]->getName(), getName());
|
|
|
|
|
2020-03-23 02:12:31 +00:00
|
|
|
return Impl::getReturnType();
|
2019-03-29 01:02:05 +00:00
|
|
|
}
|
|
|
|
|
2020-11-17 13:24:45 +00:00
|
|
|
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
|
2019-03-29 01:02:05 +00:00
|
|
|
{
|
2022-07-06 14:29:42 +00:00
|
|
|
const ColumnPtr & haystack_ptr = arguments[0].column;
|
|
|
|
const ColumnPtr & edit_distance_ptr = arguments[1].column;
|
|
|
|
const ColumnPtr & needles_ptr = arguments[2].column;
|
2019-03-29 01:02:05 +00:00
|
|
|
|
2022-07-06 14:29:42 +00:00
|
|
|
const ColumnString * col_haystack_vector = checkAndGetColumn<ColumnString>(&*haystack_ptr);
|
2022-07-06 14:30:21 +00:00
|
|
|
const ColumnConst * col_haystack_const = checkAndGetColumnConst<ColumnString>(&*haystack_ptr);
|
2022-06-26 16:45:16 +00:00
|
|
|
assert(static_cast<bool>(col_haystack_vector) ^ static_cast<bool>(col_haystack_const));
|
2019-03-29 01:02:05 +00:00
|
|
|
|
|
|
|
UInt32 edit_distance = 0;
|
2022-07-06 14:29:42 +00:00
|
|
|
if (const auto * col_const_uint8 = checkAndGetColumnConst<ColumnUInt8>(edit_distance_ptr.get()))
|
2022-06-25 15:41:24 +00:00
|
|
|
edit_distance = col_const_uint8->getValue<UInt8>();
|
2022-07-06 14:29:42 +00:00
|
|
|
else if (const auto * col_const_uint16 = checkAndGetColumnConst<ColumnUInt16>(edit_distance_ptr.get()))
|
2022-06-25 15:41:24 +00:00
|
|
|
edit_distance = col_const_uint16->getValue<UInt16>();
|
2022-07-06 14:29:42 +00:00
|
|
|
else if (const auto * col_const_uint32 = checkAndGetColumnConst<ColumnUInt32>(edit_distance_ptr.get()))
|
2022-06-25 15:41:24 +00:00
|
|
|
edit_distance = col_const_uint32->getValue<UInt32>();
|
2019-03-29 01:02:05 +00:00
|
|
|
else
|
2023-01-23 21:13:58 +00:00
|
|
|
throw Exception(ErrorCodes::ILLEGAL_COLUMN,
|
|
|
|
"Illegal column {}. The number is not const or does not fit in UInt32",
|
|
|
|
arguments[1].column->getName());
|
2019-03-29 01:02:05 +00:00
|
|
|
|
2022-07-06 14:55:02 +00:00
|
|
|
const ColumnArray * col_needles_vector = checkAndGetColumn<ColumnArray>(needles_ptr.get());
|
2022-07-06 14:29:42 +00:00
|
|
|
const ColumnConst * col_needles_const = checkAndGetColumnConst<ColumnArray>(needles_ptr.get());
|
2022-07-06 14:55:02 +00:00
|
|
|
assert(static_cast<bool>(col_needles_vector) ^ static_cast<bool>(col_needles_const));
|
2022-06-26 16:45:16 +00:00
|
|
|
|
2022-07-06 14:55:02 +00:00
|
|
|
if (col_haystack_const && col_needles_vector)
|
2022-06-26 16:45:16 +00:00
|
|
|
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support search with non-constant needles in constant haystack", name);
|
2019-03-29 01:02:05 +00:00
|
|
|
|
2022-06-24 13:34:40 +00:00
|
|
|
using ResultType = typename Impl::ResultType;
|
2019-03-29 01:02:05 +00:00
|
|
|
auto col_res = ColumnVector<ResultType>::create();
|
2019-10-13 13:22:09 +00:00
|
|
|
auto col_offsets = ColumnArray::ColumnOffsets::create();
|
2019-03-29 01:02:05 +00:00
|
|
|
|
|
|
|
auto & vec_res = col_res->getData();
|
2019-10-13 13:22:09 +00:00
|
|
|
auto & offsets_res = col_offsets->getData();
|
2022-06-28 07:51:36 +00:00
|
|
|
/// the implementations are responsible for resizing the output column
|
2022-06-24 13:44:42 +00:00
|
|
|
|
2022-06-26 16:45:16 +00:00
|
|
|
if (col_needles_const)
|
|
|
|
Impl::vectorConstant(
|
2022-07-06 21:36:14 +00:00
|
|
|
col_haystack_vector->getChars(), col_haystack_vector->getOffsets(),
|
|
|
|
col_needles_const->getValue<Array>(),
|
|
|
|
vec_res, offsets_res,
|
|
|
|
edit_distance,
|
2023-02-08 13:07:27 +00:00
|
|
|
allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
|
2022-06-26 16:45:16 +00:00
|
|
|
else
|
|
|
|
Impl::vectorVector(
|
2022-07-06 21:36:14 +00:00
|
|
|
col_haystack_vector->getChars(), col_haystack_vector->getOffsets(),
|
|
|
|
col_needles_vector->getData(), col_needles_vector->getOffsets(),
|
|
|
|
vec_res, offsets_res,
|
|
|
|
edit_distance,
|
2023-02-08 13:07:27 +00:00
|
|
|
allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
|
2019-03-29 01:02:05 +00:00
|
|
|
|
2022-07-06 15:27:01 +00:00
|
|
|
// the combination of const haystack + const needle is not implemented because
|
|
|
|
// useDefaultImplementationForConstants() == true makes upper layers convert both to
|
|
|
|
// non-const columns
|
|
|
|
|
2019-10-13 13:22:09 +00:00
|
|
|
if constexpr (Impl::is_column_array)
|
2020-10-18 19:00:13 +00:00
|
|
|
return ColumnArray::create(std::move(col_res), std::move(col_offsets));
|
2019-10-13 13:22:09 +00:00
|
|
|
else
|
2020-10-18 19:00:13 +00:00
|
|
|
return col_res;
|
2019-03-29 01:02:05 +00:00
|
|
|
}
|
2021-08-07 05:07:41 +00:00
|
|
|
|
|
|
|
private:
|
2022-06-25 15:53:11 +00:00
|
|
|
const bool allow_hyperscan;
|
2022-06-24 13:21:32 +00:00
|
|
|
const size_t max_hyperscan_regexp_length;
|
|
|
|
const size_t max_hyperscan_regexp_total_length;
|
2023-02-08 13:07:27 +00:00
|
|
|
const bool reject_expensive_hyperscan_regexps;
|
2019-03-29 01:02:05 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
}
|