Merge pull request #13237 from vdimir/string-locate-startpos-3776

Support start_pos argument in `position` function
This commit is contained in:
Anton Popov 2020-08-24 12:23:19 +03:00 committed by GitHub
commit 93d49e5815
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 733 additions and 69 deletions

View File

@ -21,15 +21,16 @@ For a case-insensitive search, use the function [positionCaseInsensitive](#posit
**Syntax**
``` sql
position(haystack, needle)
position(haystack, needle[, start_pos])
```
Alias: `locate(haystack, needle)`.
Alias: `locate(haystack, needle[, start_pos])`.
**Parameters**
- `haystack` — string, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Returned values**
@ -56,6 +57,18 @@ Result:
└────────────────────────────────┘
```
``` sql
SELECT
position('Hello, world!', 'o', 1),
position('Hello, world!', 'o', 7)
```
``` text
┌─position('Hello, world!', 'o', 1)─┬─position('Hello, world!', 'o', 7)─┐
│ 5 │ 9 │
└───────────────────────────────────┴───────────────────────────────────┘
```
The same phrase in Russian contains characters which cant be represented using a single byte. The function returns some unexpected result (use [positionUTF8](#positionutf8) function for multi-byte encoded text):
Query:
@ -81,13 +94,14 @@ Works under the assumption that the string contains a set of bytes representing
**Syntax**
``` sql
positionCaseInsensitive(haystack, needle)
positionCaseInsensitive(haystack, needle[, start_pos])
```
**Parameters**
- `haystack` — string, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Returned values**
@ -123,13 +137,14 @@ For a case-insensitive search, use the function [positionCaseInsensitiveUTF8](#p
**Syntax**
``` sql
positionUTF8(haystack, needle)
positionUTF8(haystack, needle[, start_pos])
```
**Parameters**
- `haystack` — string, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Returned values**
@ -195,13 +210,14 @@ Works under the assumption that the string contains a set of bytes representing
**Syntax**
``` sql
positionCaseInsensitiveUTF8(haystack, needle)
positionCaseInsensitiveUTF8(haystack, needle[, start_pos])
```
**Parameters**
- `haystack` — string, in which substring will to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Returned value**

View File

@ -20,15 +20,16 @@ Para una búsqueda sin distinción de mayúsculas y minúsculas, utilice la func
**Sintaxis**
``` sql
position(haystack, needle)
position(haystack, needle[, start_pos])
```
Apodo: `locate(haystack, needle)`.
Apodo: `locate(haystack, needle[, start_pos])`.
**Parámetros**
- `haystack` — string, in which substring will to be searched. [Cadena](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [Cadena](../syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Valores devueltos**
@ -80,13 +81,14 @@ Funciona bajo el supuesto de que la cadena contiene un conjunto de bytes que rep
**Sintaxis**
``` sql
positionCaseInsensitive(haystack, needle)
positionCaseInsensitive(haystack, needle[, start_pos])
```
**Parámetros**
- `haystack` — string, in which substring will to be searched. [Cadena](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [Cadena](../syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Valores devueltos**
@ -122,13 +124,14 @@ Para una búsqueda sin distinción de mayúsculas y minúsculas, utilice la func
**Sintaxis**
``` sql
positionUTF8(haystack, needle)
positionUTF8(haystack, needle[, start_pos])
```
**Parámetros**
- `haystack` — string, in which substring will to be searched. [Cadena](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [Cadena](../syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Valores devueltos**
@ -194,13 +197,14 @@ Funciona bajo el supuesto de que la cadena contiene un conjunto de bytes que rep
**Sintaxis**
``` sql
positionCaseInsensitiveUTF8(haystack, needle)
positionCaseInsensitiveUTF8(haystack, needle[, start_pos])
```
**Parámetros**
- `haystack` — string, in which substring will to be searched. [Cadena](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [Cadena](../syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Valor devuelto**

View File

@ -21,15 +21,16 @@ toc_title: "\u0628\u0631\u0627\u06CC \u062C\u0633\u062A\u062C\u0648\u06CC \u0631
**نحو**
``` sql
position(haystack, needle)
position(haystack, needle[, start_pos])
```
نام مستعار: `locate(haystack, needle)`.
نام مستعار: `locate(haystack, needle[, start_pos])`.
**پارامترها**
- `haystack` — string, in which substring will to be searched. [رشته](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [رشته](../syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**مقادیر بازگشتی**
@ -81,13 +82,14 @@ SELECT position('Привет, мир!', '!')
**نحو**
``` sql
positionCaseInsensitive(haystack, needle)
positionCaseInsensitive(haystack, needle[, start_pos])
```
**پارامترها**
- `haystack` — string, in which substring will to be searched. [رشته](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [رشته](../syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**مقادیر بازگشتی**
@ -123,13 +125,14 @@ SELECT positionCaseInsensitive('Hello, world!', 'hello')
**نحو**
``` sql
positionUTF8(haystack, needle)
positionUTF8(haystack, needle[, start_pos])
```
**پارامترها**
- `haystack` — string, in which substring will to be searched. [رشته](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [رشته](../syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**مقادیر بازگشتی**
@ -195,13 +198,14 @@ SELECT positionUTF8('Salut, étudiante!', '!')
**نحو**
``` sql
positionCaseInsensitiveUTF8(haystack, needle)
positionCaseInsensitiveUTF8(haystack, needle[, start_pos])
```
**پارامترها**
- `haystack` — string, in which substring will to be searched. [رشته](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [رشته](../syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**مقدار بازگشتی**

View File

@ -20,15 +20,16 @@ toc_title: "\u6587\u5B57\u5217\u3092\u691C\u7D22\u3059\u308B\u5834\u5408"
**構文**
``` sql
position(haystack, needle)
position(haystack, needle[, start_pos])
```
別名: `locate(haystack, needle)`.
別名: `locate(haystack, needle[, start_pos])`.
**パラメータ**
- `haystack` — string, in which substring will to be searched. [文字列](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [文字列](../syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**戻り値**
@ -80,13 +81,14 @@ SELECT position('Привет, мир!', '!')
**構文**
``` sql
positionCaseInsensitive(haystack, needle)
positionCaseInsensitive(haystack, needle[, start_pos])
```
**パラメータ**
- `haystack` — string, in which substring will to be searched. [文字列](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [文字列](../syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**戻り値**
@ -122,13 +124,14 @@ SELECT positionCaseInsensitive('Hello, world!', 'hello')
**構文**
``` sql
positionUTF8(haystack, needle)
positionUTF8(haystack, needle[, start_pos])
```
**パラメータ**
- `haystack` — string, in which substring will to be searched. [文字列](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [文字列](../syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**戻り値**
@ -194,13 +197,14 @@ SELECT positionUTF8('Salut, étudiante!', '!')
**構文**
``` sql
positionCaseInsensitiveUTF8(haystack, needle)
positionCaseInsensitiveUTF8(haystack, needle[, start_pos])
```
**パラメータ**
- `haystack` — string, in which substring will to be searched. [文字列](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [文字列](../syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**戻り値**

View File

@ -15,15 +15,16 @@
**Синтаксис**
``` sql
position(haystack, needle)
position(haystack, needle[, start_pos])
```
Алиас: `locate(haystack, needle)`.
Алиас: `locate(haystack, needle[, start_pos])`.
**Параметры**
- `haystack` — строка, по которой выполняется поиск. [Строка](../syntax.md#syntax-string-literal).
- `needle` — подстрока, которую необходимо найти. [Строка](../syntax.md#syntax-string-literal).
- `start_pos` Опциональный параметр, позиция символа в строке, с которого начинается поиск. [UInt](../../sql-reference/data-types/int-uint.md)
**Возвращаемые значения**
@ -75,13 +76,14 @@ SELECT position('Привет, мир!', '!')
**Синтаксис**
``` sql
positionCaseInsensitive(haystack, needle)
positionCaseInsensitive(haystack, needle[, start_pos])
```
**Параметры**
- `haystack` — строка, по которой выполняется поиск. [Строка](../syntax.md#syntax-string-literal).
- `needle` — подстрока, которую необходимо найти. [Строка](../syntax.md#syntax-string-literal).
- `start_pos` Опциональный параметр, позиция символа в строке, с которого начинается поиск. [UInt](../../sql-reference/data-types/int-uint.md)
**Возвращаемые значения**
@ -117,13 +119,14 @@ SELECT positionCaseInsensitive('Hello, world!', 'hello')
**Синтаксис**
``` sql
positionUTF8(haystack, needle)
positionUTF8(haystack, needle[, start_pos])
```
**Параметры**
- `haystack` — строка, по которой выполняется поиск. [Строка](../syntax.md#syntax-string-literal).
- `needle` — подстрока, которую необходимо найти. [Строка](../syntax.md#syntax-string-literal).
- `start_pos` Опциональный параметр, позиция символа в строке, с которого начинается поиск. [UInt](../../sql-reference/data-types/int-uint.md)
**Возвращаемые значения**
@ -189,13 +192,14 @@ SELECT positionUTF8('Salut, étudiante!', '!')
**Синтаксис**
``` sql
positionCaseInsensitiveUTF8(haystack, needle)
positionCaseInsensitiveUTF8(haystack, needle[, start_pos])
```
**Параметры**
- `haystack` — строка, по которой выполняется поиск. [Строка](../syntax.md#syntax-string-literal).
- `needle` — подстрока, которую необходимо найти. [Строка](../syntax.md#syntax-string-literal).
- `start_pos` Опциональный параметр, позиция символа в строке, с которого начинается поиск. [UInt](../../sql-reference/data-types/int-uint.md)
**Возвращаемые значения**

View File

@ -20,15 +20,16 @@ Büyük / küçük harf duyarsız arama için işlevi kullanın [positionCaseİn
**Sözdizimi**
``` sql
position(haystack, needle)
position(haystack, needle[, start_pos])
```
Takma ad: `locate(haystack, needle)`.
Takma ad: `locate(haystack, needle[, start_pos])`.
**Parametre**
- `haystack` — string, in which substring will to be searched. [Dize](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [Dize](../syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Döndürülen değerler**
@ -80,13 +81,14 @@ Dize, tek baytlık kodlanmış bir metni temsil eden bir bayt kümesi içerdiği
**Sözdizimi**
``` sql
positionCaseInsensitive(haystack, needle)
positionCaseInsensitive(haystack, needle[, start_pos])
```
**Parametre**
- `haystack` — string, in which substring will to be searched. [Dize](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [Dize](../syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Döndürülen değerler**
@ -122,13 +124,14 @@ Büyük / küçük harf duyarsız arama için işlevi kullanın [positionCaseİn
**Sözdizimi**
``` sql
positionUTF8(haystack, needle)
positionUTF8(haystack, needle[, start_pos])
```
**Parametre**
- `haystack` — string, in which substring will to be searched. [Dize](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [Dize](../syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Döndürülen değerler**
@ -194,13 +197,14 @@ Dizenin UTF-8 kodlanmış bir metni temsil eden bir bayt kümesi içerdiği vars
**Sözdizimi**
``` sql
positionCaseInsensitiveUTF8(haystack, needle)
positionCaseInsensitiveUTF8(haystack, needle[, start_pos])
```
**Parametre**
- `haystack` — string, in which substring will to be searched. [Dize](../syntax.md#syntax-string-literal).
- `needle` — substring to be searched. [Dize](../syntax.md#syntax-string-literal).
- `start_pos` Optional parameter, position of the first character in the string to start search. [UInt](../../sql-reference/data-types/int-uint.md)
**Döndürülen değer**

View File

@ -10,7 +10,7 @@
#include <Functions/FunctionHelpers.h>
#include <Functions/IFunctionImpl.h>
#include <Interpreters/Context.h>
#include <IO/WriteHelpers.h>
namespace DB
{
@ -38,8 +38,9 @@ namespace DB
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int ILLEGAL_COLUMN;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
}
template <typename Impl, typename Name>
@ -51,19 +52,33 @@ public:
String getName() const override { return name; }
size_t getNumberOfArguments() const override { return 2; }
bool isVariadic() const override { return Impl::supports_start_pos; }
size_t getNumberOfArguments() const override
{
if (Impl::supports_start_pos)
return 0;
return 2;
}
bool useDefaultImplementationForConstants() const override { return Impl::use_default_implementation_for_constants; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override
{
return Impl::use_default_implementation_for_constants
? ColumnNumbers{1, 2}
: ColumnNumbers{};
if (!Impl::use_default_implementation_for_constants)
return ColumnNumbers{};
if (!Impl::supports_start_pos)
return ColumnNumbers{1, 2};
return ColumnNumbers{1, 2, 3};
}
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (arguments.size() < 2 || 3 < arguments.size())
throw Exception("Number of arguments for function " + String(Name::name) + " doesn't match: passed "
+ toString(arguments.size()) + ", should be 2 or 3.",
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
if (!isStringOrFixedString(arguments[0]))
throw Exception(
"Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
@ -72,6 +87,13 @@ public:
throw Exception(
"Illegal type " + arguments[1]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
if (arguments.size() >= 3)
{
if (!isUnsignedInteger(arguments[2]))
throw Exception(
"Illegal type " + arguments[2]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
return std::make_shared<DataTypeNumber<typename Impl::ResultType>>();
}
@ -82,17 +104,34 @@ public:
const ColumnPtr & column_haystack = block.getByPosition(arguments[0]).column;
const ColumnPtr & column_needle = block.getByPosition(arguments[1]).column;
ColumnPtr column_start_pos = nullptr;
if (arguments.size() >= 3)
column_start_pos = block.getByPosition(arguments[2]).column;
const ColumnConst * col_haystack_const = typeid_cast<const ColumnConst *>(&*column_haystack);
const ColumnConst * col_needle_const = typeid_cast<const ColumnConst *>(&*column_needle);
if constexpr (!Impl::use_default_implementation_for_constants)
{
bool is_col_start_pos_const = column_start_pos == nullptr || isColumnConst(*column_start_pos);
if (col_haystack_const && col_needle_const)
{
ResultType res{};
Impl::constantConstant(col_haystack_const->getValue<String>(), col_needle_const->getValue<String>(), res);
block.getByPosition(result).column
= block.getByPosition(result).type->createColumnConst(col_haystack_const->size(), toField(res));
auto col_res = ColumnVector<ResultType>::create();
typename ColumnVector<ResultType>::Container & vec_res = col_res->getData();
vec_res.resize(is_col_start_pos_const ? 1 : column_start_pos->size());
Impl::constantConstant(
col_haystack_const->getValue<String>(),
col_needle_const->getValue<String>(),
column_start_pos,
vec_res);
if (is_col_start_pos_const)
block.getByPosition(result).column
= block.getByPosition(result).type->createColumnConst(col_haystack_const->size(), toField(vec_res[0]));
else
block.getByPosition(result).column = std::move(col_res);
return;
}
}
@ -112,16 +151,28 @@ public:
col_haystack_vector->getOffsets(),
col_needle_vector->getChars(),
col_needle_vector->getOffsets(),
column_start_pos,
vec_res);
else if (col_haystack_vector && col_needle_const)
Impl::vectorConstant(
col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), col_needle_const->getValue<String>(), vec_res);
col_haystack_vector->getChars(),
col_haystack_vector->getOffsets(),
col_needle_const->getValue<String>(),
column_start_pos,
vec_res);
else if (col_haystack_vector_fixed && col_needle_const)
Impl::vectorFixedConstant(
col_haystack_vector_fixed->getChars(), col_haystack_vector_fixed->getN(), col_needle_const->getValue<String>(), vec_res);
col_haystack_vector_fixed->getChars(),
col_haystack_vector_fixed->getN(),
col_needle_const->getValue<String>(),
vec_res);
else if (col_haystack_const && col_needle_vector)
Impl::constantVector(
col_haystack_const->getValue<String>(), col_needle_vector->getChars(), col_needle_vector->getOffsets(), vec_res);
col_haystack_const->getValue<String>(),
col_needle_vector->getChars(),
col_needle_vector->getOffsets(),
column_start_pos,
vec_res);
else
throw Exception(
"Illegal columns " + block.getByPosition(arguments[0]).column->getName() + " and "

View File

@ -36,6 +36,7 @@ namespace DB
namespace ErrorCodes
{
extern const int ILLEGAL_COLUMN;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
@ -79,12 +80,19 @@ struct ExtractParamImpl
using ResultType = typename ParamExtractor::ResultType;
static constexpr bool use_default_implementation_for_constants = true;
static constexpr bool supports_start_pos = false;
/// It is assumed that `res` is the correct size and initialized with zeros.
static void vectorConstant(const ColumnString::Chars & data, const ColumnString::Offsets & offsets,
static void vectorConstant(
const ColumnString::Chars & data,
const ColumnString::Offsets & offsets,
std::string needle,
const ColumnPtr & start_pos,
PaddedPODArray<ResultType> & res)
{
if (start_pos != nullptr)
throw Exception("Functions 'visitParamHas' and 'visitParamExtract*' doesn't support start_pos argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
/// We are looking for a parameter simply as a substring of the form "name"
needle = "\"" + needle + "\":";

View File

@ -9,6 +9,7 @@ namespace DB
namespace ErrorCodes
{
extern const int ILLEGAL_COLUMN;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
/** Token search the string, means that needle must be surrounded by some separator chars, like whitespace or puctuation.
@ -19,10 +20,18 @@ struct HasTokenImpl
using ResultType = UInt8;
static constexpr bool use_default_implementation_for_constants = true;
static constexpr bool supports_start_pos = false;
static void vectorConstant(
const ColumnString::Chars & data, const ColumnString::Offsets & offsets, const std::string & pattern, PaddedPODArray<UInt8> & res)
const ColumnString::Chars & data,
const ColumnString::Offsets & offsets,
const std::string & pattern,
const ColumnPtr & start_pos,
PaddedPODArray<UInt8> & res)
{
if (start_pos != nullptr)
throw Exception("Function 'hasToken' does not support start_pos argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
if (offsets.empty())
return;

View File

@ -25,6 +25,7 @@ namespace DB
namespace ErrorCodes
{
extern const int ILLEGAL_COLUMN;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
@ -76,6 +77,7 @@ template <bool like, bool revert = false, bool case_insensitive = false>
struct MatchImpl
{
static constexpr bool use_default_implementation_for_constants = true;
static constexpr bool supports_start_pos = false;
using ResultType = UInt8;
@ -84,8 +86,15 @@ struct MatchImpl
VolnitskyUTF8>;
static void vectorConstant(
const ColumnString::Chars & data, const ColumnString::Offsets & offsets, const std::string & pattern, PaddedPODArray<UInt8> & res)
const ColumnString::Chars & data,
const ColumnString::Offsets & offsets,
const std::string & pattern,
const ColumnPtr & start_pos,
PaddedPODArray<UInt8> & res)
{
if (start_pos != nullptr)
throw Exception("Functions 'like' and 'match' don't support start_pos argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
if (offsets.empty())
return;
@ -238,7 +247,8 @@ struct MatchImpl
/// Very carefully crafted copy-paste.
static void vectorFixedConstant(
const ColumnString::Chars & data, size_t n, const std::string & pattern, PaddedPODArray<UInt8> & res)
const ColumnString::Chars & data, size_t n, const std::string & pattern,
PaddedPODArray<UInt8> & res)
{
if (data.empty())
return;

View File

@ -120,7 +120,7 @@ struct MultiMatchAnyImpl
memset(accum.data(), 0, accum.size());
for (size_t j = 0; j < needles.size(); ++j)
{
MatchImpl<false, false>::vectorConstant(haystack_data, haystack_offsets, needles[j].toString(), accum);
MatchImpl<false, false>::vectorConstant(haystack_data, haystack_offsets, needles[j].toString(), nullptr, accum);
for (size_t i = 0; i < res.size(); ++i)
{
if constexpr (FindAny)

View File

@ -1,11 +1,11 @@
#include "FunctionsStringSearch.h"
#include <algorithm>
#include <string>
#include <vector>
#include <Poco/UTF8String.h>
#include <Common/Volnitsky.h>
namespace DB
{
@ -42,6 +42,11 @@ struct PositionCaseSensitiveASCII
return MultiSearcherInBigHaystack(needles);
}
static const char * advancePos(const char * pos, const char * end, size_t n)
{
return std::min(pos + n, end);
}
/// Number of code points between 'begin' and 'end' (this has different behaviour for ASCII and UTF-8).
static size_t countChars(const char * begin, const char * end) { return end - begin; }
@ -73,6 +78,11 @@ struct PositionCaseInsensitiveASCII
return MultiSearcherInBigHaystack(needles);
}
static const char * advancePos(const char * pos, const char * end, size_t n)
{
return std::min(pos + n, end);
}
static size_t countChars(const char * begin, const char * end) { return end - begin; }
static void toLowerIfNeed(std::string & s) { std::transform(std::begin(s), std::end(s), std::begin(s), tolower); }
@ -100,6 +110,20 @@ struct PositionCaseSensitiveUTF8
return MultiSearcherInBigHaystack(needles);
}
static const char * advancePos(const char * pos, const char * end, size_t n)
{
for (auto it = pos; it != end; ++it)
{
if (!UTF8::isContinuationOctet(static_cast<UInt8>(*it)))
{
if (n == 0)
return it;
n--;
}
}
return end;
}
static size_t countChars(const char * begin, const char * end)
{
size_t res = 0;
@ -134,13 +158,16 @@ struct PositionCaseInsensitiveUTF8
return MultiSearcherInBigHaystack(needles);
}
static const char * advancePos(const char * pos, const char * end, size_t n)
{
// reuse implementation that doesn't depend on case
return PositionCaseSensitiveUTF8::advancePos(pos, end, n);
}
static size_t countChars(const char * begin, const char * end)
{
size_t res = 0;
for (auto it = begin; it != end; ++it)
if (!UTF8::isContinuationOctet(static_cast<UInt8>(*it)))
++res;
return res;
// reuse implementation that doesn't depend on case
return PositionCaseSensitiveUTF8::countChars(begin, end);
}
static void toLowerIfNeed(std::string & s) { Poco::UTF8::toLowerInPlace(s); }
@ -151,12 +178,17 @@ template <typename Impl>
struct PositionImpl
{
static constexpr bool use_default_implementation_for_constants = false;
static constexpr bool supports_start_pos = true;
using ResultType = UInt64;
/// Find one substring in many strings.
static void vectorConstant(
const ColumnString::Chars & data, const ColumnString::Offsets & offsets, const std::string & needle, PaddedPODArray<UInt64> & res)
const ColumnString::Chars & data,
const ColumnString::Offsets & offsets,
const std::string & needle,
const ColumnPtr & start_pos,
PaddedPODArray<UInt64> & res)
{
const UInt8 * begin = data.data();
const UInt8 * pos = begin;
@ -176,13 +208,26 @@ struct PositionImpl
res[i] = 0;
++i;
}
auto start = start_pos != nullptr ? start_pos->getUInt(i) : 0;
/// We check that the entry does not pass through the boundaries of strings.
if (pos + needle.size() < begin + offsets[i])
res[i] = 1 + Impl::countChars(reinterpret_cast<const char *>(begin + offsets[i - 1]), reinterpret_cast<const char *>(pos));
{
auto res_pos = 1 + Impl::countChars(reinterpret_cast<const char *>(begin + offsets[i - 1]), reinterpret_cast<const char *>(pos));
if (res_pos < start)
{
pos = reinterpret_cast<const UInt8 *>(Impl::advancePos(
reinterpret_cast<const char *>(pos),
reinterpret_cast<const char *>(begin + offsets[i]),
start - res_pos));
continue;
}
res[i] = res_pos;
}
else
{
res[i] = 0;
}
pos = begin + offsets[i];
++i;
}
@ -192,24 +237,68 @@ struct PositionImpl
}
/// Search for substring in string.
static void constantConstant(std::string data, std::string needle, UInt64 & res)
static void constantConstantScalar(
std::string data,
std::string needle,
UInt64 start_pos,
UInt64 & res)
{
Impl::toLowerIfNeed(data);
Impl::toLowerIfNeed(needle);
auto start = std::max(start_pos, UInt64(1));
res = data.find(needle);
if (needle.size() == 0)
{
size_t haystack_size = Impl::countChars(data.data(), data.data() + data.size());
res = start <= haystack_size + 1 ? start : 0;
return;
}
size_t start_byte = Impl::advancePos(data.data(), data.data() + data.size(), start - 1) - data.data();
res = data.find(needle, start_byte);
if (res == std::string::npos)
res = 0;
else
res = 1 + Impl::countChars(data.data(), data.data() + res);
}
/// Search for substring in string starting from different positions.
static void constantConstant(
std::string data,
std::string needle,
const ColumnPtr & start_pos,
PaddedPODArray<UInt64> & res)
{
Impl::toLowerIfNeed(data);
Impl::toLowerIfNeed(needle);
if (start_pos == nullptr)
{
constantConstantScalar(data, needle, 0, res[0]);
return;
}
size_t haystack_size = Impl::countChars(data.data(), data.data() + data.size());
size_t size = start_pos != nullptr ? start_pos->size() : 0;
for (size_t i = 0; i < size; ++i)
{
auto start = start_pos->getUInt(i);
if (start > haystack_size + 1)
{
res[i] = 0;
continue;
}
constantConstantScalar(data, needle, start, res[i]);
}
}
/// Search each time for a different single substring inside each time different string.
static void vectorVector(
const ColumnString::Chars & haystack_data,
const ColumnString::Offsets & haystack_offsets,
const ColumnString::Chars & needle_data,
const ColumnString::Offsets & needle_offsets,
const ColumnPtr & start_pos,
PaddedPODArray<UInt64> & res)
{
ColumnString::Offset prev_haystack_offset = 0;
@ -222,10 +311,16 @@ struct PositionImpl
size_t needle_size = needle_offsets[i] - prev_needle_offset - 1;
size_t haystack_size = haystack_offsets[i] - prev_haystack_offset - 1;
if (0 == needle_size)
auto start = start_pos != nullptr ? std::max(start_pos->getUInt(i), UInt64(1)) : UInt64(1);
if (start > haystack_size + 1)
{
/// An empty string is always at the very beginning of `haystack`.
res[i] = 1;
res[i] = 0;
}
else if (0 == needle_size)
{
/// An empty string is always at any position in `haystack`.
res[i] = start;
}
else
{
@ -234,8 +329,12 @@ struct PositionImpl
reinterpret_cast<const char *>(&needle_data[prev_needle_offset]),
needle_offsets[i] - prev_needle_offset - 1); /// zero byte at the end
const char * beg = Impl::advancePos(
reinterpret_cast<const char *>(&haystack_data[prev_haystack_offset]),
reinterpret_cast<const char *>(&haystack_data[haystack_offsets[i] - 1]),
start - 1);
/// searcher returns a pointer to the found substring or to the end of `haystack`.
size_t pos = searcher.search(&haystack_data[prev_haystack_offset], &haystack_data[haystack_offsets[i] - 1])
size_t pos = searcher.search(reinterpret_cast<const UInt8 *>(beg), &haystack_data[haystack_offsets[i] - 1])
- &haystack_data[prev_haystack_offset];
if (pos != haystack_size)
@ -259,9 +358,10 @@ struct PositionImpl
const String & haystack,
const ColumnString::Chars & needle_data,
const ColumnString::Offsets & needle_offsets,
const ColumnPtr & start_pos,
PaddedPODArray<UInt64> & res)
{
// NOTE You could use haystack indexing. But this is a rare case.
/// NOTE You could use haystack indexing. But this is a rare case.
ColumnString::Offset prev_needle_offset = 0;
@ -271,17 +371,24 @@ struct PositionImpl
{
size_t needle_size = needle_offsets[i] - prev_needle_offset - 1;
if (0 == needle_size)
auto start = start_pos != nullptr ? std::max(start_pos->getUInt(i), UInt64(1)) : UInt64(1);
if (start > haystack.size() + 1)
{
res[i] = 1;
res[i] = 0;
}
else if (0 == needle_size)
{
res[i] = start;
}
else
{
typename Impl::SearcherInSmallHaystack searcher = Impl::createSearcherInSmallHaystack(
reinterpret_cast<const char *>(&needle_data[prev_needle_offset]), needle_offsets[i] - prev_needle_offset - 1);
const char * beg = Impl::advancePos(haystack.data(), haystack.data() + haystack.size(), start - 1);
size_t pos = searcher.search(
reinterpret_cast<const UInt8 *>(haystack.data()),
reinterpret_cast<const UInt8 *>(beg),
reinterpret_cast<const UInt8 *>(haystack.data()) + haystack.size())
- reinterpret_cast<const UInt8 *>(haystack.data());

View File

@ -23441,3 +23441,358 @@
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1
1

View File

@ -6,6 +6,23 @@ select 1 = position('abc', 'abc');
select 2 = position('abc', 'bc');
select 3 = position('abc', 'c');
select 1 = position('', '', 0);
select 1 = position('', '', 1);
select 0 = position('', '', 2);
select 1 = position('a', '', 1);
select 2 = position('a', '', 2);
select 0 = position('a', '', 3);
select [1, 1, 2, 3, 4, 5, 0, 0, 0, 0] = groupArray(position('aaaa', '', number)) from numbers(10);
select [1, 1, 2, 3, 4, 5, 0, 0, 0, 0] = groupArray(position(materialize('aaaa'), '', number)) from numbers(10);
select [1, 1, 2, 3, 4, 5, 0, 0, 0, 0] = groupArray(position('aaaa', materialize(''), number)) from numbers(10);
select [1, 1, 2, 3, 4, 5, 0, 0, 0, 0] = groupArray(position(materialize('aaaa'), materialize(''), number)) from numbers(10);
select [1, 1, 2, 3, 4, 0, 0, 0, 0, 0] = groupArray(position('aaaa', 'a', number)) from numbers(10);
select [1, 1, 2, 3, 4, 0, 0, 0, 0, 0] = groupArray(position(materialize('aaaa'), 'a', number)) from numbers(10);
select [1, 1, 2, 3, 4, 0, 0, 0, 0, 0] = groupArray(position('aaaa', materialize('a'), number)) from numbers(10);
select [1, 1, 2, 3, 4, 0, 0, 0, 0, 0] = groupArray(position(materialize('aaaa'), materialize('a'), number)) from numbers(10);
select 1 = position(materialize(''), '');
select 1 = position(materialize('abc'), '');
select 0 = position(materialize(''), 'abc');
@ -27,6 +44,16 @@ select 1 = position('абв', 'абв');
select 3 = position('абв', 'бв');
select 5 = position('абв', 'в');
select 2 = position('abcabc', 'b', 0);
select 2 = position('abcabc', 'b', 1);
select 2 = position('abcabc', 'b', 2);
select 5 = position('abcabc', 'b', 3);
select 5 = position('abcabc', 'b', 4);
select 5 = position('abcabc', 'b', 5);
select 0 = position('abcabc', 'b', 6);
select 2 = position('abcabc', 'bca', 0);
select 0 = position('abcabc', 'bca', 3);
select 1 = position(materialize(''), '');
select 1 = position(materialize('абв'), '');
select 0 = position(materialize(''), 'абв');
@ -48,6 +75,14 @@ select 1 = positionUTF8('абв', 'абв');
select 2 = positionUTF8('абв', 'бв');
select 3 = positionUTF8('абв', 'в');
select 3 = position('абвабв', 'б', 2);
select 3 = position('абвабв', 'б', 3);
select 3 = position('абвабв', 'бва', 2);
select 9 = position('абвабв', 'б', 4);
select 0 = position('абвабв', 'бва', 4);
select 5 = position('абвабв', 'в', 0);
select 11 = position('абвабв', 'в', 6);
select 1 = positionUTF8(materialize(''), '');
select 1 = positionUTF8(materialize('абв'), '');
select 0 = positionUTF8(materialize(''), 'абв');
@ -62,6 +97,51 @@ select 1 = positionUTF8(materialize('абв'), 'абв') from system.numbers lim
select 2 = positionUTF8(materialize('абв'), 'бв') from system.numbers limit 10;
select 3 = positionUTF8(materialize('абв'), 'в') from system.numbers limit 10;
select 2 = positionUTF8('абвабв', 'б', 0);
select 2 = positionUTF8('абвабв', 'б', 1);
select 2 = positionUTF8('абвабв', 'б', 2);
select 5 = positionUTF8('абвабв', 'б', 3);
select 5 = positionUTF8('абвабв', 'б', 4);
select 5 = positionUTF8('абвабв', 'б', 5);
select 0 = positionUTF8('абвабв', 'б', 6);
select 2 = positionUTF8('абвабв', 'бва', 0);
select 0 = positionUTF8('абвабв', 'бва', 3);
select 2 = positionUTF8(materialize('абвабв'), 'б', 0) from system.numbers limit 10;
select 2 = positionUTF8(materialize('абвабв'), 'б', 1) from system.numbers limit 10;
select 2 = positionUTF8(materialize('абвабв'), 'б', 2) from system.numbers limit 10;
select 5 = positionUTF8(materialize('абвабв'), 'б', 3) from system.numbers limit 10;
select 5 = positionUTF8(materialize('абвабв'), 'б', 4) from system.numbers limit 10;
select 5 = positionUTF8(materialize('абвабв'), 'б', 5) from system.numbers limit 10;
select 0 = positionUTF8(materialize('абвабв'), 'б', 6) from system.numbers limit 10;
select 2 = positionUTF8(materialize('абвабв'), 'бва', 0) from system.numbers limit 10;
select 0 = positionUTF8(materialize('абвабв'), 'бва', 3) from system.numbers limit 10;
select 2 = positionUTF8('абвабв', materialize('б'), 0) from system.numbers limit 10;
select 2 = positionUTF8('абвабв', materialize('б'), 1) from system.numbers limit 10;
select 2 = positionUTF8('абвабв', materialize('б'), 2) from system.numbers limit 10;
select 5 = positionUTF8('абвабв', materialize('б'), 3) from system.numbers limit 10;
select 5 = positionUTF8('абвабв', materialize('б'), 4) from system.numbers limit 10;
select 5 = positionUTF8('абвабв', materialize('б'), 5) from system.numbers limit 10;
select 0 = positionUTF8('абвабв', materialize('б'), 6) from system.numbers limit 10;
select 2 = positionUTF8('абвабв', materialize('бва'), 0) from system.numbers limit 10;
select 0 = positionUTF8('абвабв', materialize('бва'), 3) from system.numbers limit 10;
select 2 = positionUTF8(materialize('абвабв'), materialize('б'), 0) from system.numbers limit 10;
select 2 = positionUTF8(materialize('абвабв'), materialize('б'), 1) from system.numbers limit 10;
select 2 = positionUTF8(materialize('абвабв'), materialize('б'), 2) from system.numbers limit 10;
select 5 = positionUTF8(materialize('абвабв'), materialize('б'), 3) from system.numbers limit 10;
select 5 = positionUTF8(materialize('абвабв'), materialize('б'), 4) from system.numbers limit 10;
select 5 = positionUTF8(materialize('абвабв'), materialize('б'), 5) from system.numbers limit 10;
select 0 = positionUTF8(materialize('абвабв'), materialize('б'), 6) from system.numbers limit 10;
select 2 = positionUTF8(materialize('абвабв'), materialize('бва'), 0) from system.numbers limit 10;
select 0 = positionUTF8(materialize('абвабв'), materialize('бва'), 3) from system.numbers limit 10;
select [2, 2, 2, 5, 5, 5, 0, 0, 0, 0] = groupArray(positionUTF8(materialize('абвабв'), materialize('б'), number)) from numbers(10);
select [2, 2, 2, 5, 5, 5, 0, 0, 0, 0] = groupArray(positionUTF8('абвабв', materialize('б'), number)) from numbers(10);
select [2, 2, 2, 5, 5, 5, 0, 0, 0, 0] = groupArray(positionUTF8('абвабв', 'б', number)) from numbers(10);
select [2, 2, 2, 5, 5, 5, 0, 0, 0, 0] = groupArray(positionUTF8(materialize('абвабв'), 'б', number)) from numbers(10);
select 1 = positionCaseInsensitive('', '');
select 1 = positionCaseInsensitive('abc', '');
select 0 = positionCaseInsensitive('', 'aBc');
@ -83,6 +163,10 @@ select 1 = positionCaseInsensitive(materialize('abc'), 'aBc') from system.number
select 2 = positionCaseInsensitive(materialize('abc'), 'Bc') from system.numbers limit 10;
select 3 = positionCaseInsensitive(materialize('abc'), 'C') from system.numbers limit 10;
select 6 = positionCaseInsensitive(materialize('abcabc'), 'C', 4);
select 6 = positionCaseInsensitive(materialize('abcabc'), 'C', 4) from system.numbers limit 10;
select 6 = positionCaseInsensitive(materialize('abcabc'), 'C', materialize(4)) from system.numbers limit 10;
select 1 = positionCaseInsensitive('', '');
select 1 = positionCaseInsensitive('абв', '');
select 0 = positionCaseInsensitive('', 'аБв');
@ -125,6 +209,10 @@ select 1 = positionCaseInsensitiveUTF8(materialize('абв'), 'аБв') from sys
select 2 = positionCaseInsensitiveUTF8(materialize('абв'), 'Бв') from system.numbers limit 10;
select 3 = positionCaseInsensitiveUTF8(materialize('абв'), 'В') from system.numbers limit 10;
select 6 = positionCaseInsensitiveUTF8(materialize('абвабв'), 'В', 4);
select 6 = positionCaseInsensitiveUTF8(materialize('абвабв'), 'В', 4) from system.numbers limit 10;
select 6 = positionCaseInsensitiveUTF8(materialize('абвабв'), 'В', materialize(4)) from system.numbers limit 10;
select position('' as h, '' as n) = positionCaseInsensitive(h, n);
select position('abc' as h, '' as n) = positionCaseInsensitive(n, n);
select 0 = positionCaseInsensitive('', 'aBc');