thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #37251 from ClickHouse/non_const_like

Browse Source 
Support non-constant SQL functions (NOT) (I)LIKE and MATCH
This commit is contained in:
Robert Schulze 2022-05-24 20:28:31 +02:00 committed by GitHub
commit 7348a0eb28
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 709 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/en/development/adding_test_queries.md
View File

@ -106,7 +106,7 @@ vim tests/queries/0_stateless/01521_dummy_test.sql
4) run the test, and put the result of that into the reference file:
```
clickhouse-client -nmT < tests/queries/0_stateless/01521_dummy_test.sql | tee tests/queries/0_stateless/01521_dummy_test.reference
clickhouse-client -nm < tests/queries/0_stateless/01521_dummy_test.sql | tee tests/queries/0_stateless/01521_dummy_test.reference
```
5) ensure everything is correct, if the test output is incorrect (due to some bug for example), adjust the reference file using text editor.

1
src/Common/OptimizedRegularExpression.h
View File

@ -107,3 +107,4 @@ private:
};
using OptimizedRegularExpression = OptimizedRegularExpressionImpl<true>;
using OptimizedRegularExpressionSingleThreaded = OptimizedRegularExpressionImpl<false>;

40
src/Functions/CountSubstringsImpl.h
View File

@ -26,19 +26,21 @@ struct CountSubstringsImpl
static constexpr bool supports_start_pos = true;
static constexpr auto name = Name::name;
static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {};}
using ResultType = UInt64;
/// Count occurrences of one substring in many strings.
static void vectorConstant(
const ColumnString::Chars & data,
const ColumnString::Offsets & offsets,
const ColumnString::Chars & haystack_data,
const ColumnString::Offsets & haystack_offsets,
const std::string & needle,
const ColumnPtr & start_pos,
PaddedPODArray<UInt64> & res)
{
const UInt8 * begin = data.data();
const UInt8 * const begin = haystack_data.data();
const UInt8 * const end = haystack_data.data() + haystack_data.size();
const UInt8 * pos = begin;
const UInt8 * end = pos + data.size();
/// FIXME: suboptimal
memset(&res[0], 0, res.size() * sizeof(res[0]));
@ -52,15 +54,15 @@ struct CountSubstringsImpl
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
{
/// Determine which index it refers to.
while (begin + offsets[i] <= pos)
while (begin + haystack_offsets[i] <= pos)
++i;
auto start = start_pos != nullptr ? start_pos->getUInt(i) : 0;
/// We check that the entry does not pass through the boundaries of strings.
if (pos + needle.size() < begin + offsets[i])
if (pos + needle.size() < begin + haystack_offsets[i])
{
auto res_pos = needle.size() + Impl::countChars(reinterpret_cast<const char *>(begin + offsets[i - 1]), reinterpret_cast<const char *>(pos));
auto res_pos = needle.size() + Impl::countChars(reinterpret_cast<const char *>(begin + haystack_offsets[i - 1]), reinterpret_cast<const char *>(pos));
if (res_pos >= start)
{
++res[i];
@ -69,14 +71,14 @@ struct CountSubstringsImpl
pos += needle.size();
continue;
}
pos = begin + offsets[i];
pos = begin + haystack_offsets[i];
++i;
}
}
/// Count number of occurrences of substring in string.
static void constantConstantScalar(
std::string data,
std::string haystack,
std::string needle,
UInt64 start_pos,
UInt64 & res)
@ -87,9 +89,9 @@ struct CountSubstringsImpl
return;
auto start = std::max(start_pos, UInt64(1));
size_t start_byte = Impl::advancePos(data.data(), data.data() + data.size(), start - 1) - data.data();
size_t start_byte = Impl::advancePos(haystack.data(), haystack.data() + haystack.size(), start - 1) - haystack.data();
size_t new_start_byte;
while ((new_start_byte = data.find(needle, start_byte)) != std::string::npos)
while ((new_start_byte = haystack.find(needle, start_byte)) != std::string::npos)
{
++res;
/// Intersecting substrings in haystack accounted only once
@ -99,21 +101,21 @@ struct CountSubstringsImpl
/// Count number of occurrences of substring in string starting from different positions.
static void constantConstant(
std::string data,
std::string haystack,
std::string needle,
const ColumnPtr & start_pos,
PaddedPODArray<UInt64> & res)
{
Impl::toLowerIfNeed(data);
Impl::toLowerIfNeed(haystack);
Impl::toLowerIfNeed(needle);
if (start_pos == nullptr)
{
constantConstantScalar(data, needle, 0, res[0]);
constantConstantScalar(haystack, needle, 0, res[0]);
return;
}
size_t haystack_size = Impl::countChars(data.data(), data.data() + data.size());
size_t haystack_size = Impl::countChars(haystack.data(), haystack.data() + haystack.size());
size_t size = start_pos != nullptr ? start_pos->size() : 0;
for (size_t i = 0; i < size; ++i)
@ -125,7 +127,7 @@ struct CountSubstringsImpl
res[i] = 0;
continue;
}
constantConstantScalar(data, needle, start, res[i]);
constantConstantScalar(haystack, needle, start, res[i]);
}
}
@ -228,6 +230,12 @@ struct CountSubstringsImpl
{
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support FixedString haystack argument", name);
}
template <typename... Args>
static void vectorFixedVector(Args &&...)
{
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support FixedString haystack argument", name);
}
};
}

41
src/Functions/FunctionsStringSearch.h
View File

@ -15,7 +15,6 @@
namespace DB
{
/** Search and replace functions in strings:
*
* position(haystack, needle) - the normal search for a substring in a string, returns the position (in bytes) of the found substring starting with 1, or 0 if no substring is found.
* positionUTF8(haystack, needle) - the same, but the position is calculated at code points, provided that the string is encoded in UTF-8.
* positionCaseInsensitive(haystack, needle)
@ -24,13 +23,29 @@ namespace DB
* like(haystack, pattern) - search by the regular expression LIKE; Returns 0 or 1. Case-insensitive, but only for Latin.
* notLike(haystack, pattern)
*
* ilike(haystack, pattern) - like 'like' but case-insensitive
* notIlike(haystack, pattern)
*
* match(haystack, pattern) - search by regular expression re2; Returns 0 or 1.
* multiMatchAny(haystack, [pattern_1, pattern_2, ..., pattern_n]) -- search by re2 regular expressions pattern_i; Returns 0 or 1 if any pattern_i matches.
* multiMatchAnyIndex(haystack, [pattern_1, pattern_2, ..., pattern_n]) -- search by re2 regular expressions pattern_i; Returns index of any match or zero if none;
* multiMatchAllIndices(haystack, [pattern_1, pattern_2, ..., pattern_n]) -- search by re2 regular expressions pattern_i; Returns an array of matched indices in any order;
*
* countSubstrings(haystack, needle) -- count number of occurrences of needle in haystack.
* countSubstringsCaseInsensitive(haystack, needle)
* countSubstringsCaseInsensitiveUTF8(haystack, needle)
*
* hasToken()
* hasTokenCaseInsensitive()
*
* JSON stuff:
* visitParamExtractBool()
* simpleJSONExtractBool()
* visitParamExtractFloat()
* simpleJSONExtractFloat()
* visitParamExtractInt()
* simpleJSONExtractInt()
* visitParamExtractUInt()
* simpleJSONExtractUInt()
* visitParamHas()
* simpleJSONHas()
*
* Applies regexp re2 and pulls:
* - the first subpattern, if the regexp has a subpattern;
@ -70,11 +85,7 @@ public:
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override
{
if (!Impl::use_default_implementation_for_constants)
return ColumnNumbers{};
if (!Impl::supports_start_pos)
return ColumnNumbers{1, 2};
return ColumnNumbers{1, 2, 3};
return Impl::getArgumentsThatAreAlwaysConstant();
}
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
@ -104,8 +115,6 @@ public:
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override
{
using ResultType = typename Impl::ResultType;
const ColumnPtr & column_haystack = arguments[0].column;
const ColumnPtr & column_needle = arguments[1].column;
@ -116,6 +125,8 @@ public:
const ColumnConst * col_haystack_const = typeid_cast<const ColumnConst *>(&*column_haystack);
const ColumnConst * col_needle_const = typeid_cast<const ColumnConst *>(&*column_needle);
using ResultType = typename Impl::ResultType;
if constexpr (!Impl::use_default_implementation_for_constants)
{
bool is_col_start_pos_const = column_start_pos == nullptr || isColumnConst(*column_start_pos);
@ -162,6 +173,14 @@ public:
col_needle_const->getValue<String>(),
column_start_pos,
vec_res);
else if (col_haystack_vector_fixed && col_needle_vector)
Impl::vectorFixedVector(
col_haystack_vector_fixed->getChars(),
col_haystack_vector_fixed->getN(),
col_needle_vector->getChars(),
col_needle_vector->getOffsets(),
column_start_pos,
vec_res);
else if (col_haystack_vector_fixed && col_needle_const)
Impl::vectorFixedConstant(
col_haystack_vector_fixed->getChars(),

42
src/Functions/FunctionsVisitParam.h
View File

@ -83,10 +83,12 @@ struct ExtractParamImpl
static constexpr bool supports_start_pos = false;
static constexpr auto name = Name::name;
static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {1, 2};}
/// It is assumed that `res` is the correct size and initialized with zeros.
static void vectorConstant(
const ColumnString::Chars & data,
const ColumnString::Offsets & offsets,
const ColumnString::Chars & haystack_data,
const ColumnString::Offsets & haystack_offsets,
std::string needle,
const ColumnPtr & start_pos,
PaddedPODArray<ResultType> & res)
@ -97,9 +99,9 @@ struct ExtractParamImpl
/// We are looking for a parameter simply as a substring of the form "name"
needle = "\"" + needle + "\":";
const UInt8 * begin = data.data();
const UInt8 * const begin = haystack_data.data();
const UInt8 * const end = haystack_data.data() + haystack_data.size();
const UInt8 * pos = begin;
const UInt8 * end = pos + data.size();
/// The current index in the string array.
size_t i = 0;
@ -110,19 +112,19 @@ struct ExtractParamImpl
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
{
/// Let's determine which index it belongs to.
while (begin + offsets[i] <= pos)
while (begin + haystack_offsets[i] <= pos)
{
res[i] = 0;
++i;
}
/// We check that the entry does not pass through the boundaries of strings.
if (pos + needle.size() < begin + offsets[i])
res[i] = ParamExtractor::extract(pos + needle.size(), begin + offsets[i] - 1); /// don't include terminating zero
if (pos + needle.size() < begin + haystack_offsets[i])
res[i] = ParamExtractor::extract(pos + needle.size(), begin + haystack_offsets[i] - 1); /// don't include terminating zero
else
res[i] = 0;
pos = begin + offsets[i];
pos = begin + haystack_offsets[i];
++i;
}
@ -145,6 +147,12 @@ struct ExtractParamImpl
{
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support FixedString haystack argument", name);
}
template <typename... Args>
static void vectorFixedVector(Args &&...)
{
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support FixedString haystack argument", name);
}
};
@ -153,20 +161,20 @@ struct ExtractParamImpl
template <typename ParamExtractor>
struct ExtractParamToStringImpl
{
static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets,
static void vector(const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets,
std::string needle,
ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets)
{
/// Constant 5 is taken from a function that performs a similar task FunctionsStringSearch.h::ExtractImpl
res_data.reserve(data.size() / 5);
res_offsets.resize(offsets.size());
res_data.reserve(haystack_data.size() / 5);
res_offsets.resize(haystack_offsets.size());
/// We are looking for a parameter simply as a substring of the form "name"
needle = "\"" + needle + "\":";
const UInt8 * begin = data.data();
const UInt8 * const begin = haystack_data.data();
const UInt8 * const end = haystack_data.data() + haystack_data.size();
const UInt8 * pos = begin;
const UInt8 * end = pos + data.size();
/// The current index in the string array.
size_t i = 0;
@ -177,7 +185,7 @@ struct ExtractParamToStringImpl
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
{
/// Determine which index it belongs to.
while (begin + offsets[i] <= pos)
while (begin + haystack_offsets[i] <= pos)
{
res_data.push_back(0);
res_offsets[i] = res_data.size();
@ -185,10 +193,10 @@ struct ExtractParamToStringImpl
}
/// We check that the entry does not pass through the boundaries of strings.
if (pos + needle.size() < begin + offsets[i])
ParamExtractor::extract(pos + needle.size(), begin + offsets[i], res_data);
if (pos + needle.size() < begin + haystack_offsets[i])
ParamExtractor::extract(pos + needle.size(), begin + haystack_offsets[i], res_data);
pos = begin + offsets[i];
pos = begin + haystack_offsets[i];
res_data.push_back(0);
res_offsets[i] = res_data.size();

35
src/Functions/HasTokenImpl.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <Columns/ColumnString.h>
#include <Core/ColumnNumbers.h>
namespace DB
@ -14,7 +15,7 @@ namespace ErrorCodes
/** Token search the string, means that needle must be surrounded by some separator chars, like whitespace or puctuation.
*/
template <typename Name, typename TokenSearcher, bool negate_result = false>
template <typename Name, typename TokenSearcher, bool negate>
struct HasTokenImpl
{
using ResultType = UInt8;
@ -23,9 +24,11 @@ struct HasTokenImpl
static constexpr bool supports_start_pos = false;
static constexpr auto name = Name::name;
static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {1, 2};}
static void vectorConstant(
const ColumnString::Chars & data,
const ColumnString::Offsets & offsets,
const ColumnString::Chars & haystack_data,
const ColumnString::Offsets & haystack_offsets,
const std::string & pattern,
const ColumnPtr & start_pos,
PaddedPODArray<UInt8> & res)
@ -33,12 +36,12 @@ struct HasTokenImpl
if (start_pos != nullptr)
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' does not support start_pos argument", name);
if (offsets.empty())
if (haystack_offsets.empty())
return;
const UInt8 * begin = data.data();
const UInt8 * const begin = haystack_data.data();
const UInt8 * const end = haystack_data.data() + haystack_data.size();
const UInt8 * pos = begin;
const UInt8 * end = pos + data.size();
/// The current index in the array of strings.
size_t i = 0;
@ -49,25 +52,25 @@ struct HasTokenImpl
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
{
/// Let's determine which index it refers to.
while (begin + offsets[i] <= pos)
while (begin + haystack_offsets[i] <= pos)
{
res[i] = negate_result;
res[i] = negate;
++i;
}
/// We check that the entry does not pass through the boundaries of strings.
if (pos + pattern.size() < begin + offsets[i])
res[i] = !negate_result;
if (pos + pattern.size() < begin + haystack_offsets[i])
res[i] = !negate;
else
res[i] = negate_result;
res[i] = negate;
pos = begin + offsets[i];
pos = begin + haystack_offsets[i];
++i;
}
/// Tail, in which there can be no substring.
if (i < res.size())
memset(&res[i], negate_result, (res.size() - i) * sizeof(res[0]));
memset(&res[i], negate, (res.size() - i) * sizeof(res[0]));
}
template <typename... Args>
@ -88,6 +91,12 @@ struct HasTokenImpl
{
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support FixedString haystack argument", name);
}
template <typename... Args>
static void vectorFixedVector(Args &&...)
{
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support FixedString haystack argument", name);
}
};
}

458
src/Functions/MatchImpl.h
View File

@ -4,6 +4,7 @@
#include <base/types.h>
#include <Common/Volnitsky.h>
#include <Columns/ColumnString.h>
#include <Core/ColumnNumbers.h>
#include "Regexps.h"
#include "config_functions.h"
@ -17,24 +18,24 @@ namespace DB
namespace ErrorCodes
{
extern const int ILLEGAL_COLUMN;
extern const int LOGICAL_ERROR;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
namespace impl
{
/// Is the [I]LIKE expression reduced to finding a substring in a string?
static inline bool likePatternIsStrstr(const String & pattern, String & res)
inline bool likePatternIsSubstring(std::string_view pattern, String & res)
{
if (pattern.size() < 2 || pattern.front() != '%' || pattern.back() != '%')
return false;
res = "";
res.clear();
res.reserve(pattern.size() - 2);
const char * pos = pattern.data();
const char * end = pos + pattern.size();
++pos;
--end;
const char * pos = pattern.data() + 1;
const char * const end = pattern.data() + pattern.size() - 1;
while (pos < end)
{
@ -60,17 +61,24 @@ static inline bool likePatternIsStrstr(const String & pattern, String & res)
return true;
}
/** 'like' - if true, treat pattern as SQL LIKE or ILIKE; if false - treat pattern as re2 regexp.
}
/** 'like' - if true, treat pattern as SQL LIKE, otherwise as re2 regexp.
* 'negate' - if true, negate result
* 'case_insensitive' - if true, match case insensitively
*
* NOTE: We want to run regexp search for whole columns by one call (as implemented in function 'position')
* but for that, regexp engine must support \0 bytes and their interpretation as string boundaries.
*/
template <typename Name, bool like, bool revert = false, bool case_insensitive = false>
template <typename Name, bool like, bool negate, bool case_insensitive>
struct MatchImpl
{
static constexpr bool use_default_implementation_for_constants = true;
static constexpr bool supports_start_pos = false;
static constexpr auto name = Name::name;
static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {2};}
using ResultType = UInt8;
using Searcher = std::conditional_t<case_insensitive,
@ -78,25 +86,25 @@ struct MatchImpl
VolnitskyUTF8>;
static void vectorConstant(
const ColumnString::Chars & data,
const ColumnString::Offsets & offsets,
const String & pattern,
const ColumnPtr & start_pos,
const ColumnString::Chars & haystack_data,
const ColumnString::Offsets & haystack_offsets,
const String & needle,
const ColumnPtr & start_pos_,
PaddedPODArray<UInt8> & res)
{
if (start_pos != nullptr)
if (start_pos_ != nullptr)
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Function '{}' doesn't support start_pos argument", name);
if (offsets.empty())
if (haystack_offsets.empty())
return;
/// A simple case where the [I]LIKE expression reduces to finding a substring in a string
String strstr_pattern;
if (like && likePatternIsStrstr(pattern, strstr_pattern))
if (like && impl::likePatternIsSubstring(needle, strstr_pattern))
{
const UInt8 * const begin = data.data();
const UInt8 * const end = data.data() + data.size();
const UInt8 * const begin = haystack_data.data();
const UInt8 * const end = haystack_data.data() + haystack_data.size();
const UInt8 * pos = begin;
/// The current index in the array of strings.
@ -109,31 +117,29 @@ struct MatchImpl
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
{
/// Let's determine which index it refers to.
while (begin + offsets[i] <= pos)
while (begin + haystack_offsets[i] <= pos)
{
res[i] = revert;
res[i] = negate;
++i;
}
/// We check that the entry does not pass through the boundaries of strings.
if (pos + strstr_pattern.size() < begin + offsets[i])
res[i] = !revert;
if (pos + strstr_pattern.size() < begin + haystack_offsets[i])
res[i] = !negate;
else
res[i] = revert;
res[i] = negate;
pos = begin + offsets[i];
pos = begin + haystack_offsets[i];
++i;
}
/// Tail, in which there can be no substring.
if (i < res.size())
memset(&res[i], revert, (res.size() - i) * sizeof(res[0]));
memset(&res[i], negate, (res.size() - i) * sizeof(res[0]));
}
else
{
size_t size = offsets.size();
auto regexp = Regexps::get<like, true, case_insensitive>(pattern);
auto regexp = Regexps::get<like, true, case_insensitive>(needle);
String required_substring;
bool is_trivial;
@ -141,37 +147,39 @@ struct MatchImpl
regexp->getAnalyzeResult(required_substring, is_trivial, required_substring_is_prefix);
size_t haystack_size = haystack_offsets.size();
if (required_substring.empty())
{
if (!regexp->getRE2()) /// An empty regexp. Always matches.
{
if (size)
memset(res.data(), 1, size * sizeof(res[0]));
if (haystack_size)
memset(res.data(), 1, haystack_size * sizeof(res[0]));
}
else
{
size_t prev_offset = 0;
for (size_t i = 0; i < size; ++i)
for (size_t i = 0; i < haystack_size; ++i)
{
res[i] = revert
res[i] = negate
^ regexp->getRE2()->Match(
re2_st::StringPiece(reinterpret_cast<const char *>(&data[prev_offset]), offsets[i] - prev_offset - 1),
{reinterpret_cast<const char *>(&haystack_data[prev_offset]), haystack_offsets[i] - prev_offset - 1},
0,
offsets[i] - prev_offset - 1,
haystack_offsets[i] - prev_offset - 1,
re2_st::RE2::UNANCHORED,
nullptr,
0);
prev_offset = offsets[i];
prev_offset = haystack_offsets[i];
}
}
}
else
{
/// NOTE This almost matches with the case of LikePatternIsStrstr.
/// NOTE This almost matches with the case of impl::likePatternIsSubstring.
const UInt8 * const begin = data.data();
const UInt8 * const end = data.begin() + data.size();
const UInt8 * const begin = haystack_data.data();
const UInt8 * const end = haystack_data.begin() + haystack_data.size();
const UInt8 * pos = begin;
/// The current index in the array of strings.
@ -183,79 +191,78 @@ struct MatchImpl
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
{
/// Determine which index it refers to.
while (begin + offsets[i] <= pos)
while (begin + haystack_offsets[i] <= pos)
{
res[i] = revert;
res[i] = negate;
++i;
}
/// We check that the entry does not pass through the boundaries of strings.
if (pos + required_substring.size() < begin + offsets[i])
if (pos + required_substring.size() < begin + haystack_offsets[i])
{
/// And if it does not, if necessary, we check the regexp.
if (is_trivial)
res[i] = !revert;
res[i] = !negate;
else
{
const char * str_data = reinterpret_cast<const char *>(&data[offsets[i - 1]]);
size_t str_size = offsets[i] - offsets[i - 1] - 1;
const char * str_data = reinterpret_cast<const char *>(&haystack_data[haystack_offsets[i - 1]]);
size_t str_size = haystack_offsets[i] - haystack_offsets[i - 1] - 1;
/** Even in the case of `required_substring_is_prefix` use UNANCHORED check for regexp,
* so that it can match when `required_substring` occurs into the string several times,
* and at the first occurrence, the regexp is not a match.
*/
const size_t start_pos = (required_substring_is_prefix) ? (reinterpret_cast<const char *>(pos) - str_data) : 0;
const size_t end_pos = str_size;
if (required_substring_is_prefix)
res[i] = revert
^ regexp->getRE2()->Match(
re2_st::StringPiece(str_data, str_size),
reinterpret_cast<const char *>(pos) - str_data,
str_size,
re2_st::RE2::UNANCHORED,
nullptr,
0);
else
res[i] = revert
^ regexp->getRE2()->Match(
re2_st::StringPiece(str_data, str_size), 0, str_size, re2_st::RE2::UNANCHORED, nullptr, 0);
res[i] = negate
^ regexp->getRE2()->Match(
{str_data, str_size},
start_pos,
end_pos,
re2_st::RE2::UNANCHORED,
nullptr,
0);
}
}
else
res[i] = revert;
res[i] = negate;
pos = begin + offsets[i];
pos = begin + haystack_offsets[i];
++i;
}
/// Tail, in which there can be no substring.
if (i < res.size())
memset(&res[i], revert, (res.size() - i) * sizeof(res[0]));
memset(&res[i], negate, (res.size() - i) * sizeof(res[0]));
}
}
}
/// Very carefully crafted copy-paste.
static void vectorFixedConstant(
const ColumnString::Chars & data, size_t n, const String & pattern,
const ColumnString::Chars & haystack,
size_t N,
const String & needle,
PaddedPODArray<UInt8> & res)
{
if (data.empty())
if (haystack.empty())
return;
/// A simple case where the LIKE expression reduces to finding a substring in a string
String strstr_pattern;
if (like && likePatternIsStrstr(pattern, strstr_pattern))
if (like && impl::likePatternIsSubstring(needle, strstr_pattern))
{
const UInt8 * begin = data.data();
const UInt8 * const begin = haystack.data();
const UInt8 * const end = haystack.data() + haystack.size();
const UInt8 * pos = begin;
const UInt8 * end = pos + data.size();
size_t i = 0;
const UInt8 * next_pos = begin;
/// If pattern is larger than string size - it cannot be found.
if (strstr_pattern.size() <= n)
/// If needle is larger than string size - it cannot be found.
if (strstr_pattern.size() <= N)
{
Searcher searcher(strstr_pattern.data(), strstr_pattern.size(), end - pos);
@ -263,19 +270,19 @@ struct MatchImpl
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
{
/// Let's determine which index it refers to.
while (next_pos + n <= pos)
while (next_pos + N <= pos)
{
res[i] = revert;
next_pos += n;
res[i] = negate;
next_pos += N;
++i;
}
next_pos += n;
next_pos += N;
/// We check that the entry does not pass through the boundaries of strings.
if (pos + strstr_pattern.size() <= next_pos)
res[i] = !revert;
res[i] = !negate;
else
res[i] = revert;
res[i] = negate;
pos = next_pos;
++i;
@ -284,13 +291,11 @@ struct MatchImpl
/// Tail, in which there can be no substring.
if (i < res.size())
memset(&res[i], revert, (res.size() - i) * sizeof(res[0]));
memset(&res[i], negate, (res.size() - i) * sizeof(res[0]));
}
else
{
size_t size = data.size() / n;
auto regexp = Regexps::get<like, true, case_insensitive>(pattern);
auto regexp = Regexps::get<like, true, case_insensitive>(needle);
String required_substring;
bool is_trivial;
@ -298,44 +303,46 @@ struct MatchImpl
regexp->getAnalyzeResult(required_substring, is_trivial, required_substring_is_prefix);
const size_t haystack_size = haystack.size() / N;
if (required_substring.empty())
{
if (!regexp->getRE2()) /// An empty regexp. Always matches.
{
if (size)
memset(res.data(), 1, size * sizeof(res[0]));
if (haystack_size)
memset(res.data(), 1, haystack_size * sizeof(res[0]));
}
else
{
size_t offset = 0;
for (size_t i = 0; i < size; ++i)
for (size_t i = 0; i < haystack_size; ++i)
{
res[i] = revert
res[i] = negate
^ regexp->getRE2()->Match(
re2_st::StringPiece(reinterpret_cast<const char *>(&data[offset]), n),
{reinterpret_cast<const char *>(&haystack[offset]), N},
0,
n,
N,
re2_st::RE2::UNANCHORED,
nullptr,
0);
offset += n;
offset += N;
}
}
}
else
{
/// NOTE This almost matches with the case of LikePatternIsStrstr.
/// NOTE This almost matches with the case of likePatternIsSubstring.
const UInt8 * begin = data.data();
const UInt8 * const begin = haystack.data();
const UInt8 * const end = haystack.data() + haystack.size();
const UInt8 * pos = begin;
const UInt8 * end = pos + data.size();
size_t i = 0;
const UInt8 * next_pos = begin;
/// If required substring is larger than string size - it cannot be found.
if (required_substring.size() <= n)
if (required_substring.size() <= N)
{
Searcher searcher(required_substring.data(), required_substring.size(), end - pos);
@ -343,46 +350,43 @@ struct MatchImpl
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
{
/// Let's determine which index it refers to.
while (next_pos + n <= pos)
while (next_pos + N <= pos)
{
res[i] = revert;
next_pos += n;
res[i] = negate;
next_pos += N;
++i;
}
next_pos += n;
next_pos += N;
if (pos + required_substring.size() <= next_pos)
{
/// And if it does not, if necessary, we check the regexp.
if (is_trivial)
res[i] = !revert;
res[i] = !negate;
else
{
const char * str_data = reinterpret_cast<const char *>(next_pos - n);
const char * str_data = reinterpret_cast<const char *>(next_pos - N);
/** Even in the case of `required_substring_is_prefix` use UNANCHORED check for regexp,
* so that it can match when `required_substring` occurs into the string several times,
* and at the first occurrence, the regexp is not a match.
*/
const size_t start_pos = (required_substring_is_prefix) ? (reinterpret_cast<const char *>(pos) - str_data) : 0;
const size_t end_pos = N;
if (required_substring_is_prefix)
res[i] = revert
^ regexp->getRE2()->Match(
re2_st::StringPiece(str_data, n),
reinterpret_cast<const char *>(pos) - str_data,
n,
re2_st::RE2::UNANCHORED,
nullptr,
0);
else
res[i] = revert
^ regexp->getRE2()->Match(
re2_st::StringPiece(str_data, n), 0, n, re2_st::RE2::UNANCHORED, nullptr, 0);
res[i] = negate
^ regexp->getRE2()->Match(
{str_data, N},
start_pos,
end_pos,
re2_st::RE2::UNANCHORED,
nullptr,
0);
}
}
else
res[i] = revert;
res[i] = negate;
pos = next_pos;
++i;
@ -391,22 +395,248 @@ struct MatchImpl
/// Tail, in which there can be no substring.
if (i < res.size())
memset(&res[i], revert, (res.size() - i) * sizeof(res[0]));
memset(&res[i], negate, (res.size() - i) * sizeof(res[0]));
}
}
}
template <typename... Args>
static void vectorVector(Args &&...)
static void vectorVector(
const ColumnString::Chars & haystack_data,
const ColumnString::Offsets & haystack_offsets,
const ColumnString::Chars & needle_data,
const ColumnString::Offsets & needle_offset,
const ColumnPtr & start_pos_,
PaddedPODArray<UInt8> & res)
{
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support non-constant needle argument", name);
const size_t haystack_size = haystack_offsets.size();
if (haystack_size != needle_offset.size())
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Function '{}' unexpectedly received a different number of haystacks and needles", name);
if (start_pos_ != nullptr)
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Function '{}' doesn't support start_pos argument", name);
if (haystack_offsets.empty())
return;
String required_substr;
bool is_trivial;
bool required_substring_is_prefix; /// for `anchored` execution of the regexp.
size_t prev_haystack_offset = 0;
size_t prev_needle_offset = 0;
for (size_t i = 0; i < haystack_size; ++i)
{
const auto * const cur_haystack_data = &haystack_data[prev_haystack_offset];
const size_t cur_haystack_length = haystack_offsets[i] - prev_haystack_offset - 1;
const auto * const cur_needle_data = &needle_data[prev_needle_offset];
const size_t cur_needle_length = needle_offset[i] - prev_needle_offset - 1;
const auto & needle = String(
reinterpret_cast<const char *>(cur_needle_data),
cur_needle_length);
if (like && impl::likePatternIsSubstring(needle, required_substr))
{
if (required_substr.size() > cur_haystack_length)
res[i] = negate;
else
{
Searcher searcher(required_substr.data(), required_substr.size(), cur_haystack_length);
const auto * match = searcher.search(cur_haystack_data, cur_haystack_length);
res[i] = negate
^ (match != cur_haystack_data + cur_haystack_length);
}
}
else
{
// each row is expected to contain a different like/re2 pattern
// --> bypass the regexp cache, instead construct the pattern on-the-fly
const int flags = Regexps::buildRe2Flags<true, case_insensitive>();
const auto & regexp = Regexps::Regexp(Regexps::createRegexp<like>(needle, flags));
regexp.getAnalyzeResult(required_substr, is_trivial, required_substring_is_prefix);
if (required_substr.empty())
{
if (!regexp.getRE2()) /// An empty regexp. Always matches.
{
res[i] = 1;
}
else
{
res[i] = negate
^ regexp.getRE2()->Match(
{reinterpret_cast<const char *>(cur_haystack_data), cur_haystack_length},
0,
cur_haystack_length,
re2_st::RE2::UNANCHORED,
nullptr,
0);
}
}
else
{
Searcher searcher(required_substr.data(), required_substr.size(), cur_haystack_length);
const auto * match = searcher.search(cur_haystack_data, cur_haystack_length);
if (match == cur_haystack_data + cur_haystack_length)
{
res[i] = negate; // no match
}
else
{
if (is_trivial)
{
res[i] = !negate; // no wildcards in pattern
}
else
{
const size_t start_pos = (required_substring_is_prefix) ? (match - cur_haystack_data) : 0;
const size_t end_pos = cur_haystack_length;
res[i] = negate
^ regexp.getRE2()->Match(
{reinterpret_cast<const char *>(cur_haystack_data), cur_haystack_length},
start_pos,
end_pos,
re2_st::RE2::UNANCHORED,
nullptr,
0);
}
}
}
}
prev_haystack_offset = haystack_offsets[i];
prev_needle_offset = needle_offset[i];
}
}
static void vectorFixedVector(
const ColumnString::Chars & haystack,
size_t N,
const ColumnString::Chars & needle_data,
const ColumnString::Offsets & needle_offset,
const ColumnPtr & start_pos_,
PaddedPODArray<UInt8> & res)
{
const size_t haystack_size = haystack.size()/N;
if (haystack_size != needle_offset.size())
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Function '{}' unexpectedly received a different number of haystacks and needles", name);
if (start_pos_ != nullptr)
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Function '{}' doesn't support start_pos argument", name);
if (haystack.empty())
return;
String required_substr;
bool is_trivial;
bool required_substring_is_prefix; // for `anchored` execution of the regexp.
size_t prev_haystack_offset = 0;
size_t prev_needle_offset = 0;
for (size_t i = 0; i < haystack_size; ++i)
{
const auto * const cur_haystack_data = &haystack[prev_haystack_offset];
const size_t cur_haystack_length = N;
const auto * const cur_needle_data = &needle_data[prev_needle_offset];
const size_t cur_needle_length = needle_offset[i] - prev_needle_offset - 1;
const auto & needle = String(
reinterpret_cast<const char *>(cur_needle_data),
cur_needle_length);
if (like && impl::likePatternIsSubstring(needle, required_substr))
{
if (required_substr.size() > cur_haystack_length)
res[i] = negate;
else
{
Searcher searcher(required_substr.data(), required_substr.size(), cur_haystack_length);
const auto * match = searcher.search(cur_haystack_data, cur_haystack_length);
res[i] = negate
^ (match != cur_haystack_data + cur_haystack_length);
}
}
else
{
// each row is expected to contain a different like/re2 pattern
// --> bypass the regexp cache, instead construct the pattern on-the-fly
const int flags = Regexps::buildRe2Flags<true, case_insensitive>();
const auto & regexp = Regexps::Regexp(Regexps::createRegexp<like>(needle, flags));
regexp.getAnalyzeResult(required_substr, is_trivial, required_substring_is_prefix);
if (required_substr.empty())
{
if (!regexp.getRE2()) /// An empty regexp. Always matches.
{
res[i] = 1;
}
else
{
res[i] = negate
^ regexp.getRE2()->Match(
{reinterpret_cast<const char *>(cur_haystack_data), cur_haystack_length},
0,
cur_haystack_length,
re2_st::RE2::UNANCHORED,
nullptr,
0);
}
}
else
{
Searcher searcher(required_substr.data(), required_substr.size(), cur_haystack_length);
const auto * match = searcher.search(cur_haystack_data, cur_haystack_length);
if (match == cur_haystack_data + cur_haystack_length)
{
res[i] = negate; // no match
}
else
{
if (is_trivial)
{
res[i] = !negate; // no wildcards in pattern
}
else
{
const size_t start_pos = (required_substring_is_prefix) ? (match - cur_haystack_data) : 0;
const size_t end_pos = cur_haystack_length;
res[i] = negate
^ regexp.getRE2()->Match(
{reinterpret_cast<const char *>(cur_haystack_data), cur_haystack_length},
start_pos,
end_pos,
re2_st::RE2::UNANCHORED,
nullptr,
0);
}
}
}
}
prev_haystack_offset += N;
prev_needle_offset = needle_offset[i];
}
}
/// Search different needles in single haystack.
template <typename... Args>
static void constantVector(Args &&...)
{
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support non-constant needle argument", name);
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support search with non-constant needles in constant haystack", name);
}
};

2
src/Functions/MultiMatchAllIndicesImpl.h
View File

@ -11,8 +11,6 @@
#if USE_HYPERSCAN
# include <hs.h>
#else
# include "MatchImpl.h"
#endif

2
src/Functions/MultiMatchAnyImpl.h
View File

@ -120,7 +120,7 @@ struct MultiMatchAnyImpl
memset(accum.data(), 0, accum.size());
for (size_t j = 0; j < needles.size(); ++j)
{
MatchImpl<Name, false, false>::vectorConstant(haystack_data, haystack_offsets, needles[j].toString(), nullptr, accum);
MatchImpl<Name, false, false, false>::vectorConstant(haystack_data, haystack_offsets, needles[j].toString(), nullptr, accum);
for (size_t i = 0; i < res.size(); ++i)
{
if constexpr (FindAny)

26
src/Functions/PositionImpl.h
View File

@ -182,19 +182,21 @@ struct PositionImpl
static constexpr bool supports_start_pos = true;
static constexpr auto name = Name::name;
static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {};}
using ResultType = UInt64;
/// Find one substring in many strings.
static void vectorConstant(
const ColumnString::Chars & data,
const ColumnString::Offsets & offsets,
const ColumnString::Chars & haystack_data,
const ColumnString::Offsets & haystack_offsets,
const std::string & needle,
const ColumnPtr & start_pos,
PaddedPODArray<UInt64> & res)
{
const UInt8 * begin = data.data();
const UInt8 * const begin = haystack_data.data();
const UInt8 * const end = haystack_data.data() + haystack_data.size();
const UInt8 * pos = begin;
const UInt8 * end = pos + data.size();
/// Current index in the array of strings.
size_t i = 0;
@ -205,7 +207,7 @@ struct PositionImpl
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
{
/// Determine which index it refers to.
while (begin + offsets[i] <= pos)
while (begin + haystack_offsets[i] <= pos)
{
res[i] = 0;
++i;
@ -213,14 +215,14 @@ struct PositionImpl
auto start = start_pos != nullptr ? start_pos->getUInt(i) : 0;
/// We check that the entry does not pass through the boundaries of strings.
if (pos + needle.size() < begin + offsets[i])
if (pos + needle.size() < begin + haystack_offsets[i])
{
auto res_pos = 1 + Impl::countChars(reinterpret_cast<const char *>(begin + offsets[i - 1]), reinterpret_cast<const char *>(pos));
auto res_pos = 1 + Impl::countChars(reinterpret_cast<const char *>(begin + haystack_offsets[i - 1]), reinterpret_cast<const char *>(pos));
if (res_pos < start)
{
pos = reinterpret_cast<const UInt8 *>(Impl::advancePos(
reinterpret_cast<const char *>(pos),
reinterpret_cast<const char *>(begin + offsets[i]),
reinterpret_cast<const char *>(begin + haystack_offsets[i]),
start - res_pos));
continue;
}
@ -230,7 +232,7 @@ struct PositionImpl
{
res[i] = 0;
}
pos = begin + offsets[i];
pos = begin + haystack_offsets[i];
++i;
}
@ -411,6 +413,12 @@ struct PositionImpl
{
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support FixedString haystack argument", name);
}
template <typename... Args>
static void vectorFixedVector(Args &&...)
{
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support FixedString haystack argument", name);
}
};
}

22
src/Functions/Regexps.h
View File

@ -38,7 +38,7 @@ namespace ErrorCodes
namespace Regexps
{
using Regexp = OptimizedRegularExpressionImpl<false>;
using Regexp = OptimizedRegularExpressionSingleThreaded;
using Pool = ObjectPoolMap<Regexp, String>;
template <bool like>
@ -50,6 +50,17 @@ namespace Regexps
return {pattern, flags};
}
template<bool no_capture, bool case_insensitive>
inline int buildRe2Flags()
{
int flags = OptimizedRegularExpression::RE_DOT_NL;
if constexpr (no_capture)
flags |= OptimizedRegularExpression::RE_NO_CAPTURE;
if constexpr (case_insensitive)
flags |= OptimizedRegularExpression::RE_CASELESS;
return flags;
}
/** Returns holder of an object from Pool.
* You must hold the ownership while using the object.
* In destructor, it returns the object back to the Pool for further reuse.
@ -62,14 +73,7 @@ namespace Regexps
return known_regexps.get(pattern, [&pattern]
{
int flags = OptimizedRegularExpression::RE_DOT_NL;
if (no_capture)
flags |= OptimizedRegularExpression::RE_NO_CAPTURE;
if (case_insensitive)
flags |= Regexps::Regexp::RE_CASELESS;
const int flags = buildRe2Flags<no_capture, case_insensitive>();
ProfileEvents::increment(ProfileEvents::RegexpCreated);
return new Regexp{createRegexp<like>(pattern, flags)};
});

2
src/Functions/ilike.cpp
View File

@ -12,7 +12,7 @@ struct NameILike
static constexpr auto name = "ilike";
};
using ILikeImpl = MatchImpl<NameILike, true, false, /*case-insensitive*/true>;
using ILikeImpl = MatchImpl<NameILike, true, false, true>;
using FunctionILike = FunctionsStringSearch<ILikeImpl>;
}

2
src/Functions/like.h
View File

@ -11,7 +11,7 @@ struct NameLike
static constexpr auto name = "like";
};
using LikeImpl = MatchImpl<NameLike, /*SQL LIKE */ true, /*revert*/false>;
using LikeImpl = MatchImpl<NameLike, true, false, false>;
using FunctionLike = FunctionsStringSearch<LikeImpl>;
}

2
src/Functions/match.cpp
View File

@ -13,7 +13,7 @@ struct NameMatch
static constexpr auto name = "match";
};
using FunctionMatch = FunctionsStringSearch<MatchImpl<NameMatch, false>>;
using FunctionMatch = FunctionsStringSearch<MatchImpl<NameMatch, false, false, false>>;
}

2
src/Functions/notILike.cpp
View File

@ -12,7 +12,7 @@ struct NameNotILike
static constexpr auto name = "notILike";
};
using NotILikeImpl = MatchImpl<NameNotILike, true, true, /*case-insensitive*/true>;
using NotILikeImpl = MatchImpl<NameNotILike, true, true, true>;
using FunctionNotILike = FunctionsStringSearch<NotILikeImpl>;
}

2
src/Functions/notLike.cpp
View File

@ -12,7 +12,7 @@ struct NameNotLike
static constexpr auto name = "notLike";
};
using FunctionNotLike = FunctionsStringSearch<MatchImpl<NameNotLike, true, true>>;
using FunctionNotLike = FunctionsStringSearch<MatchImpl<NameNotLike, true, true, false>>;
}

190
tests/queries/0_stateless/02294_stringsearch_with_nonconst_needle.reference Normal file
View File

@ -0,0 +1,190 @@
LIKE
1 Hello 0
2 Hello % 1
3 Hello %% 1
4 Hello %%% 1
5 Hello %_% 1
6 Hello _ 0
7 Hello _% 1
8 Hello %_ 1
9 Hello H%o 1
10 hello H%0 0
11 hello h%o 1
12 Hello h%o 0
13 OHello %lhell% 0
14 OHello %hell% 0
15 hEllo %HEL% 0
16 abcdef %aBc%def% 0
17 ABCDDEF %abc%def% 0
18 Abc\nDef %abc%def% 0
19 abc\ntdef %abc%def% 1
20 abct\ndef %abc%dEf% 0
21 abc\n\ndeF %abc%def% 0
22 abc\n\ntdef %abc%deF% 0
23 Abc\nt\ndef %abc%def% 0
24 abct\n\ndef %abc%def% 1
25 ab\ndef %Abc%def% 0
26 aBc\nef %ABC%DEF% 0
27 ёЁё Ё%Ё 0
28 ощщЁё Щ%Ё 0
29 ощЩЁё %Щ%Ё 0
30 Щущпандер %щп%е% 1
31 Щущпандер %щП%е% 0
32 ощщЁё %щ% 1
33 ощЩЁё %ё% 1
34 Hello .* 0
35 Hello .*ell.* 0
36 Hello o$ 0
37 Hello hE.*lO 0
NOT LIKE
1 Hello 1
2 Hello % 0
3 Hello %% 0
4 Hello %%% 0
5 Hello %_% 0
6 Hello _ 1
7 Hello _% 0
8 Hello %_ 0
9 Hello H%o 0
10 hello H%0 1
11 hello h%o 0
12 Hello h%o 1
13 OHello %lhell% 1
14 OHello %hell% 1
15 hEllo %HEL% 1
16 abcdef %aBc%def% 1
17 ABCDDEF %abc%def% 1
18 Abc\nDef %abc%def% 1
19 abc\ntdef %abc%def% 0
20 abct\ndef %abc%dEf% 1
21 abc\n\ndeF %abc%def% 1
22 abc\n\ntdef %abc%deF% 1
23 Abc\nt\ndef %abc%def% 1
24 abct\n\ndef %abc%def% 0
25 ab\ndef %Abc%def% 1
26 aBc\nef %ABC%DEF% 1
27 ёЁё Ё%Ё 1
28 ощщЁё Щ%Ё 1
29 ощЩЁё %Щ%Ё 1
30 Щущпандер %щп%е% 0
31 Щущпандер %щП%е% 1
32 ощщЁё %щ% 0
33 ощЩЁё %ё% 0
34 Hello .* 1
35 Hello .*ell.* 1
36 Hello o$ 1
37 Hello hE.*lO 1
ILIKE
1 Hello 0
2 Hello % 1
3 Hello %% 1
4 Hello %%% 1
5 Hello %_% 1
6 Hello _ 0
7 Hello _% 1
8 Hello %_ 1
9 Hello H%o 1
10 hello H%0 0
11 hello h%o 1
12 Hello h%o 1
13 OHello %lhell% 0
14 OHello %hell% 1
15 hEllo %HEL% 1
16 abcdef %aBc%def% 1
17 ABCDDEF %abc%def% 1
18 Abc\nDef %abc%def% 1
19 abc\ntdef %abc%def% 1
20 abct\ndef %abc%dEf% 1
21 abc\n\ndeF %abc%def% 1
22 abc\n\ntdef %abc%deF% 1
23 Abc\nt\ndef %abc%def% 1
24 abct\n\ndef %abc%def% 1
25 ab\ndef %Abc%def% 0
26 aBc\nef %ABC%DEF% 0
27 ёЁё Ё%Ё 1
28 ощщЁё Щ%Ё 0
29 ощЩЁё %Щ%Ё 1
30 Щущпандер %щп%е% 1
31 Щущпандер %щП%е% 1
32 ощщЁё %щ% 1
33 ощЩЁё %ё% 1
34 Hello .* 0
35 Hello .*ell.* 0
36 Hello o$ 0
37 Hello hE.*lO 0
NOT ILIKE
1 Hello 1
2 Hello % 0
3 Hello %% 0
4 Hello %%% 0
5 Hello %_% 0
6 Hello _ 1
7 Hello _% 0
8 Hello %_ 0
9 Hello H%o 0
10 hello H%0 1
11 hello h%o 0
12 Hello h%o 0
13 OHello %lhell% 1
14 OHello %hell% 0
15 hEllo %HEL% 0
16 abcdef %aBc%def% 0
17 ABCDDEF %abc%def% 0
18 Abc\nDef %abc%def% 0
19 abc\ntdef %abc%def% 0
20 abct\ndef %abc%dEf% 0
21 abc\n\ndeF %abc%def% 0
22 abc\n\ntdef %abc%deF% 0
23 Abc\nt\ndef %abc%def% 0
24 abct\n\ndef %abc%def% 0
25 ab\ndef %Abc%def% 1
26 aBc\nef %ABC%DEF% 1
27 ёЁё Ё%Ё 0
28 ощщЁё Щ%Ё 1
29 ощЩЁё %Щ%Ё 0
30 Щущпандер %щп%е% 0
31 Щущпандер %щП%е% 0
32 ощщЁё %щ% 0
33 ощЩЁё %ё% 0
34 Hello .* 1
35 Hello .*ell.* 1
36 Hello o$ 1
37 Hello hE.*lO 1
MATCH
1 Hello 1
2 Hello % 0
3 Hello %% 0
4 Hello %%% 0
5 Hello %_% 0
6 Hello _ 0
7 Hello _% 0
8 Hello %_ 0
9 Hello H%o 0
10 hello H%0 0
11 hello h%o 0
12 Hello h%o 0
13 OHello %lhell% 0
14 OHello %hell% 0
15 hEllo %HEL% 0
16 abcdef %aBc%def% 0
17 ABCDDEF %abc%def% 0
18 Abc\nDef %abc%def% 0
19 abc\ntdef %abc%def% 0
20 abct\ndef %abc%dEf% 0
21 abc\n\ndeF %abc%def% 0
22 abc\n\ntdef %abc%deF% 0
23 Abc\nt\ndef %abc%def% 0
24 abct\n\ndef %abc%def% 0
25 ab\ndef %Abc%def% 0
26 aBc\nef %ABC%DEF% 0
27 ёЁё Ё%Ё 0
28 ощщЁё Щ%Ё 0
29 ощЩЁё %Щ%Ё 0
30 Щущпандер %щп%е% 0
31 Щущпандер %щП%е% 0
32 ощщЁё %щ% 0
33 ощЩЁё %ё% 0
34 Hello .* 1
35 Hello .*ell.* 1
36 Hello o$ 1
37 Hello hE.*lO 0

36
tests/queries/0_stateless/02294_stringsearch_with_nonconst_needle.sql Normal file
View File

@ -0,0 +1,36 @@
drop table if exists non_const_needle;
create table non_const_needle
(id UInt32, haystack String, needle String)
engine = MergeTree()
order by id;
-- 1 - 33: LIKE-syntax, 34-37: re2-syntax
insert into non_const_needle values (1, 'Hello', '') (2, 'Hello', '%') (3, 'Hello', '%%') (4, 'Hello', '%%%') (5, 'Hello', '%_%') (6, 'Hello', '_') (7, 'Hello', '_%') (8, 'Hello', '%_') (9, 'Hello', 'H%o') (10, 'hello', 'H%0') (11, 'hello', 'h%o') (12, 'Hello', 'h%o') (13, 'OHello', '%lhell%') (14, 'OHello', '%hell%') (15, 'hEllo', '%HEL%') (16, 'abcdef', '%aBc%def%') (17, 'ABCDDEF', '%abc%def%') (18, 'Abc\nDef', '%abc%def%') (19, 'abc\ntdef', '%abc%def%') (20, 'abct\ndef', '%abc%dEf%') (21, 'abc\n\ndeF', '%abc%def%') (22, 'abc\n\ntdef', '%abc%deF%') (23, 'Abc\nt\ndef', '%abc%def%') (24, 'abct\n\ndef', '%abc%def%') (25, 'ab\ndef', '%Abc%def%') (26, 'aBc\nef', '%ABC%DEF%') (27, 'ёЁё', 'Ё%Ё') (28, 'ощщЁё', 'Щ%Ё') (29, 'ощЩЁё', '%Щ%Ё') (30, 'Щущпандер', '%щп%е%') (31, 'Щущпандер', '%щП%е%') (32, 'ощщЁё', '%щ%') (33, 'ощЩЁё', '%ё%') (34, 'Hello', '.*') (35, 'Hello', '.*ell.*') (36, 'Hello', 'o$') (37, 'Hello', 'hE.*lO');
select 'LIKE';
select id, haystack, needle, like(haystack, needle)
from non_const_needle
order by id;
select 'NOT LIKE';
select id, haystack, needle, not like(haystack, needle)
from non_const_needle
order by id;
select 'ILIKE';
select id, haystack, needle, ilike(haystack, needle)
from non_const_needle
order by id;
select 'NOT ILIKE';
select id, haystack, needle, not ilike(haystack, needle)
from non_const_needle
order by id;
select 'MATCH';
select id, haystack, needle, match(haystack, needle)
from non_const_needle
order by id;
drop table if exists non_const_needle;