thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-03 04:52:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve naming consistency of string search code

Browse Source 
Just renamings, nothing major ...
This commit is contained in:
Robert Schulze 2022-05-13 10:52:25 +02:00
parent 9b354268fa
commit 0299cc87e4
No known key found for this signature in database
GPG Key ID: 26703B55FB13728A
15 changed files with 178 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/Common/OptimizedRegularExpression.h
View File

@ -107,3 +107,4 @@ private:
}; };
using OptimizedRegularExpression = OptimizedRegularExpressionImpl<true>; using OptimizedRegularExpression = OptimizedRegularExpressionImpl<true>;
using OptimizedRegularExpressionSingleThreaded = OptimizedRegularExpressionImpl<false>;

32
src/Functions/CountSubstringsImpl.h
View File

@ -30,15 +30,15 @@ struct CountSubstringsImpl
/// Count occurrences of one substring in many strings. /// Count occurrences of one substring in many strings.
static void vectorConstant( static void vectorConstant(
const ColumnString::Chars & data, const ColumnString::Chars & haystack_data,
const ColumnString::Offsets & offsets, const ColumnString::Offsets & haystack_offsets,
const std::string & needle, const std::string & needle,
const ColumnPtr & start_pos, const ColumnPtr & start_pos,
PaddedPODArray<UInt64> & res) PaddedPODArray<UInt64> & res)
{ {
const UInt8 * begin = data.data(); const UInt8 * const begin = haystack_data.data();
const UInt8 * const end = haystack_data.data() + haystack_data.size();
const UInt8 * pos = begin; const UInt8 * pos = begin;
const UInt8 * end = pos + data.size();
/// FIXME: suboptimal /// FIXME: suboptimal
memset(&res[0], 0, res.size() * sizeof(res[0])); memset(&res[0], 0, res.size() * sizeof(res[0]));
@ -52,15 +52,15 @@ struct CountSubstringsImpl
while (pos < end && end != (pos = searcher.search(pos, end - pos))) while (pos < end && end != (pos = searcher.search(pos, end - pos)))
{ {
/// Determine which index it refers to. /// Determine which index it refers to.
while (begin + offsets[i] <= pos) while (begin + haystack_offsets[i] <= pos)
++i; ++i;
auto start = start_pos != nullptr ? start_pos->getUInt(i) : 0; auto start = start_pos != nullptr ? start_pos->getUInt(i) : 0;
/// We check that the entry does not pass through the boundaries of strings. /// We check that the entry does not pass through the boundaries of strings.
if (pos + needle.size() < begin + offsets[i]) if (pos + needle.size() < begin + haystack_offsets[i])
{ {
auto res_pos = needle.size() + Impl::countChars(reinterpret_cast<const char *>(begin + offsets[i - 1]), reinterpret_cast<const char *>(pos)); auto res_pos = needle.size() + Impl::countChars(reinterpret_cast<const char *>(begin + haystack_offsets[i - 1]), reinterpret_cast<const char *>(pos));
if (res_pos >= start) if (res_pos >= start)
{ {
++res[i]; ++res[i];
@ -69,14 +69,14 @@ struct CountSubstringsImpl
pos += needle.size(); pos += needle.size();
continue; continue;
} }
pos = begin + offsets[i]; pos = begin + haystack_offsets[i];
++i; ++i;
} }
} }
/// Count number of occurrences of substring in string. /// Count number of occurrences of substring in string.
static void constantConstantScalar( static void constantConstantScalar(
std::string data, std::string haystack,
std::string needle, std::string needle,
UInt64 start_pos, UInt64 start_pos,
UInt64 & res) UInt64 & res)
@ -87,9 +87,9 @@ struct CountSubstringsImpl
return; return;
auto start = std::max(start_pos, UInt64(1)); auto start = std::max(start_pos, UInt64(1));
size_t start_byte = Impl::advancePos(data.data(), data.data() + data.size(), start - 1) - data.data(); size_t start_byte = Impl::advancePos(haystack.data(), haystack.data() + haystack.size(), start - 1) - haystack.data();
size_t new_start_byte; size_t new_start_byte;
while ((new_start_byte = data.find(needle, start_byte)) != std::string::npos) while ((new_start_byte = haystack.find(needle, start_byte)) != std::string::npos)
{ {
++res; ++res;
/// Intersecting substrings in haystack accounted only once /// Intersecting substrings in haystack accounted only once
@ -99,21 +99,21 @@ struct CountSubstringsImpl
/// Count number of occurrences of substring in string starting from different positions. /// Count number of occurrences of substring in string starting from different positions.
static void constantConstant( static void constantConstant(
std::string data, std::string haystack,
std::string needle, std::string needle,
const ColumnPtr & start_pos, const ColumnPtr & start_pos,
PaddedPODArray<UInt64> & res) PaddedPODArray<UInt64> & res)
{ {
Impl::toLowerIfNeed(data); Impl::toLowerIfNeed(haystack);
Impl::toLowerIfNeed(needle); Impl::toLowerIfNeed(needle);
if (start_pos == nullptr) if (start_pos == nullptr)
{ {
constantConstantScalar(data, needle, 0, res[0]); constantConstantScalar(haystack, needle, 0, res[0]);
return; return;
} }
size_t haystack_size = Impl::countChars(data.data(), data.data() + data.size()); size_t haystack_size = Impl::countChars(haystack.data(), haystack.data() + haystack.size());
size_t size = start_pos != nullptr ? start_pos->size() : 0; size_t size = start_pos != nullptr ? start_pos->size() : 0;
for (size_t i = 0; i < size; ++i) for (size_t i = 0; i < size; ++i)
@ -125,7 +125,7 @@ struct CountSubstringsImpl
res[i] = 0; res[i] = 0;
continue; continue;
} }
constantConstantScalar(data, needle, start, res[i]); constantConstantScalar(haystack, needle, start, res[i]);
} }
} }

23
src/Functions/FunctionsStringSearch.h
View File

@ -15,7 +15,6 @@
namespace DB namespace DB
{ {
/** Search and replace functions in strings: /** Search and replace functions in strings:
*
* position(haystack, needle) - the normal search for a substring in a string, returns the position (in bytes) of the found substring starting with 1, or 0 if no substring is found. * position(haystack, needle) - the normal search for a substring in a string, returns the position (in bytes) of the found substring starting with 1, or 0 if no substring is found.
* positionUTF8(haystack, needle) - the same, but the position is calculated at code points, provided that the string is encoded in UTF-8. * positionUTF8(haystack, needle) - the same, but the position is calculated at code points, provided that the string is encoded in UTF-8.
* positionCaseInsensitive(haystack, needle) * positionCaseInsensitive(haystack, needle)
@ -24,13 +23,29 @@ namespace DB
* like(haystack, pattern) - search by the regular expression LIKE; Returns 0 or 1. Case-insensitive, but only for Latin. * like(haystack, pattern) - search by the regular expression LIKE; Returns 0 or 1. Case-insensitive, but only for Latin.
* notLike(haystack, pattern) * notLike(haystack, pattern)
* *
* ilike(haystack, pattern) - like 'like' but case-insensitive
* notIlike(haystack, pattern)
*
* match(haystack, pattern) - search by regular expression re2; Returns 0 or 1. * match(haystack, pattern) - search by regular expression re2; Returns 0 or 1.
* multiMatchAny(haystack, [pattern_1, pattern_2, ..., pattern_n]) -- search by re2 regular expressions pattern_i; Returns 0 or 1 if any pattern_i matches.
* multiMatchAnyIndex(haystack, [pattern_1, pattern_2, ..., pattern_n]) -- search by re2 regular expressions pattern_i; Returns index of any match or zero if none;
* multiMatchAllIndices(haystack, [pattern_1, pattern_2, ..., pattern_n]) -- search by re2 regular expressions pattern_i; Returns an array of matched indices in any order;
* *
* countSubstrings(haystack, needle) -- count number of occurrences of needle in haystack. * countSubstrings(haystack, needle) -- count number of occurrences of needle in haystack.
* countSubstringsCaseInsensitive(haystack, needle) * countSubstringsCaseInsensitive(haystack, needle)
* countSubstringsCaseInsensitiveUTF8(haystack, needle)
*
* hasToken()
* hasTokenCaseInsensitive()
*
* JSON stuff:
* visitParamExtractBool()
* simpleJSONExtractBool()
* visitParamExtractFloat()
* simpleJSONExtractFloat()
* visitParamExtractInt()
* simpleJSONExtractInt()
* visitParamExtractUInt()
* simpleJSONExtractUInt()
* visitParamHas()
* simpleJSONHas()
* *
* Applies regexp re2 and pulls: * Applies regexp re2 and pulls:
* - the first subpattern, if the regexp has a subpattern; * - the first subpattern, if the regexp has a subpattern;

34
src/Functions/FunctionsVisitParam.h
View File

@ -85,8 +85,8 @@ struct ExtractParamImpl
/// It is assumed that `res` is the correct size and initialized with zeros. /// It is assumed that `res` is the correct size and initialized with zeros.
static void vectorConstant( static void vectorConstant(
const ColumnString::Chars & data, const ColumnString::Chars & haystack_data,
const ColumnString::Offsets & offsets, const ColumnString::Offsets & haystack_offsets,
std::string needle, std::string needle,
const ColumnPtr & start_pos, const ColumnPtr & start_pos,
PaddedPODArray<ResultType> & res) PaddedPODArray<ResultType> & res)
@ -97,9 +97,9 @@ struct ExtractParamImpl
/// We are looking for a parameter simply as a substring of the form "name" /// We are looking for a parameter simply as a substring of the form "name"
needle = "\"" + needle + "\":"; needle = "\"" + needle + "\":";
const UInt8 * begin = data.data(); const UInt8 * const begin = haystack_data.data();
const UInt8 * const end = haystack_data.data() + haystack_data.size();
const UInt8 * pos = begin; const UInt8 * pos = begin;
const UInt8 * end = pos + data.size();
/// The current index in the string array. /// The current index in the string array.
size_t i = 0; size_t i = 0;
@ -110,19 +110,19 @@ struct ExtractParamImpl
while (pos < end && end != (pos = searcher.search(pos, end - pos))) while (pos < end && end != (pos = searcher.search(pos, end - pos)))
{ {
/// Let's determine which index it belongs to. /// Let's determine which index it belongs to.
while (begin + offsets[i] <= pos) while (begin + haystack_offsets[i] <= pos)
{ {
res[i] = 0; res[i] = 0;
++i; ++i;
} }
/// We check that the entry does not pass through the boundaries of strings. /// We check that the entry does not pass through the boundaries of strings.
if (pos + needle.size() < begin + offsets[i]) if (pos + needle.size() < begin + haystack_offsets[i])
res[i] = ParamExtractor::extract(pos + needle.size(), begin + offsets[i] - 1); /// don't include terminating zero res[i] = ParamExtractor::extract(pos + needle.size(), begin + haystack_offsets[i] - 1); /// don't include terminating zero
else else
res[i] = 0; res[i] = 0;
pos = begin + offsets[i]; pos = begin + haystack_offsets[i];
++i; ++i;
} }
@ -153,20 +153,20 @@ struct ExtractParamImpl
template <typename ParamExtractor> template <typename ParamExtractor>
struct ExtractParamToStringImpl struct ExtractParamToStringImpl
{ {
static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, static void vector(const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets,
std::string needle, std::string needle,
ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets) ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets)
{ {
/// Constant 5 is taken from a function that performs a similar task FunctionsStringSearch.h::ExtractImpl /// Constant 5 is taken from a function that performs a similar task FunctionsStringSearch.h::ExtractImpl
res_data.reserve(data.size() / 5); res_data.reserve(haystack_data.size() / 5);
res_offsets.resize(offsets.size()); res_offsets.resize(haystack_offsets.size());
/// We are looking for a parameter simply as a substring of the form "name" /// We are looking for a parameter simply as a substring of the form "name"
needle = "\"" + needle + "\":"; needle = "\"" + needle + "\":";
const UInt8 * begin = data.data(); const UInt8 * const begin = haystack_data.data();
const UInt8 * const end = haystack_data.data() + haystack_data.size();
const UInt8 * pos = begin; const UInt8 * pos = begin;
const UInt8 * end = pos + data.size();
/// The current index in the string array. /// The current index in the string array.
size_t i = 0; size_t i = 0;
@ -177,7 +177,7 @@ struct ExtractParamToStringImpl
while (pos < end && end != (pos = searcher.search(pos, end - pos))) while (pos < end && end != (pos = searcher.search(pos, end - pos)))
{ {
/// Determine which index it belongs to. /// Determine which index it belongs to.
while (begin + offsets[i] <= pos) while (begin + haystack_offsets[i] <= pos)
{ {
res_data.push_back(0); res_data.push_back(0);
res_offsets[i] = res_data.size(); res_offsets[i] = res_data.size();
@ -185,10 +185,10 @@ struct ExtractParamToStringImpl
} }
/// We check that the entry does not pass through the boundaries of strings. /// We check that the entry does not pass through the boundaries of strings.
if (pos + needle.size() < begin + offsets[i]) if (pos + needle.size() < begin + haystack_offsets[i])
ParamExtractor::extract(pos + needle.size(), begin + offsets[i], res_data); ParamExtractor::extract(pos + needle.size(), begin + haystack_offsets[i], res_data);
pos = begin + offsets[i]; pos = begin + haystack_offsets[i];
res_data.push_back(0); res_data.push_back(0);
res_offsets[i] = res_data.size(); res_offsets[i] = res_data.size();

26
src/Functions/HasTokenImpl.h
View File

@ -14,7 +14,7 @@ namespace ErrorCodes
/** Token search the string, means that needle must be surrounded by some separator chars, like whitespace or puctuation. /** Token search the string, means that needle must be surrounded by some separator chars, like whitespace or puctuation.
*/ */
template <typename Name, typename TokenSearcher, bool negate_result = false> template <typename Name, typename TokenSearcher, bool negate>
struct HasTokenImpl struct HasTokenImpl
{ {
using ResultType = UInt8; using ResultType = UInt8;
@ -24,8 +24,8 @@ struct HasTokenImpl
static constexpr auto name = Name::name; static constexpr auto name = Name::name;
static void vectorConstant( static void vectorConstant(
const ColumnString::Chars & data, const ColumnString::Chars & haystack_data,
const ColumnString::Offsets & offsets, const ColumnString::Offsets & haystack_offsets,
const std::string & pattern, const std::string & pattern,
const ColumnPtr & start_pos, const ColumnPtr & start_pos,
PaddedPODArray<UInt8> & res) PaddedPODArray<UInt8> & res)
@ -33,12 +33,12 @@ struct HasTokenImpl
if (start_pos != nullptr) if (start_pos != nullptr)
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' does not support start_pos argument", name); throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' does not support start_pos argument", name);
if (offsets.empty()) if (haystack_offsets.empty())
return; return;
const UInt8 * begin = data.data(); const UInt8 * const begin = haystack_data.data();
const UInt8 * const end = haystack_data.data() + haystack_data.size();
const UInt8 * pos = begin; const UInt8 * pos = begin;
const UInt8 * end = pos + data.size();
/// The current index in the array of strings. /// The current index in the array of strings.
size_t i = 0; size_t i = 0;
@ -49,25 +49,25 @@ struct HasTokenImpl
while (pos < end && end != (pos = searcher.search(pos, end - pos))) while (pos < end && end != (pos = searcher.search(pos, end - pos)))
{ {
/// Let's determine which index it refers to. /// Let's determine which index it refers to.
while (begin + offsets[i] <= pos) while (begin + haystack_offsets[i] <= pos)
{ {
res[i] = negate_result; res[i] = negate;
++i; ++i;
} }
/// We check that the entry does not pass through the boundaries of strings. /// We check that the entry does not pass through the boundaries of strings.
if (pos + pattern.size() < begin + offsets[i]) if (pos + pattern.size() < begin + haystack_offsets[i])
res[i] = !negate_result; res[i] = !negate;
else else
res[i] = negate_result; res[i] = negate;
pos = begin + offsets[i]; pos = begin + haystack_offsets[i];
++i; ++i;
} }
/// Tail, in which there can be no substring. /// Tail, in which there can be no substring.
if (i < res.size()) if (i < res.size())
memset(&res[i], negate_result, (res.size() - i) * sizeof(res[0])); memset(&res[i], negate, (res.size() - i) * sizeof(res[0]));
} }
template <typename... Args> template <typename... Args>

185
src/Functions/MatchImpl.h
View File

@ -20,21 +20,19 @@ namespace ErrorCodes
extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_TYPE_OF_ARGUMENT;
} }
namespace impl {
/// Is the [I]LIKE expression reduced to finding a substring in a string? /// Is the [I]LIKE expression reduced to finding a substring in a string?
static inline bool likePatternIsStrstr(const String & pattern, String & res) inline bool likePatternIsSubstring(const String & pattern, String & res)
{ {
if (pattern.size() < 2 || pattern.front() != '%' || pattern.back() != '%') if (pattern.size() < 2 || pattern.front() != '%' || pattern.back() != '%')
return false; return false;
res = ""; res.clear();
res.reserve(pattern.size() - 2); res.reserve(pattern.size() - 2);
const char * pos = pattern.data(); const char * pos = pattern.data() + 1;
const char * end = pos + pattern.size(); const char * const end = pattern.data() + pattern.size() - 1;
++pos;
--end;
while (pos < end) while (pos < end)
{ {
@ -60,11 +58,16 @@ static inline bool likePatternIsStrstr(const String & pattern, String & res)
return true; return true;
} }
/** 'like' - if true, treat pattern as SQL LIKE or ILIKE; if false - treat pattern as re2 regexp. }
/** 'like' - if true, treat pattern as SQL LIKE, otherwise as re2 regexp.
* 'negate' - if true, negate result
* 'case_insensitive' - if true, match case insensitively
*
* NOTE: We want to run regexp search for whole columns by one call (as implemented in function 'position') * NOTE: We want to run regexp search for whole columns by one call (as implemented in function 'position')
* but for that, regexp engine must support \0 bytes and their interpretation as string boundaries. * but for that, regexp engine must support \0 bytes and their interpretation as string boundaries.
*/ */
template <typename Name, bool like, bool revert = false, bool case_insensitive = false> template <typename Name, bool like, bool negate, bool case_insensitive>
struct MatchImpl struct MatchImpl
{ {
static constexpr bool use_default_implementation_for_constants = true; static constexpr bool use_default_implementation_for_constants = true;
@ -78,9 +81,9 @@ struct MatchImpl
VolnitskyUTF8>; VolnitskyUTF8>;
static void vectorConstant( static void vectorConstant(
const ColumnString::Chars & data, const ColumnString::Chars & haystack_data,
const ColumnString::Offsets & offsets, const ColumnString::Offsets & haystack_offsets,
const String & pattern, const String & needle,
const ColumnPtr & start_pos, const ColumnPtr & start_pos,
PaddedPODArray<UInt8> & res) PaddedPODArray<UInt8> & res)
{ {
@ -88,15 +91,15 @@ struct MatchImpl
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Function '{}' doesn't support start_pos argument", name); "Function '{}' doesn't support start_pos argument", name);
if (offsets.empty()) if (haystack_offsets.empty())
return; return;
/// A simple case where the [I]LIKE expression reduces to finding a substring in a string /// A simple case where the [I]LIKE expression reduces to finding a substring in a string
String strstr_pattern; String strstr_pattern;
if (like && likePatternIsStrstr(pattern, strstr_pattern)) if (like && impl::likePatternIsSubstring(needle, strstr_pattern))
{ {
const UInt8 * const begin = data.data(); const UInt8 * const begin = haystack_data.data();
const UInt8 * const end = data.data() + data.size(); const UInt8 * const end = haystack_data.data() + haystack_data.size();
const UInt8 * pos = begin; const UInt8 * pos = begin;
/// The current index in the array of strings. /// The current index in the array of strings.
@ -109,31 +112,29 @@ struct MatchImpl
while (pos < end && end != (pos = searcher.search(pos, end - pos))) while (pos < end && end != (pos = searcher.search(pos, end - pos)))
{ {
/// Let's determine which index it refers to. /// Let's determine which index it refers to.
while (begin + offsets[i] <= pos) while (begin + haystack_offsets[i] <= pos)
{ {
res[i] = revert; res[i] = negate;
++i; ++i;
} }
/// We check that the entry does not pass through the boundaries of strings. /// We check that the entry does not pass through the boundaries of strings.
if (pos + strstr_pattern.size() < begin + offsets[i]) if (pos + strstr_pattern.size() < begin + haystack_offsets[i])
res[i] = !revert; res[i] = !negate;
else else
res[i] = revert; res[i] = negate;
pos = begin + offsets[i]; pos = begin + haystack_offsets[i];
++i; ++i;
} }
/// Tail, in which there can be no substring. /// Tail, in which there can be no substring.
if (i < res.size()) if (i < res.size())
memset(&res[i], revert, (res.size() - i) * sizeof(res[0])); memset(&res[i], negate, (res.size() - i) * sizeof(res[0]));
} }
else else
{ {
size_t size = offsets.size(); auto regexp = Regexps::get<like, true, case_insensitive>(needle);
auto regexp = Regexps::get<like, true, case_insensitive>(pattern);
String required_substring; String required_substring;
bool is_trivial; bool is_trivial;
@ -141,37 +142,39 @@ struct MatchImpl
regexp->getAnalyzeResult(required_substring, is_trivial, required_substring_is_prefix); regexp->getAnalyzeResult(required_substring, is_trivial, required_substring_is_prefix);
size_t haystack_size = haystack_offsets.size();
if (required_substring.empty()) if (required_substring.empty())
{ {
if (!regexp->getRE2()) /// An empty regexp. Always matches. if (!regexp->getRE2()) /// An empty regexp. Always matches.
{ {
if (size) if (haystack_size)
memset(res.data(), 1, size * sizeof(res[0])); memset(res.data(), 1, haystack_size * sizeof(res[0]));
} }
else else
{ {
size_t prev_offset = 0; size_t prev_offset = 0;
for (size_t i = 0; i < size; ++i) for (size_t i = 0; i < haystack_size; ++i)
{ {
res[i] = revert res[i] = negate
^ regexp->getRE2()->Match( ^ regexp->getRE2()->Match(
re2_st::StringPiece(reinterpret_cast<const char *>(&data[prev_offset]), offsets[i] - prev_offset - 1), {reinterpret_cast<const char *>(&haystack_data[prev_offset]), haystack_offsets[i] - prev_offset - 1},
0, 0,
offsets[i] - prev_offset - 1, haystack_offsets[i] - prev_offset - 1,
re2_st::RE2::UNANCHORED, re2_st::RE2::UNANCHORED,
nullptr, nullptr,
0); 0);
prev_offset = offsets[i]; prev_offset = haystack_offsets[i];
} }
} }
} }
else else
{ {
/// NOTE This almost matches with the case of LikePatternIsStrstr. /// NOTE This almost matches with the case of impl::likePatternIsSubstring.
const UInt8 * const begin = data.data(); const UInt8 * const begin = haystack_data.data();
const UInt8 * const end = data.begin() + data.size(); const UInt8 * const end = haystack_data.begin() + haystack_data.size();
const UInt8 * pos = begin; const UInt8 * pos = begin;
/// The current index in the array of strings. /// The current index in the array of strings.
@ -183,23 +186,23 @@ struct MatchImpl
while (pos < end && end != (pos = searcher.search(pos, end - pos))) while (pos < end && end != (pos = searcher.search(pos, end - pos)))
{ {
/// Determine which index it refers to. /// Determine which index it refers to.
while (begin + offsets[i] <= pos) while (begin + haystack_offsets[i] <= pos)
{ {
res[i] = revert; res[i] = negate;
++i; ++i;
} }
/// We check that the entry does not pass through the boundaries of strings. /// We check that the entry does not pass through the boundaries of strings.
if (pos + required_substring.size() < begin + offsets[i]) if (pos + required_substring.size() < begin + haystack_offsets[i])
{ {
/// And if it does not, if necessary, we check the regexp. /// And if it does not, if necessary, we check the regexp.
if (is_trivial) if (is_trivial)
res[i] = !revert; res[i] = !negate;
else else
{ {
const char * str_data = reinterpret_cast<const char *>(&data[offsets[i - 1]]); const char * str_data = reinterpret_cast<const char *>(&haystack_data[haystack_offsets[i - 1]]);
size_t str_size = offsets[i] - offsets[i - 1] - 1; size_t str_size = haystack_offsets[i] - haystack_offsets[i - 1] - 1;
/** Even in the case of `required_substring_is_prefix` use UNANCHORED check for regexp, /** Even in the case of `required_substring_is_prefix` use UNANCHORED check for regexp,
* so that it can match when `required_substring` occurs into the string several times, * so that it can match when `required_substring` occurs into the string several times,
@ -207,55 +210,57 @@ struct MatchImpl
*/ */
if (required_substring_is_prefix) if (required_substring_is_prefix)
res[i] = revert res[i] = negate
^ regexp->getRE2()->Match( ^ regexp->getRE2()->Match(
re2_st::StringPiece(str_data, str_size), {str_data, str_size},
reinterpret_cast<const char *>(pos) - str_data, reinterpret_cast<const char *>(pos) - str_data,
str_size, str_size,
re2_st::RE2::UNANCHORED, re2_st::RE2::UNANCHORED,
nullptr, nullptr,
0); 0);
else else
res[i] = revert res[i] = negate
^ regexp->getRE2()->Match( ^ regexp->getRE2()->Match(
re2_st::StringPiece(str_data, str_size), 0, str_size, re2_st::RE2::UNANCHORED, nullptr, 0); {str_data, str_size}, 0, str_size, re2_st::RE2::UNANCHORED, nullptr, 0);
} }
} }
else else
res[i] = revert; res[i] = negate;
pos = begin + offsets[i]; pos = begin + haystack_offsets[i];
++i; ++i;
} }
/// Tail, in which there can be no substring. /// Tail, in which there can be no substring.
if (i < res.size()) if (i < res.size())
memset(&res[i], revert, (res.size() - i) * sizeof(res[0])); memset(&res[i], negate, (res.size() - i) * sizeof(res[0]));
} }
} }
} }
/// Very carefully crafted copy-paste. /// Very carefully crafted copy-paste.
static void vectorFixedConstant( static void vectorFixedConstant(
const ColumnString::Chars & data, size_t n, const String & pattern, const ColumnString::Chars & haystack,
size_t N,
const String & needle,
PaddedPODArray<UInt8> & res) PaddedPODArray<UInt8> & res)
{ {
if (data.empty()) if (haystack.empty())
return; return;
/// A simple case where the LIKE expression reduces to finding a substring in a string /// A simple case where the LIKE expression reduces to finding a substring in a string
String strstr_pattern; String strstr_pattern;
if (like && likePatternIsStrstr(pattern, strstr_pattern)) if (like && impl::likePatternIsSubstring(needle, strstr_pattern))
{ {
const UInt8 * begin = data.data(); const UInt8 * const begin = haystack.data();
const UInt8 * const end = haystack.data() + haystack.size();
const UInt8 * pos = begin; const UInt8 * pos = begin;
const UInt8 * end = pos + data.size();
size_t i = 0; size_t i = 0;
const UInt8 * next_pos = begin; const UInt8 * next_pos = begin;
/// If pattern is larger than string size - it cannot be found. /// If needle is larger than string size - it cannot be found.
if (strstr_pattern.size() <= n) if (strstr_pattern.size() <= N)
{ {
Searcher searcher(strstr_pattern.data(), strstr_pattern.size(), end - pos); Searcher searcher(strstr_pattern.data(), strstr_pattern.size(), end - pos);
@ -263,19 +268,19 @@ struct MatchImpl
while (pos < end && end != (pos = searcher.search(pos, end - pos))) while (pos < end && end != (pos = searcher.search(pos, end - pos)))
{ {
/// Let's determine which index it refers to. /// Let's determine which index it refers to.
while (next_pos + n <= pos) while (next_pos + N <= pos)
{ {
res[i] = revert; res[i] = negate;
next_pos += n; next_pos += N;
++i; ++i;
} }
next_pos += n; next_pos += N;
/// We check that the entry does not pass through the boundaries of strings. /// We check that the entry does not pass through the boundaries of strings.
if (pos + strstr_pattern.size() <= next_pos) if (pos + strstr_pattern.size() <= next_pos)
res[i] = !revert; res[i] = !negate;
else else
res[i] = revert; res[i] = negate;
pos = next_pos; pos = next_pos;
++i; ++i;
@ -284,13 +289,11 @@ struct MatchImpl
/// Tail, in which there can be no substring. /// Tail, in which there can be no substring.
if (i < res.size()) if (i < res.size())
memset(&res[i], revert, (res.size() - i) * sizeof(res[0])); memset(&res[i], negate, (res.size() - i) * sizeof(res[0]));
} }
else else
{ {
size_t size = data.size() / n; auto regexp = Regexps::get<like, true, case_insensitive>(needle);
auto regexp = Regexps::get<like, true, case_insensitive>(pattern);
String required_substring; String required_substring;
bool is_trivial; bool is_trivial;
@ -298,44 +301,46 @@ struct MatchImpl
regexp->getAnalyzeResult(required_substring, is_trivial, required_substring_is_prefix); regexp->getAnalyzeResult(required_substring, is_trivial, required_substring_is_prefix);
const size_t haystack_size = haystack.size() / N;
if (required_substring.empty()) if (required_substring.empty())
{ {
if (!regexp->getRE2()) /// An empty regexp. Always matches. if (!regexp->getRE2()) /// An empty regexp. Always matches.
{ {
if (size) if (haystack_size)
memset(res.data(), 1, size * sizeof(res[0])); memset(res.data(), 1, haystack_size * sizeof(res[0]));
} }
else else
{ {
size_t offset = 0; size_t offset = 0;
for (size_t i = 0; i < size; ++i) for (size_t i = 0; i < haystack_size; ++i)
{ {
res[i] = revert res[i] = negate
^ regexp->getRE2()->Match( ^ regexp->getRE2()->Match(
re2_st::StringPiece(reinterpret_cast<const char *>(&data[offset]), n), {reinterpret_cast<const char *>(&haystack[offset]), N},
0, 0,
n, N,
re2_st::RE2::UNANCHORED, re2_st::RE2::UNANCHORED,
nullptr, nullptr,
0); 0);
offset += n; offset += N;
} }
} }
} }
else else
{ {
/// NOTE This almost matches with the case of LikePatternIsStrstr. /// NOTE This almost matches with the case of likePatternIsSubstring.
const UInt8 * begin = data.data(); const UInt8 * const begin = haystack.data();
const UInt8 * const end = haystack.data() + haystack.size();
const UInt8 * pos = begin; const UInt8 * pos = begin;
const UInt8 * end = pos + data.size();
size_t i = 0; size_t i = 0;
const UInt8 * next_pos = begin; const UInt8 * next_pos = begin;
/// If required substring is larger than string size - it cannot be found. /// If required substring is larger than string size - it cannot be found.
if (required_substring.size() <= n) if (required_substring.size() <= N)
{ {
Searcher searcher(required_substring.data(), required_substring.size(), end - pos); Searcher searcher(required_substring.data(), required_substring.size(), end - pos);
@ -343,23 +348,23 @@ struct MatchImpl
while (pos < end && end != (pos = searcher.search(pos, end - pos))) while (pos < end && end != (pos = searcher.search(pos, end - pos)))
{ {
/// Let's determine which index it refers to. /// Let's determine which index it refers to.
while (next_pos + n <= pos) while (next_pos + N <= pos)
{ {
res[i] = revert; res[i] = negate;
next_pos += n; next_pos += N;
++i; ++i;
} }
next_pos += n; next_pos += N;
if (pos + required_substring.size() <= next_pos) if (pos + required_substring.size() <= next_pos)
{ {
/// And if it does not, if necessary, we check the regexp. /// And if it does not, if necessary, we check the regexp.
if (is_trivial) if (is_trivial)
res[i] = !revert; res[i] = !negate;
else else
{ {
const char * str_data = reinterpret_cast<const char *>(next_pos - n); const char * str_data = reinterpret_cast<const char *>(next_pos - N);
/** Even in the case of `required_substring_is_prefix` use UNANCHORED check for regexp, /** Even in the case of `required_substring_is_prefix` use UNANCHORED check for regexp,
* so that it can match when `required_substring` occurs into the string several times, * so that it can match when `required_substring` occurs into the string several times,
@ -367,22 +372,22 @@ struct MatchImpl
*/ */
if (required_substring_is_prefix) if (required_substring_is_prefix)
res[i] = revert res[i] = negate
^ regexp->getRE2()->Match( ^ regexp->getRE2()->Match(
re2_st::StringPiece(str_data, n), {str_data, N},
reinterpret_cast<const char *>(pos) - str_data, reinterpret_cast<const char *>(pos) - str_data,
n, N,
re2_st::RE2::UNANCHORED, re2_st::RE2::UNANCHORED,
nullptr, nullptr,
0); 0);
else else
res[i] = revert res[i] = negate
^ regexp->getRE2()->Match( ^ regexp->getRE2()->Match(
re2_st::StringPiece(str_data, n), 0, n, re2_st::RE2::UNANCHORED, nullptr, 0); {str_data, N}, 0, N, re2_st::RE2::UNANCHORED, nullptr, 0);
} }
} }
else else
res[i] = revert; res[i] = negate;
pos = next_pos; pos = next_pos;
++i; ++i;
@ -391,7 +396,7 @@ struct MatchImpl
/// Tail, in which there can be no substring. /// Tail, in which there can be no substring.
if (i < res.size()) if (i < res.size())
memset(&res[i], revert, (res.size() - i) * sizeof(res[0])); memset(&res[i], negate, (res.size() - i) * sizeof(res[0]));
} }
} }
} }

2
src/Functions/MultiMatchAllIndicesImpl.h
View File

@ -11,8 +11,6 @@
#if USE_HYPERSCAN #if USE_HYPERSCAN
# include <hs.h> # include <hs.h>
#else
# include "MatchImpl.h"
#endif #endif

2
src/Functions/MultiMatchAnyImpl.h
View File

@ -120,7 +120,7 @@ struct MultiMatchAnyImpl
memset(accum.data(), 0, accum.size()); memset(accum.data(), 0, accum.size());
for (size_t j = 0; j < needles.size(); ++j) for (size_t j = 0; j < needles.size(); ++j)
{ {
MatchImpl<Name, false, false>::vectorConstant(haystack_data, haystack_offsets, needles[j].toString(), nullptr, accum); MatchImpl<Name, false, false, false>::vectorConstant(haystack_data, haystack_offsets, needles[j].toString(), nullptr, accum);
for (size_t i = 0; i < res.size(); ++i) for (size_t i = 0; i < res.size(); ++i)
{ {
if constexpr (FindAny) if constexpr (FindAny)

18
src/Functions/PositionImpl.h
View File

@ -186,15 +186,15 @@ struct PositionImpl
/// Find one substring in many strings. /// Find one substring in many strings.
static void vectorConstant( static void vectorConstant(
const ColumnString::Chars & data, const ColumnString::Chars & haystack_data,
const ColumnString::Offsets & offsets, const ColumnString::Offsets & haystack_offsets,
const std::string & needle, const std::string & needle,
const ColumnPtr & start_pos, const ColumnPtr & start_pos,
PaddedPODArray<UInt64> & res) PaddedPODArray<UInt64> & res)
{ {
const UInt8 * begin = data.data(); const UInt8 * const begin = haystack_data.data();
const UInt8 * const end = haystack_data.data() + haystack_data.size();
const UInt8 * pos = begin; const UInt8 * pos = begin;
const UInt8 * end = pos + data.size();
/// Current index in the array of strings. /// Current index in the array of strings.
size_t i = 0; size_t i = 0;
@ -205,7 +205,7 @@ struct PositionImpl
while (pos < end && end != (pos = searcher.search(pos, end - pos))) while (pos < end && end != (pos = searcher.search(pos, end - pos)))
{ {
/// Determine which index it refers to. /// Determine which index it refers to.
while (begin + offsets[i] <= pos) while (begin + haystack_offsets[i] <= pos)
{ {
res[i] = 0; res[i] = 0;
++i; ++i;
@ -213,14 +213,14 @@ struct PositionImpl
auto start = start_pos != nullptr ? start_pos->getUInt(i) : 0; auto start = start_pos != nullptr ? start_pos->getUInt(i) : 0;
/// We check that the entry does not pass through the boundaries of strings. /// We check that the entry does not pass through the boundaries of strings.
if (pos + needle.size() < begin + offsets[i]) if (pos + needle.size() < begin + haystack_offsets[i])
{ {
auto res_pos = 1 + Impl::countChars(reinterpret_cast<const char *>(begin + offsets[i - 1]), reinterpret_cast<const char *>(pos)); auto res_pos = 1 + Impl::countChars(reinterpret_cast<const char *>(begin + haystack_offsets[i - 1]), reinterpret_cast<const char *>(pos));
if (res_pos < start) if (res_pos < start)
{ {
pos = reinterpret_cast<const UInt8 *>(Impl::advancePos( pos = reinterpret_cast<const UInt8 *>(Impl::advancePos(
reinterpret_cast<const char *>(pos), reinterpret_cast<const char *>(pos),
reinterpret_cast<const char *>(begin + offsets[i]), reinterpret_cast<const char *>(begin + haystack_offsets[i]),
start - res_pos)); start - res_pos));
continue; continue;
} }
@ -230,7 +230,7 @@ struct PositionImpl
{ {
res[i] = 0; res[i] = 0;
} }
pos = begin + offsets[i]; pos = begin + haystack_offsets[i];
++i; ++i;
} }

4
src/Functions/Regexps.h
View File

@ -38,7 +38,7 @@ namespace ErrorCodes
namespace Regexps namespace Regexps
{ {
using Regexp = OptimizedRegularExpressionImpl<false>; using Regexp = OptimizedRegularExpressionSingleThreaded;
using Pool = ObjectPoolMap<Regexp, String>; using Pool = ObjectPoolMap<Regexp, String>;
template <bool like> template <bool like>
@ -68,7 +68,7 @@ namespace Regexps
flags |= OptimizedRegularExpression::RE_NO_CAPTURE; flags |= OptimizedRegularExpression::RE_NO_CAPTURE;
if (case_insensitive) if (case_insensitive)
flags |= Regexps::Regexp::RE_CASELESS; flags |= OptimizedRegularExpression::RE_CASELESS;
ProfileEvents::increment(ProfileEvents::RegexpCreated); ProfileEvents::increment(ProfileEvents::RegexpCreated);
return new Regexp{createRegexp<like>(pattern, flags)}; return new Regexp{createRegexp<like>(pattern, flags)};

2
src/Functions/ilike.cpp
View File

@ -12,7 +12,7 @@ struct NameILike
static constexpr auto name = "ilike"; static constexpr auto name = "ilike";
}; };
using ILikeImpl = MatchImpl<NameILike, true, false, /*case-insensitive*/true>; using ILikeImpl = MatchImpl<NameILike, true, false, true>;
using FunctionILike = FunctionsStringSearch<ILikeImpl>; using FunctionILike = FunctionsStringSearch<ILikeImpl>;
} }

2
src/Functions/like.h
View File

@ -11,7 +11,7 @@ struct NameLike
static constexpr auto name = "like"; static constexpr auto name = "like";
}; };
using LikeImpl = MatchImpl<NameLike, /*SQL LIKE */ true, /*revert*/false>; using LikeImpl = MatchImpl<NameLike, true, false, false>;
using FunctionLike = FunctionsStringSearch<LikeImpl>; using FunctionLike = FunctionsStringSearch<LikeImpl>;
} }

2
src/Functions/match.cpp
View File

@ -13,7 +13,7 @@ struct NameMatch
static constexpr auto name = "match"; static constexpr auto name = "match";
}; };
using FunctionMatch = FunctionsStringSearch<MatchImpl<NameMatch, false>>; using FunctionMatch = FunctionsStringSearch<MatchImpl<NameMatch, false, false, false>>;
} }

2
src/Functions/notILike.cpp
View File

@ -12,7 +12,7 @@ struct NameNotILike
static constexpr auto name = "notILike"; static constexpr auto name = "notILike";
}; };
using NotILikeImpl = MatchImpl<NameNotILike, true, true, /*case-insensitive*/true>; using NotILikeImpl = MatchImpl<NameNotILike, true, true, true>;
using FunctionNotILike = FunctionsStringSearch<NotILikeImpl>; using FunctionNotILike = FunctionsStringSearch<NotILikeImpl>;
} }

2
src/Functions/notLike.cpp
View File

@ -12,7 +12,7 @@ struct NameNotLike
static constexpr auto name = "notLike"; static constexpr auto name = "notLike";
}; };
using FunctionNotLike = FunctionsStringSearch<MatchImpl<NameNotLike, true, true>>; using FunctionNotLike = FunctionsStringSearch<MatchImpl<NameNotLike, true, true, false>>;
} }