2015-10-14 12:12:56 +00:00
|
|
|
#pragma once
|
|
|
|
|
2021-10-02 07:13:14 +00:00
|
|
|
#include <base/getPageSize.h>
|
2019-08-21 08:12:39 +00:00
|
|
|
#include <Common/Exception.h>
|
|
|
|
#include <Common/StringUtils/StringUtils.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Common/UTF8Helpers.h>
|
2019-05-14 09:58:33 +00:00
|
|
|
#include <Core/Defines.h>
|
2015-10-14 12:12:56 +00:00
|
|
|
#include <Poco/Unicode.h>
|
2022-06-28 11:29:07 +00:00
|
|
|
#include <cstdint>
|
|
|
|
#include <cstring>
|
2015-10-14 12:12:56 +00:00
|
|
|
|
2019-01-04 12:10:00 +00:00
|
|
|
#ifdef __SSE2__
|
2017-02-07 21:26:32 +00:00
|
|
|
#include <emmintrin.h>
|
|
|
|
#endif
|
|
|
|
|
2019-01-04 12:10:00 +00:00
|
|
|
#ifdef __SSE4_1__
|
2016-01-16 00:45:19 +00:00
|
|
|
#include <smmintrin.h>
|
|
|
|
#endif
|
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2016-01-12 02:21:15 +00:00
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
2019-08-21 08:12:39 +00:00
|
|
|
extern const int BAD_ARGUMENTS;
|
2016-01-12 02:21:15 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-05-10 04:00:19 +00:00
|
|
|
/** Variants for searching a substring in a string.
|
|
|
|
* In most cases, performance is less than Volnitsky (see Volnitsky.h).
|
2016-01-27 03:26:36 +00:00
|
|
|
*/
|
|
|
|
|
2023-02-23 12:58:42 +00:00
|
|
|
namespace impl
|
|
|
|
{
|
2016-01-27 03:26:36 +00:00
|
|
|
|
2022-01-25 18:56:05 +00:00
|
|
|
class StringSearcherBase
|
2016-01-16 00:45:19 +00:00
|
|
|
{
|
2022-01-25 18:56:05 +00:00
|
|
|
public:
|
|
|
|
bool force_fallback = false;
|
2023-02-23 12:58:42 +00:00
|
|
|
|
2019-01-04 12:10:00 +00:00
|
|
|
#ifdef __SSE2__
|
2022-01-25 18:56:05 +00:00
|
|
|
protected:
|
2023-02-23 12:58:42 +00:00
|
|
|
static constexpr size_t N = sizeof(__m128i);
|
2016-01-16 00:45:19 +00:00
|
|
|
|
2023-02-23 12:58:42 +00:00
|
|
|
bool isPageSafe(const void * const ptr) const
|
2016-01-16 00:45:19 +00:00
|
|
|
{
|
2023-02-23 12:58:42 +00:00
|
|
|
return ((page_size - 1) & reinterpret_cast<std::uintptr_t>(ptr)) <= page_size - N;
|
2016-01-16 00:45:19 +00:00
|
|
|
}
|
2023-02-23 12:58:42 +00:00
|
|
|
|
|
|
|
private:
|
|
|
|
const Int64 page_size = ::getPageSize();
|
2016-01-16 00:45:19 +00:00
|
|
|
#endif
|
|
|
|
};
|
|
|
|
|
|
|
|
|
2023-02-23 12:58:42 +00:00
|
|
|
/// Performs case-sensitive or case-insensitive search of ASCII or UTF-8 strings
|
2015-10-14 12:12:56 +00:00
|
|
|
template <bool CaseSensitive, bool ASCII> class StringSearcher;
|
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
|
|
|
|
/// Case-sensitive ASCII and UTF8 searcher
|
|
|
|
template <bool ASCII>
|
|
|
|
class StringSearcher<true, ASCII> : public StringSearcherBase
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2016-01-16 00:45:19 +00:00
|
|
|
private:
|
2023-02-24 10:13:35 +00:00
|
|
|
/// string to be searched for
|
2020-02-22 05:46:35 +00:00
|
|
|
const uint8_t * const needle;
|
2023-02-24 10:13:35 +00:00
|
|
|
const uint8_t * const needle_end;
|
|
|
|
/// first character in `needle`
|
|
|
|
uint8_t first_needle_character = 0;
|
2016-01-16 00:45:19 +00:00
|
|
|
|
2019-01-04 12:10:00 +00:00
|
|
|
#ifdef __SSE4_1__
|
2023-02-24 10:13:35 +00:00
|
|
|
/// second character of "needle" (if its length is > 1)
|
|
|
|
uint8_t second_needle_character = 0;
|
|
|
|
/// first/second needle character broadcasted into a 16 bytes vector
|
|
|
|
__m128i first_needle_character_vec;
|
|
|
|
__m128i second_needle_character_vec;
|
|
|
|
/// vector of first 16 characters of `needle`
|
|
|
|
__m128i cache = _mm_setzero_si128();
|
|
|
|
uint16_t cachemask = 0;
|
2016-01-16 00:45:19 +00:00
|
|
|
#endif
|
2015-10-14 12:12:56 +00:00
|
|
|
|
|
|
|
public:
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
2023-02-24 10:13:35 +00:00
|
|
|
StringSearcher(const CharT * needle_, size_t needle_size)
|
2023-02-23 12:58:42 +00:00
|
|
|
: needle(reinterpret_cast<const uint8_t *>(needle_))
|
2023-02-24 10:13:35 +00:00
|
|
|
, needle_end(needle + needle_size)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
if (needle_size == 0)
|
2015-10-14 12:12:56 +00:00
|
|
|
return;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
first_needle_character = *needle;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
#ifdef __SSE4_1__
|
|
|
|
first_needle_character_vec = _mm_set1_epi8(first_needle_character);
|
|
|
|
if (needle_size > 1)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
second_needle_character = *(needle + 1);
|
|
|
|
second_needle_character_vec = _mm_set1_epi8(second_needle_character);
|
2015-10-14 12:12:56 +00:00
|
|
|
}
|
2022-03-11 21:47:28 +00:00
|
|
|
const auto * needle_pos = needle;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
for (uint8_t i = 0; i < N; ++i)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
cache = _mm_srli_si128(cache, 1);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
if (needle_pos != needle_end)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
cache = _mm_insert_epi8(cache, *needle_pos, N - 1);
|
|
|
|
cachemask |= 1 << i;
|
|
|
|
++needle_pos;
|
2015-10-14 12:12:56 +00:00
|
|
|
}
|
|
|
|
}
|
2016-01-16 00:45:19 +00:00
|
|
|
#endif
|
2015-10-14 12:12:56 +00:00
|
|
|
}
|
|
|
|
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
2023-02-24 10:13:35 +00:00
|
|
|
ALWAYS_INLINE bool compare(const CharT * /*haystack*/, const CharT * /*haystack_end*/, const CharT * pos) const
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2019-01-04 12:10:00 +00:00
|
|
|
#ifdef __SSE4_1__
|
2023-02-24 10:13:35 +00:00
|
|
|
if (isPageSafe(pos))
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
const __m128i haystack_characters = _mm_loadu_si128(reinterpret_cast<const __m128i *>(pos));
|
|
|
|
const __m128i comparison_result = _mm_cmpeq_epi8(haystack_characters, cache);
|
|
|
|
const uint16_t comparison_result_mask = _mm_movemask_epi8(comparison_result);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
if (0xffff == cachemask)
|
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
if (comparison_result_mask == cachemask)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
pos += N;
|
|
|
|
const auto * needle_pos = needle + N;
|
|
|
|
|
|
|
|
while (needle_pos < needle_end && *pos == *needle_pos)
|
|
|
|
++pos, ++needle_pos;
|
|
|
|
|
|
|
|
if (needle_pos == needle_end)
|
2015-10-14 12:12:56 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
2023-02-24 10:13:35 +00:00
|
|
|
else if ((comparison_result_mask & cachemask) == cachemask)
|
|
|
|
return true;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
return false;
|
|
|
|
}
|
2016-01-16 00:45:19 +00:00
|
|
|
#endif
|
2015-10-14 12:12:56 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
if (*pos == first_needle_character)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
++pos;
|
|
|
|
const auto * needle_pos = needle + 1;
|
2015-10-14 12:12:56 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
while (needle_pos < needle_end && *pos == *needle_pos)
|
|
|
|
++pos, ++needle_pos;
|
|
|
|
|
|
|
|
if (needle_pos == needle_end)
|
2015-10-14 12:12:56 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
2020-02-22 05:46:35 +00:00
|
|
|
const CharT * search(const CharT * haystack, const CharT * const haystack_end) const
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
const auto needle_size = needle_end - needle;
|
|
|
|
|
|
|
|
if (needle == needle_end)
|
2015-10-14 12:12:56 +00:00
|
|
|
return haystack;
|
|
|
|
|
2019-01-04 12:10:00 +00:00
|
|
|
#ifdef __SSE4_1__
|
2023-02-24 10:13:35 +00:00
|
|
|
/// Fast path for single-character needles. Compare 16 characters of the haystack against the needle character at once.
|
|
|
|
if (needle_size == 1)
|
|
|
|
{
|
|
|
|
while (haystack < haystack_end)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
if (haystack + N <= haystack_end && isPageSafe(haystack))
|
|
|
|
{
|
|
|
|
const __m128i haystack_characters = _mm_loadu_si128(reinterpret_cast<const __m128i *>(haystack));
|
|
|
|
const __m128i comparison_result = _mm_cmpeq_epi8(haystack_characters, first_needle_character_vec);
|
|
|
|
const uint16_t comparison_result_mask = _mm_movemask_epi8(comparison_result);
|
|
|
|
if (comparison_result_mask == 0)
|
|
|
|
{
|
|
|
|
haystack += N;
|
|
|
|
continue;
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
const int offset = std::countr_zero(comparison_result_mask);
|
|
|
|
haystack += offset;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
return haystack;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (haystack == haystack_end)
|
|
|
|
return haystack_end;
|
|
|
|
|
|
|
|
if (*haystack == first_needle_character)
|
|
|
|
return haystack;
|
|
|
|
|
|
|
|
++haystack;
|
|
|
|
}
|
|
|
|
|
|
|
|
return haystack_end;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
|
|
|
while (haystack < haystack_end && haystack_end - haystack >= needle_size)
|
|
|
|
{
|
|
|
|
#ifdef __SSE4_1__
|
|
|
|
/// Compare the [0:15] bytes from haystack and broadcasted 16 bytes vector from first character of needle.
|
|
|
|
/// Compare the [1:16] bytes from haystack and broadcasted 16 bytes vector from second character of needle.
|
|
|
|
/// Bit AND the results of above two comparisons and get the mask.
|
|
|
|
if ((haystack + 1 + N) <= haystack_end && isPageSafe(haystack + 1))
|
|
|
|
{
|
|
|
|
const __m128i haystack_characters_from_1st = _mm_loadu_si128(reinterpret_cast<const __m128i *>(haystack));
|
|
|
|
const __m128i haystack_characters_from_2nd = _mm_loadu_si128(reinterpret_cast<const __m128i *>(haystack + 1));
|
|
|
|
const __m128i comparison_result_1st = _mm_cmpeq_epi8(haystack_characters_from_1st, first_needle_character_vec);
|
|
|
|
const __m128i comparison_result_2nd = _mm_cmpeq_epi8(haystack_characters_from_2nd, second_needle_character_vec);
|
|
|
|
const __m128i comparison_result_combined = _mm_and_si128(comparison_result_1st, comparison_result_2nd);
|
|
|
|
const uint16_t comparison_result_mask = _mm_movemask_epi8(comparison_result_combined);
|
|
|
|
/// If the mask = 0, then first two characters [0:1] from needle are not in the [0:17] bytes of haystack.
|
|
|
|
if (comparison_result_mask == 0)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-23 12:58:42 +00:00
|
|
|
haystack += N;
|
2015-10-14 12:12:56 +00:00
|
|
|
continue;
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
const int offset = std::countr_zero(comparison_result_mask);
|
2015-10-14 12:12:56 +00:00
|
|
|
haystack += offset;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-02-23 12:58:42 +00:00
|
|
|
if (haystack + N <= haystack_end && isPageSafe(haystack))
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
/// Already find the haystack position where the [pos:pos + 1] two characters exactly match the first two characters of needle.
|
|
|
|
/// Compare the 16 bytes from needle (cache) and the first 16 bytes from haystack at once if the haystack size >= 16 bytes.
|
|
|
|
const __m128i haystack_characters = _mm_loadu_si128(reinterpret_cast<const __m128i *>(haystack));
|
|
|
|
const __m128i comparison_result_cache = _mm_cmpeq_epi8(haystack_characters, cache);
|
|
|
|
const uint16_t mask_offset = _mm_movemask_epi8(comparison_result_cache);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
if (0xffff == cachemask)
|
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
if (mask_offset == cachemask)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
const auto * haystack_pos = haystack + N;
|
|
|
|
const auto * needle_pos = needle + N;
|
|
|
|
|
|
|
|
while (haystack_pos < haystack_end && needle_pos < needle_end &&
|
|
|
|
*haystack_pos == *needle_pos)
|
|
|
|
++haystack_pos, ++needle_pos;
|
|
|
|
|
|
|
|
if (needle_pos == needle_end)
|
2015-10-14 12:12:56 +00:00
|
|
|
return haystack;
|
|
|
|
}
|
|
|
|
}
|
2023-02-24 10:13:35 +00:00
|
|
|
else if ((mask_offset & cachemask) == cachemask)
|
|
|
|
return haystack;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
++haystack;
|
2015-10-14 12:12:56 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
2016-01-16 00:45:19 +00:00
|
|
|
#endif
|
2015-10-14 12:12:56 +00:00
|
|
|
|
|
|
|
if (haystack == haystack_end)
|
|
|
|
return haystack_end;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
if (*haystack == first_needle_character)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
const auto * haystack_pos = haystack + 1;
|
|
|
|
const auto * needle_pos = needle + 1;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
while (haystack_pos < haystack_end && needle_pos < needle_end &&
|
|
|
|
*haystack_pos == *needle_pos)
|
|
|
|
++haystack_pos, ++needle_pos;
|
|
|
|
|
|
|
|
if (needle_pos == needle_end)
|
2015-10-14 12:12:56 +00:00
|
|
|
return haystack;
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
++haystack;
|
2015-10-14 12:12:56 +00:00
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
return haystack_end;
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
2023-02-23 12:58:42 +00:00
|
|
|
const CharT * search(const CharT * haystack, size_t haystack_size) const
|
2016-01-27 03:11:28 +00:00
|
|
|
{
|
|
|
|
return search(haystack, haystack + haystack_size);
|
|
|
|
}
|
2015-10-14 12:12:56 +00:00
|
|
|
};
|
|
|
|
|
2016-01-16 00:45:19 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
/// Case-insensitive ASCII searcher
|
2016-01-16 00:45:19 +00:00
|
|
|
template <>
|
2022-01-25 18:56:05 +00:00
|
|
|
class StringSearcher<false, true> : public StringSearcherBase
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2016-01-16 00:45:19 +00:00
|
|
|
private:
|
2015-10-14 12:12:56 +00:00
|
|
|
/// string to be searched for
|
2020-02-22 05:46:35 +00:00
|
|
|
const uint8_t * const needle;
|
|
|
|
const uint8_t * const needle_end;
|
2015-10-14 12:12:56 +00:00
|
|
|
/// lower and uppercase variants of the first character in `needle`
|
2023-02-23 12:58:42 +00:00
|
|
|
uint8_t l = 0;
|
|
|
|
uint8_t u = 0;
|
2016-01-16 00:45:19 +00:00
|
|
|
|
2019-01-04 12:10:00 +00:00
|
|
|
#ifdef __SSE4_1__
|
2015-10-14 12:12:56 +00:00
|
|
|
/// vectors filled with `l` and `u`, for determining leftmost position of the first symbol
|
|
|
|
__m128i patl, patu;
|
|
|
|
/// lower and uppercase vectors of first 16 characters of `needle`
|
|
|
|
__m128i cachel = _mm_setzero_si128(), cacheu = _mm_setzero_si128();
|
2023-02-23 12:58:42 +00:00
|
|
|
int cachemask = 0;
|
2016-01-16 00:45:19 +00:00
|
|
|
#endif
|
2015-10-14 12:12:56 +00:00
|
|
|
|
|
|
|
public:
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
2023-02-23 12:58:42 +00:00
|
|
|
StringSearcher(const CharT * needle_, size_t needle_size)
|
|
|
|
: needle(reinterpret_cast<const uint8_t *>(needle_))
|
|
|
|
, needle_end(needle + needle_size)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:14:58 +00:00
|
|
|
if (needle_size == 0)
|
2015-10-14 12:12:56 +00:00
|
|
|
return;
|
|
|
|
|
2020-02-22 05:46:35 +00:00
|
|
|
l = static_cast<uint8_t>(std::tolower(*needle));
|
|
|
|
u = static_cast<uint8_t>(std::toupper(*needle));
|
2015-10-14 12:12:56 +00:00
|
|
|
|
2019-01-04 12:10:00 +00:00
|
|
|
#ifdef __SSE4_1__
|
2015-10-14 12:12:56 +00:00
|
|
|
patl = _mm_set1_epi8(l);
|
|
|
|
patu = _mm_set1_epi8(u);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2022-03-11 21:47:28 +00:00
|
|
|
const auto * needle_pos = needle;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-02-23 12:58:42 +00:00
|
|
|
for (size_t i = 0; i < N; ++i)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
|
|
|
cachel = _mm_srli_si128(cachel, 1);
|
|
|
|
cacheu = _mm_srli_si128(cacheu, 1);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
if (needle_pos != needle_end)
|
|
|
|
{
|
2023-02-23 12:58:42 +00:00
|
|
|
cachel = _mm_insert_epi8(cachel, std::tolower(*needle_pos), N - 1);
|
|
|
|
cacheu = _mm_insert_epi8(cacheu, std::toupper(*needle_pos), N - 1);
|
2015-10-14 12:12:56 +00:00
|
|
|
cachemask |= 1 << i;
|
|
|
|
++needle_pos;
|
|
|
|
}
|
|
|
|
}
|
2016-01-16 00:45:19 +00:00
|
|
|
#endif
|
2015-10-14 12:12:56 +00:00
|
|
|
}
|
|
|
|
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
2020-02-22 05:46:35 +00:00
|
|
|
ALWAYS_INLINE bool compare(const CharT * /*haystack*/, const CharT * /*haystack_end*/, const CharT * pos) const
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2019-01-04 12:10:00 +00:00
|
|
|
#ifdef __SSE4_1__
|
2023-02-23 12:58:42 +00:00
|
|
|
if (isPageSafe(pos))
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
|
|
|
const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(pos));
|
|
|
|
const auto v_against_l = _mm_cmpeq_epi8(v_haystack, cachel);
|
|
|
|
const auto v_against_u = _mm_cmpeq_epi8(v_haystack, cacheu);
|
|
|
|
const auto v_against_l_or_u = _mm_or_si128(v_against_l, v_against_u);
|
|
|
|
const auto mask = _mm_movemask_epi8(v_against_l_or_u);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
if (0xffff == cachemask)
|
|
|
|
{
|
|
|
|
if (mask == cachemask)
|
|
|
|
{
|
2023-02-23 12:58:42 +00:00
|
|
|
pos += N;
|
|
|
|
const auto * needle_pos = needle + N;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
while (needle_pos < needle_end && std::tolower(*pos) == std::tolower(*needle_pos))
|
2018-08-26 01:24:21 +00:00
|
|
|
{
|
|
|
|
++pos;
|
|
|
|
++needle_pos;
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
if (needle_pos == needle_end)
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else if ((mask & cachemask) == cachemask)
|
|
|
|
return true;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
return false;
|
|
|
|
}
|
2016-01-16 00:45:19 +00:00
|
|
|
#endif
|
2015-10-14 12:12:56 +00:00
|
|
|
|
|
|
|
if (*pos == l || *pos == u)
|
|
|
|
{
|
|
|
|
++pos;
|
2022-03-11 21:47:28 +00:00
|
|
|
const auto * needle_pos = needle + 1;
|
2015-10-14 12:12:56 +00:00
|
|
|
|
|
|
|
while (needle_pos < needle_end && std::tolower(*pos) == std::tolower(*needle_pos))
|
2018-08-26 01:24:21 +00:00
|
|
|
{
|
|
|
|
++pos;
|
|
|
|
++needle_pos;
|
|
|
|
}
|
2015-10-14 12:12:56 +00:00
|
|
|
|
|
|
|
if (needle_pos == needle_end)
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
2020-02-22 05:46:35 +00:00
|
|
|
const CharT * search(const CharT * haystack, const CharT * const haystack_end) const
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2019-06-12 02:35:25 +00:00
|
|
|
if (needle == needle_end)
|
2015-10-14 12:12:56 +00:00
|
|
|
return haystack;
|
|
|
|
|
|
|
|
while (haystack < haystack_end)
|
|
|
|
{
|
2019-01-04 12:10:00 +00:00
|
|
|
#ifdef __SSE4_1__
|
2023-02-23 12:58:42 +00:00
|
|
|
if (haystack + N <= haystack_end && isPageSafe(haystack))
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
|
|
|
const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(haystack));
|
|
|
|
const auto v_against_l = _mm_cmpeq_epi8(v_haystack, patl);
|
|
|
|
const auto v_against_u = _mm_cmpeq_epi8(v_haystack, patu);
|
|
|
|
const auto v_against_l_or_u = _mm_or_si128(v_against_l, v_against_u);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
const auto mask = _mm_movemask_epi8(v_against_l_or_u);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
if (mask == 0)
|
|
|
|
{
|
2023-02-23 12:58:42 +00:00
|
|
|
haystack += N;
|
2015-10-14 12:12:56 +00:00
|
|
|
continue;
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2016-03-07 06:00:25 +00:00
|
|
|
const auto offset = __builtin_ctz(mask);
|
2015-10-14 12:12:56 +00:00
|
|
|
haystack += offset;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-02-23 12:58:42 +00:00
|
|
|
if (haystack + N <= haystack_end && isPageSafe(haystack))
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2019-01-04 12:10:00 +00:00
|
|
|
const auto v_haystack_offset = _mm_loadu_si128(reinterpret_cast<const __m128i *>(haystack));
|
|
|
|
const auto v_against_l_offset = _mm_cmpeq_epi8(v_haystack_offset, cachel);
|
|
|
|
const auto v_against_u_offset = _mm_cmpeq_epi8(v_haystack_offset, cacheu);
|
|
|
|
const auto v_against_l_or_u_offset = _mm_or_si128(v_against_l_offset, v_against_u_offset);
|
|
|
|
const auto mask_offset = _mm_movemask_epi8(v_against_l_or_u_offset);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
if (0xffff == cachemask)
|
|
|
|
{
|
2019-01-04 12:10:00 +00:00
|
|
|
if (mask_offset == cachemask)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-23 12:58:42 +00:00
|
|
|
const auto * haystack_pos = haystack + N;
|
|
|
|
const auto * needle_pos = needle + N;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
while (haystack_pos < haystack_end && needle_pos < needle_end &&
|
|
|
|
std::tolower(*haystack_pos) == std::tolower(*needle_pos))
|
2018-08-26 01:24:21 +00:00
|
|
|
{
|
|
|
|
++haystack_pos;
|
|
|
|
++needle_pos;
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
if (needle_pos == needle_end)
|
|
|
|
return haystack;
|
|
|
|
}
|
|
|
|
}
|
2019-01-04 12:10:00 +00:00
|
|
|
else if ((mask_offset & cachemask) == cachemask)
|
2015-10-14 12:12:56 +00:00
|
|
|
return haystack;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
++haystack;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
2023-02-24 10:13:35 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
if (haystack == haystack_end)
|
|
|
|
return haystack_end;
|
|
|
|
|
|
|
|
if (*haystack == l || *haystack == u)
|
|
|
|
{
|
|
|
|
const auto * haystack_pos = haystack + 1;
|
|
|
|
const auto * needle_pos = needle + 1;
|
|
|
|
|
|
|
|
while (haystack_pos < haystack_end && needle_pos < needle_end &&
|
|
|
|
std::tolower(*haystack_pos) == std::tolower(*needle_pos))
|
|
|
|
{
|
|
|
|
++haystack_pos;
|
|
|
|
++needle_pos;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (needle_pos == needle_end)
|
|
|
|
return haystack;
|
|
|
|
}
|
|
|
|
|
|
|
|
++haystack;
|
|
|
|
}
|
|
|
|
|
|
|
|
return haystack_end;
|
|
|
|
}
|
|
|
|
|
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
|
|
|
const CharT * search(const CharT * haystack, size_t haystack_size) const
|
|
|
|
{
|
|
|
|
return search(haystack, haystack + haystack_size);
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
/// Case-insensitive UTF-8 searcher
|
|
|
|
template <>
|
|
|
|
class StringSearcher<false, false> : public StringSearcherBase
|
|
|
|
{
|
|
|
|
private:
|
|
|
|
using UTF8SequenceBuffer = uint8_t[6];
|
|
|
|
|
|
|
|
/// substring to be searched for
|
|
|
|
const uint8_t * const needle;
|
|
|
|
const size_t needle_size;
|
|
|
|
const uint8_t * const needle_end = needle + needle_size;
|
|
|
|
/// lower and uppercase variants of the first octet of the first character in `needle`
|
|
|
|
bool first_needle_symbol_is_ascii = false;
|
|
|
|
uint8_t l = 0;
|
|
|
|
uint8_t u = 0;
|
|
|
|
|
|
|
|
#ifdef __SSE4_1__
|
|
|
|
/// vectors filled with `l` and `u`, for determining leftmost position of the first symbol
|
|
|
|
__m128i patl;
|
|
|
|
__m128i patu;
|
|
|
|
/// lower and uppercase vectors of first 16 characters of `needle`
|
|
|
|
__m128i cachel = _mm_setzero_si128();
|
|
|
|
__m128i cacheu = _mm_setzero_si128();
|
|
|
|
int cachemask = 0;
|
|
|
|
size_t cache_valid_len = 0;
|
|
|
|
size_t cache_actual_len = 0;
|
|
|
|
#endif
|
|
|
|
|
|
|
|
public:
|
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
|
|
|
StringSearcher(const CharT * needle_, size_t needle_size_)
|
|
|
|
: needle(reinterpret_cast<const uint8_t *>(needle_))
|
|
|
|
, needle_size(needle_size_)
|
|
|
|
{
|
2023-02-24 10:14:58 +00:00
|
|
|
if (needle_size == 0)
|
2023-02-24 10:13:35 +00:00
|
|
|
return;
|
|
|
|
|
|
|
|
UTF8SequenceBuffer l_seq;
|
|
|
|
UTF8SequenceBuffer u_seq;
|
|
|
|
|
|
|
|
if (*needle < 0x80u)
|
|
|
|
{
|
|
|
|
first_needle_symbol_is_ascii = true;
|
|
|
|
l = std::tolower(*needle);
|
|
|
|
u = std::toupper(*needle);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
auto first_u32 = UTF8::convertUTF8ToCodePoint(needle, needle_size);
|
|
|
|
|
|
|
|
/// Invalid UTF-8
|
|
|
|
if (!first_u32)
|
|
|
|
{
|
|
|
|
/// Process it verbatim as a sequence of bytes.
|
|
|
|
size_t src_len = UTF8::seqLength(*needle);
|
|
|
|
|
|
|
|
memcpy(l_seq, needle, src_len);
|
|
|
|
memcpy(u_seq, needle, src_len);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
uint32_t first_l_u32 = Poco::Unicode::toLower(*first_u32);
|
|
|
|
uint32_t first_u_u32 = Poco::Unicode::toUpper(*first_u32);
|
|
|
|
|
|
|
|
/// lower and uppercase variants of the first octet of the first character in `needle`
|
|
|
|
size_t length_l = UTF8::convertCodePointToUTF8(first_l_u32, l_seq, sizeof(l_seq));
|
|
|
|
size_t length_u = UTF8::convertCodePointToUTF8(first_u_u32, u_seq, sizeof(u_seq));
|
|
|
|
|
|
|
|
if (length_l != length_u)
|
|
|
|
force_fallback = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
l = l_seq[0];
|
|
|
|
u = u_seq[0];
|
|
|
|
|
|
|
|
if (force_fallback)
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef __SSE4_1__
|
|
|
|
/// for detecting leftmost position of the first symbol
|
|
|
|
patl = _mm_set1_epi8(l);
|
|
|
|
patu = _mm_set1_epi8(u);
|
|
|
|
/// lower and uppercase vectors of first 16 octets of `needle`
|
|
|
|
|
|
|
|
const auto * needle_pos = needle;
|
|
|
|
|
|
|
|
for (size_t i = 0; i < N;)
|
|
|
|
{
|
|
|
|
if (needle_pos == needle_end)
|
|
|
|
{
|
|
|
|
cachel = _mm_srli_si128(cachel, 1);
|
|
|
|
cacheu = _mm_srli_si128(cacheu, 1);
|
|
|
|
++i;
|
|
|
|
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
size_t src_len = std::min<size_t>(needle_end - needle_pos, UTF8::seqLength(*needle_pos));
|
|
|
|
auto c_u32 = UTF8::convertUTF8ToCodePoint(needle_pos, src_len);
|
|
|
|
|
|
|
|
if (c_u32)
|
|
|
|
{
|
|
|
|
int c_l_u32 = Poco::Unicode::toLower(*c_u32);
|
|
|
|
int c_u_u32 = Poco::Unicode::toUpper(*c_u32);
|
|
|
|
|
|
|
|
size_t dst_l_len = UTF8::convertCodePointToUTF8(c_l_u32, l_seq, sizeof(l_seq));
|
|
|
|
size_t dst_u_len = UTF8::convertCodePointToUTF8(c_u_u32, u_seq, sizeof(u_seq));
|
|
|
|
|
|
|
|
/// @note Unicode standard states it is a rare but possible occasion
|
|
|
|
if (!(dst_l_len == dst_u_len && dst_u_len == src_len))
|
|
|
|
{
|
|
|
|
force_fallback = true;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
2015-10-14 12:12:56 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
cache_actual_len += src_len;
|
|
|
|
if (cache_actual_len < N)
|
|
|
|
cache_valid_len += src_len;
|
2015-10-14 12:12:56 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
for (size_t j = 0; j < src_len && i < N; ++j, ++i)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
cachel = _mm_srli_si128(cachel, 1);
|
|
|
|
cacheu = _mm_srli_si128(cacheu, 1);
|
2015-10-14 12:12:56 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
if (needle_pos != needle_end)
|
2018-08-26 01:24:21 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
cachel = _mm_insert_epi8(cachel, l_seq[j], N - 1);
|
|
|
|
cacheu = _mm_insert_epi8(cacheu, u_seq[j], N - 1);
|
|
|
|
|
|
|
|
cachemask |= 1 << i;
|
2018-08-26 01:24:21 +00:00
|
|
|
++needle_pos;
|
|
|
|
}
|
2015-10-14 12:12:56 +00:00
|
|
|
}
|
|
|
|
}
|
2023-02-24 10:13:35 +00:00
|
|
|
#endif
|
2015-10-14 12:12:56 +00:00
|
|
|
}
|
2016-01-27 03:11:28 +00:00
|
|
|
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
2023-02-24 10:13:35 +00:00
|
|
|
ALWAYS_INLINE bool compareTrivial(const CharT * haystack_pos, const CharT * const haystack_end, const uint8_t * needle_pos) const
|
2016-01-27 03:11:28 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
while (haystack_pos < haystack_end && needle_pos < needle_end)
|
|
|
|
{
|
|
|
|
auto haystack_code_point = UTF8::convertUTF8ToCodePoint(haystack_pos, haystack_end - haystack_pos);
|
|
|
|
auto needle_code_point = UTF8::convertUTF8ToCodePoint(needle_pos, needle_end - needle_pos);
|
2016-01-16 00:45:19 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
/// Invalid UTF-8, should not compare equals
|
|
|
|
if (!haystack_code_point || !needle_code_point)
|
|
|
|
break;
|
2015-10-14 12:12:56 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
/// Not equals case insensitive.
|
|
|
|
if (Poco::Unicode::toLower(*haystack_code_point) != Poco::Unicode::toLower(*needle_code_point))
|
|
|
|
break;
|
2015-10-14 12:12:56 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
auto len = UTF8::seqLength(*haystack_pos);
|
|
|
|
haystack_pos += len;
|
2015-10-14 12:12:56 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
len = UTF8::seqLength(*needle_pos);
|
|
|
|
needle_pos += len;
|
2023-02-10 14:48:52 +00:00
|
|
|
}
|
2015-10-14 12:12:56 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
return needle_pos == needle_end;
|
2015-10-14 12:12:56 +00:00
|
|
|
}
|
|
|
|
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
2023-02-24 10:13:35 +00:00
|
|
|
ALWAYS_INLINE bool compare(const CharT * /*haystack*/, const CharT * haystack_end, const CharT * pos) const
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
|
2019-01-04 12:10:00 +00:00
|
|
|
#ifdef __SSE4_1__
|
2023-02-24 10:13:35 +00:00
|
|
|
if (isPageSafe(pos) && !force_fallback)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(pos));
|
|
|
|
const auto v_against_l = _mm_cmpeq_epi8(v_haystack, cachel);
|
|
|
|
const auto v_against_u = _mm_cmpeq_epi8(v_haystack, cacheu);
|
|
|
|
const auto v_against_l_or_u = _mm_or_si128(v_against_l, v_against_u);
|
|
|
|
const auto mask = _mm_movemask_epi8(v_against_l_or_u);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
if (0xffff == cachemask)
|
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
if (mask == cachemask)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
if (compareTrivial(pos, haystack_end, needle))
|
2015-10-14 12:12:56 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
2023-02-24 10:13:35 +00:00
|
|
|
else if ((mask & cachemask) == cachemask)
|
|
|
|
{
|
|
|
|
if (compareTrivial(pos, haystack_end, needle))
|
|
|
|
return true;
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
return false;
|
|
|
|
}
|
2016-01-16 00:45:19 +00:00
|
|
|
#endif
|
2015-10-14 12:12:56 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
if (*pos == l || *pos == u)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
pos += first_needle_symbol_is_ascii;
|
|
|
|
const auto * needle_pos = needle + first_needle_symbol_is_ascii;
|
2015-10-14 12:12:56 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
if (compareTrivial(pos, haystack_end, needle_pos))
|
2015-10-14 12:12:56 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
/** Returns haystack_end if not found.
|
|
|
|
*/
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
2020-02-22 05:46:35 +00:00
|
|
|
const CharT * search(const CharT * haystack, const CharT * const haystack_end) const
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:14:58 +00:00
|
|
|
if (needle_size == 0)
|
2015-10-14 12:12:56 +00:00
|
|
|
return haystack;
|
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
while (haystack < haystack_end)
|
2023-02-10 14:48:52 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
#ifdef __SSE4_1__
|
|
|
|
if (haystack + N <= haystack_end && isPageSafe(haystack) && !force_fallback)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(haystack));
|
|
|
|
const auto v_against_l = _mm_cmpeq_epi8(v_haystack, patl);
|
|
|
|
const auto v_against_u = _mm_cmpeq_epi8(v_haystack, patu);
|
|
|
|
const auto v_against_l_or_u = _mm_or_si128(v_against_l, v_against_u);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
const auto mask = _mm_movemask_epi8(v_against_l_or_u);
|
2023-02-10 14:48:52 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
if (mask == 0)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-23 12:58:42 +00:00
|
|
|
haystack += N;
|
2023-02-24 10:13:35 +00:00
|
|
|
UTF8::syncForward(haystack, haystack_end);
|
2015-10-14 12:12:56 +00:00
|
|
|
continue;
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
const auto offset = __builtin_ctz(mask);
|
2015-10-14 12:12:56 +00:00
|
|
|
haystack += offset;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-02-23 12:58:42 +00:00
|
|
|
if (haystack + N <= haystack_end && isPageSafe(haystack))
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
const auto v_haystack_offset = _mm_loadu_si128(reinterpret_cast<const __m128i *>(haystack));
|
|
|
|
const auto v_against_l_offset = _mm_cmpeq_epi8(v_haystack_offset, cachel);
|
|
|
|
const auto v_against_u_offset = _mm_cmpeq_epi8(v_haystack_offset, cacheu);
|
|
|
|
const auto v_against_l_or_u_offset = _mm_or_si128(v_against_l_offset, v_against_u_offset);
|
|
|
|
const auto mask_offset_both = _mm_movemask_epi8(v_against_l_or_u_offset);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
if (0xffff == cachemask)
|
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
if (mask_offset_both == cachemask)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
if (compareTrivial(haystack, haystack_end, needle))
|
2015-10-14 12:12:56 +00:00
|
|
|
return haystack;
|
|
|
|
}
|
|
|
|
}
|
2023-02-24 10:13:35 +00:00
|
|
|
else if ((mask_offset_both & cachemask) == cachemask)
|
|
|
|
{
|
|
|
|
if (compareTrivial(haystack, haystack_end, needle))
|
|
|
|
return haystack;
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
/// first octet was ok, but not the first 16, move to start of next sequence and reapply
|
|
|
|
haystack += UTF8::seqLength(*haystack);
|
2015-10-14 12:12:56 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
2016-01-16 00:45:19 +00:00
|
|
|
#endif
|
2015-10-14 12:12:56 +00:00
|
|
|
|
|
|
|
if (haystack == haystack_end)
|
|
|
|
return haystack_end;
|
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
if (*haystack == l || *haystack == u)
|
2015-10-14 12:12:56 +00:00
|
|
|
{
|
2023-02-24 10:13:35 +00:00
|
|
|
auto haystack_pos = haystack + first_needle_symbol_is_ascii;
|
|
|
|
const auto * needle_pos = needle + first_needle_symbol_is_ascii;
|
2015-10-14 12:12:56 +00:00
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
if (compareTrivial(haystack_pos, haystack_end, needle_pos))
|
2015-10-14 12:12:56 +00:00
|
|
|
return haystack;
|
|
|
|
}
|
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
/// advance to the start of the next sequence
|
|
|
|
haystack += UTF8::seqLength(*haystack);
|
2015-10-14 12:12:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return haystack_end;
|
|
|
|
}
|
2016-01-27 03:11:28 +00:00
|
|
|
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
2023-02-23 12:58:42 +00:00
|
|
|
const CharT * search(const CharT * haystack, size_t haystack_size) const
|
2016-01-27 03:11:28 +00:00
|
|
|
{
|
|
|
|
return search(haystack, haystack + haystack_size);
|
|
|
|
}
|
2015-10-14 12:12:56 +00:00
|
|
|
};
|
|
|
|
|
2023-02-24 10:13:35 +00:00
|
|
|
|
2019-08-21 08:12:39 +00:00
|
|
|
// Searches for needle surrounded by token-separators.
|
|
|
|
// Separators are anything inside ASCII (0-128) and not alphanum.
|
|
|
|
// Any value outside of basic ASCII (>=128) is considered a non-separator symbol, hence UTF-8 strings
|
|
|
|
// should work just fine. But any Unicode whitespace is not considered a token separtor.
|
|
|
|
template <typename StringSearcher>
|
2022-01-26 10:45:26 +00:00
|
|
|
class TokenSearcher : public StringSearcherBase
|
2019-08-21 08:12:39 +00:00
|
|
|
{
|
|
|
|
StringSearcher searcher;
|
|
|
|
size_t needle_size;
|
|
|
|
|
|
|
|
public:
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
2023-02-23 12:58:42 +00:00
|
|
|
TokenSearcher(const CharT * needle_, size_t needle_size_)
|
|
|
|
: searcher(needle_, needle_size_)
|
|
|
|
, needle_size(needle_size_)
|
2019-08-21 08:12:39 +00:00
|
|
|
{
|
2020-01-03 15:28:38 +00:00
|
|
|
if (std::any_of(needle_, needle_ + needle_size_, isTokenSeparator))
|
2023-01-23 21:13:58 +00:00
|
|
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Needle must not contain whitespace or separator characters");
|
2019-08-21 08:12:39 +00:00
|
|
|
|
|
|
|
}
|
|
|
|
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
2020-02-22 05:46:35 +00:00
|
|
|
ALWAYS_INLINE bool compare(const CharT * haystack, const CharT * haystack_end, const CharT * pos) const
|
2019-08-21 08:12:39 +00:00
|
|
|
{
|
|
|
|
// use searcher only if pos is in the beginning of token and pos + searcher.needle_size is end of token.
|
|
|
|
if (isToken(haystack, haystack_end, pos))
|
|
|
|
return searcher.compare(haystack, haystack_end, pos);
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
2020-02-22 05:46:35 +00:00
|
|
|
const CharT * search(const CharT * haystack, const CharT * const haystack_end) const
|
2019-08-21 08:12:39 +00:00
|
|
|
{
|
|
|
|
// use searcher.search(), then verify that returned value is a token
|
|
|
|
// if it is not, skip it and re-run
|
|
|
|
|
2020-02-22 05:46:35 +00:00
|
|
|
const auto * pos = haystack;
|
2019-08-21 08:12:39 +00:00
|
|
|
while (pos < haystack_end)
|
|
|
|
{
|
|
|
|
pos = searcher.search(pos, haystack_end);
|
|
|
|
if (pos == haystack_end || isToken(haystack, haystack_end, pos))
|
|
|
|
return pos;
|
|
|
|
|
|
|
|
// assuming that heendle does not contain any token separators.
|
|
|
|
pos += needle_size;
|
|
|
|
}
|
|
|
|
return haystack_end;
|
|
|
|
}
|
|
|
|
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
2023-02-23 12:58:42 +00:00
|
|
|
const CharT * search(const CharT * haystack, size_t haystack_size) const
|
2019-08-21 08:12:39 +00:00
|
|
|
{
|
|
|
|
return search(haystack, haystack + haystack_size);
|
|
|
|
}
|
|
|
|
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
2020-02-22 05:46:35 +00:00
|
|
|
ALWAYS_INLINE bool isToken(const CharT * haystack, const CharT * const haystack_end, const CharT* p) const
|
2019-08-21 08:12:39 +00:00
|
|
|
{
|
|
|
|
return (p == haystack || isTokenSeparator(*(p - 1)))
|
|
|
|
&& (p + needle_size >= haystack_end || isTokenSeparator(*(p + needle_size)));
|
|
|
|
}
|
|
|
|
|
2020-02-22 05:46:35 +00:00
|
|
|
ALWAYS_INLINE static bool isTokenSeparator(const uint8_t c)
|
2019-08-21 08:12:39 +00:00
|
|
|
{
|
2022-03-11 21:47:28 +00:00
|
|
|
return !(isAlphaNumericASCII(c) || !isASCII(c));
|
2019-08-21 08:12:39 +00:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2023-02-23 12:58:42 +00:00
|
|
|
}
|
2015-10-14 12:12:56 +00:00
|
|
|
|
2023-02-23 12:58:42 +00:00
|
|
|
using ASCIICaseSensitiveStringSearcher = impl::StringSearcher<true, true>;
|
|
|
|
using ASCIICaseInsensitiveStringSearcher = impl::StringSearcher<false, true>;
|
|
|
|
using UTF8CaseSensitiveStringSearcher = impl::StringSearcher<true, false>;
|
|
|
|
using UTF8CaseInsensitiveStringSearcher = impl::StringSearcher<false, false>;
|
|
|
|
using ASCIICaseSensitiveTokenSearcher = impl::TokenSearcher<ASCIICaseSensitiveStringSearcher>;
|
|
|
|
using ASCIICaseInsensitiveTokenSearcher = impl::TokenSearcher<ASCIICaseInsensitiveStringSearcher>;
|
2015-10-14 12:12:56 +00:00
|
|
|
|
Fix countSubstrings() & position() on patterns with 0-bytes
SQL functions countSubstrings(), countSubstringsCaseInsensitive(),
countSubstringsUTF8(), position(), positionCaseInsensitive(),
positionUTF8() with non-const pattern argument use fallback sorters
LibCASCIICaseSensitiveStringSearcher and LibCASCIICaseInsensitiveStringSearcher
which call ::strstr(), resp. ::strcasestr(). These functions assume that
the haystack is 0-terminated and they even document that. However, the
callers did not check if the haystack contains 0-byte (perhaps because
its sort of expensive). As a consequence, if the haystack contained a
zero byte in it's payload, matches behind this zero byte were ignored.
create table t (id UInt32, pattern String) engine = MergeTree() order by id;
insert into t values (1, 'x');
select countSubstrings('aaaxxxaa\0xxx', pattern) from t;
We returned 3 before this commit, now we return 6
2022-06-29 15:08:16 +00:00
|
|
|
/// Use only with short haystacks where cheap initialization is required.
|
|
|
|
template <bool CaseInsensitive>
|
2023-02-23 12:58:42 +00:00
|
|
|
struct StdLibASCIIStringSearcher
|
2016-01-27 03:26:36 +00:00
|
|
|
{
|
Fix countSubstrings() & position() on patterns with 0-bytes
SQL functions countSubstrings(), countSubstringsCaseInsensitive(),
countSubstringsUTF8(), position(), positionCaseInsensitive(),
positionUTF8() with non-const pattern argument use fallback sorters
LibCASCIICaseSensitiveStringSearcher and LibCASCIICaseInsensitiveStringSearcher
which call ::strstr(), resp. ::strcasestr(). These functions assume that
the haystack is 0-terminated and they even document that. However, the
callers did not check if the haystack contains 0-byte (perhaps because
its sort of expensive). As a consequence, if the haystack contained a
zero byte in it's payload, matches behind this zero byte were ignored.
create table t (id UInt32, pattern String) engine = MergeTree() order by id;
insert into t values (1, 'x');
select countSubstrings('aaaxxxaa\0xxx', pattern) from t;
We returned 3 before this commit, now we return 6
2022-06-29 15:08:16 +00:00
|
|
|
const char * const needle_start;
|
|
|
|
const char * const needle_end;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
2023-02-23 12:58:42 +00:00
|
|
|
StdLibASCIIStringSearcher(const CharT * const needle_start_, size_t needle_size_)
|
|
|
|
: needle_start(reinterpret_cast<const char *>(needle_start_))
|
|
|
|
, needle_end(reinterpret_cast<const char *>(needle_start) + needle_size_)
|
Fix countSubstrings() & position() on patterns with 0-bytes
SQL functions countSubstrings(), countSubstringsCaseInsensitive(),
countSubstringsUTF8(), position(), positionCaseInsensitive(),
positionUTF8() with non-const pattern argument use fallback sorters
LibCASCIICaseSensitiveStringSearcher and LibCASCIICaseInsensitiveStringSearcher
which call ::strstr(), resp. ::strcasestr(). These functions assume that
the haystack is 0-terminated and they even document that. However, the
callers did not check if the haystack contains 0-byte (perhaps because
its sort of expensive). As a consequence, if the haystack contained a
zero byte in it's payload, matches behind this zero byte were ignored.
create table t (id UInt32, pattern String) engine = MergeTree() order by id;
insert into t values (1, 'x');
select countSubstrings('aaaxxxaa\0xxx', pattern) from t;
We returned 3 before this commit, now we return 6
2022-06-29 15:08:16 +00:00
|
|
|
{}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
Fix countSubstrings() & position() on patterns with 0-bytes
SQL functions countSubstrings(), countSubstringsCaseInsensitive(),
countSubstringsUTF8(), position(), positionCaseInsensitive(),
positionUTF8() with non-const pattern argument use fallback sorters
LibCASCIICaseSensitiveStringSearcher and LibCASCIICaseInsensitiveStringSearcher
which call ::strstr(), resp. ::strcasestr(). These functions assume that
the haystack is 0-terminated and they even document that. However, the
callers did not check if the haystack contains 0-byte (perhaps because
its sort of expensive). As a consequence, if the haystack contained a
zero byte in it's payload, matches behind this zero byte were ignored.
create table t (id UInt32, pattern String) engine = MergeTree() order by id;
insert into t values (1, 'x');
select countSubstrings('aaaxxxaa\0xxx', pattern) from t;
We returned 3 before this commit, now we return 6
2022-06-29 15:08:16 +00:00
|
|
|
const CharT * search(const CharT * haystack_start, const CharT * const haystack_end) const
|
2016-01-27 03:26:36 +00:00
|
|
|
{
|
Fix countSubstrings() & position() on patterns with 0-bytes
SQL functions countSubstrings(), countSubstringsCaseInsensitive(),
countSubstringsUTF8(), position(), positionCaseInsensitive(),
positionUTF8() with non-const pattern argument use fallback sorters
LibCASCIICaseSensitiveStringSearcher and LibCASCIICaseInsensitiveStringSearcher
which call ::strstr(), resp. ::strcasestr(). These functions assume that
the haystack is 0-terminated and they even document that. However, the
callers did not check if the haystack contains 0-byte (perhaps because
its sort of expensive). As a consequence, if the haystack contained a
zero byte in it's payload, matches behind this zero byte were ignored.
create table t (id UInt32, pattern String) engine = MergeTree() order by id;
insert into t values (1, 'x');
select countSubstrings('aaaxxxaa\0xxx', pattern) from t;
We returned 3 before this commit, now we return 6
2022-06-29 15:08:16 +00:00
|
|
|
if constexpr (CaseInsensitive)
|
|
|
|
return std::search(
|
|
|
|
haystack_start, haystack_end, needle_start, needle_end,
|
|
|
|
[](char c1, char c2) {return std::toupper(c1) == std::toupper(c2);});
|
|
|
|
else
|
|
|
|
return std::search(
|
2022-06-30 11:43:25 +00:00
|
|
|
haystack_start, haystack_end, needle_start, needle_end,
|
|
|
|
[](char c1, char c2) {return c1 == c2;});
|
2016-01-27 03:26:36 +00:00
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2022-03-15 19:37:28 +00:00
|
|
|
template <typename CharT>
|
|
|
|
requires (sizeof(CharT) == 1)
|
2023-02-23 12:58:42 +00:00
|
|
|
const CharT * search(const CharT * haystack_start, size_t haystack_length) const
|
2016-01-27 03:26:36 +00:00
|
|
|
{
|
Fix countSubstrings() & position() on patterns with 0-bytes
SQL functions countSubstrings(), countSubstringsCaseInsensitive(),
countSubstringsUTF8(), position(), positionCaseInsensitive(),
positionUTF8() with non-const pattern argument use fallback sorters
LibCASCIICaseSensitiveStringSearcher and LibCASCIICaseInsensitiveStringSearcher
which call ::strstr(), resp. ::strcasestr(). These functions assume that
the haystack is 0-terminated and they even document that. However, the
callers did not check if the haystack contains 0-byte (perhaps because
its sort of expensive). As a consequence, if the haystack contained a
zero byte in it's payload, matches behind this zero byte were ignored.
create table t (id UInt32, pattern String) engine = MergeTree() order by id;
insert into t values (1, 'x');
select countSubstrings('aaaxxxaa\0xxx', pattern) from t;
We returned 3 before this commit, now we return 6
2022-06-29 15:08:16 +00:00
|
|
|
return search(haystack_start, haystack_start + haystack_length);
|
2016-01-27 03:26:36 +00:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2015-10-14 12:12:56 +00:00
|
|
|
}
|