2011-10-16 07:11:36 +00:00
|
|
|
|
#pragma once
|
|
|
|
|
|
2016-05-28 10:15:36 +00:00
|
|
|
|
#include <mutex>
|
2012-09-05 19:51:09 +00:00
|
|
|
|
|
2015-10-05 01:11:12 +00:00
|
|
|
|
#include <DB/Common/OptimizedRegularExpression.h>
|
2015-02-10 21:10:58 +00:00
|
|
|
|
#include <memory>
|
2011-10-16 07:11:36 +00:00
|
|
|
|
|
|
|
|
|
#include <DB/DataTypes/DataTypesNumberFixed.h>
|
|
|
|
|
#include <DB/DataTypes/DataTypeString.h>
|
2014-08-22 00:57:20 +00:00
|
|
|
|
#include <DB/DataTypes/DataTypeFixedString.h>
|
2011-10-16 07:11:36 +00:00
|
|
|
|
#include <DB/Columns/ColumnString.h>
|
2014-08-22 00:57:20 +00:00
|
|
|
|
#include <DB/Columns/ColumnFixedString.h>
|
2011-10-16 07:11:36 +00:00
|
|
|
|
#include <DB/Columns/ColumnConst.h>
|
2013-08-29 03:30:47 +00:00
|
|
|
|
#include <DB/Common/Volnitsky.h>
|
2011-10-16 07:11:36 +00:00
|
|
|
|
#include <DB/Functions/IFunction.h>
|
2015-10-14 12:12:56 +00:00
|
|
|
|
#include <DB/Common/StringSearcher.h>
|
2014-02-06 16:54:41 +00:00
|
|
|
|
#include <re2/re2.h>
|
|
|
|
|
#include <re2/stringpiece.h>
|
2016-01-27 03:11:28 +00:00
|
|
|
|
#include <Poco/UTF8String.h>
|
2011-10-16 07:11:36 +00:00
|
|
|
|
|
2014-07-23 15:25:48 +00:00
|
|
|
|
#include <mutex>
|
|
|
|
|
#include <stack>
|
2015-10-05 00:33:43 +00:00
|
|
|
|
#include <ext/range.hpp>
|
2015-09-22 13:43:32 +00:00
|
|
|
|
|
2014-07-23 15:25:48 +00:00
|
|
|
|
|
2011-10-16 07:11:36 +00:00
|
|
|
|
namespace DB
|
|
|
|
|
{
|
|
|
|
|
|
|
|
|
|
/** Функции поиска и замены в строках:
|
|
|
|
|
*
|
|
|
|
|
* position(haystack, needle) - обычный поиск подстроки в строке, возвращает позицию (в байтах) найденной подстроки, начиная с 1, или 0, если подстрока не найдена.
|
|
|
|
|
* positionUTF8(haystack, needle) - то же самое, но позиция вычисляется в кодовых точках, при условии, что строка в кодировке UTF-8.
|
2016-01-27 03:11:28 +00:00
|
|
|
|
* positionCaseInsensitive(haystack, needle)
|
|
|
|
|
* positionCaseInsensitiveUTF8(haystack, needle)
|
2014-06-26 00:58:14 +00:00
|
|
|
|
*
|
2011-10-17 08:28:39 +00:00
|
|
|
|
* like(haystack, pattern) - поиск по регулярному выражению LIKE; возвращает 0 или 1. Регистронезависимое, но только для латиницы.
|
|
|
|
|
* notLike(haystack, pattern)
|
|
|
|
|
*
|
2011-10-16 07:11:36 +00:00
|
|
|
|
* match(haystack, pattern) - поиск по регулярному выражению re2; возвращает 0 или 1.
|
2012-07-21 03:45:48 +00:00
|
|
|
|
*
|
2013-03-18 10:49:31 +00:00
|
|
|
|
* Применяет регексп re2 и достаёт:
|
|
|
|
|
* - первый subpattern, если в regexp-е есть subpattern;
|
|
|
|
|
* - нулевой subpattern (сматчившуюся часть, иначе);
|
|
|
|
|
* - если не сматчилось - пустую строку.
|
|
|
|
|
* extract(haystack, pattern)
|
2011-10-17 08:28:39 +00:00
|
|
|
|
*
|
2011-10-16 07:11:36 +00:00
|
|
|
|
* replaceOne(haystack, pattern, replacement) - замена шаблона по заданным правилам, только первое вхождение.
|
|
|
|
|
* replaceAll(haystack, pattern, replacement) - замена шаблона по заданным правилам, все вхождения.
|
|
|
|
|
*
|
2014-01-27 16:01:53 +00:00
|
|
|
|
* replaceRegexpOne(haystack, pattern, replacement) - замена шаблона по заданному регекспу, только первое вхождение.
|
|
|
|
|
* replaceRegexpAll(haystack, pattern, replacement) - замена шаблона по заданному регекспу, все вхождения.
|
|
|
|
|
*
|
2012-12-26 14:51:41 +00:00
|
|
|
|
* Внимание! На данный момент, аргументы needle, pattern, n, replacement обязаны быть константами.
|
2011-10-16 07:11:36 +00:00
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
/** Детали реализации функций семейства position в зависимости от ASCII/UTF8 и case sensitiveness.
|
|
|
|
|
*/
|
|
|
|
|
struct PositionCaseSensitiveASCII
|
2011-10-16 07:11:36 +00:00
|
|
|
|
{
|
2016-01-27 03:11:28 +00:00
|
|
|
|
/// Объект для поиска одной подстроки среди большого количества данных, уложенных подряд. Может слегка сложно инициализироваться.
|
|
|
|
|
using SearcherInBigHaystack = VolnitskyImpl<true, true>;
|
2011-10-17 08:28:39 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
/// Объект для поиска каждый раз разных подстрок, создаваемый на каждую подстроку. Не должен сложно инициализироваться.
|
2016-01-27 03:26:36 +00:00
|
|
|
|
using SearcherInSmallHaystack = LibCASCIICaseSensitiveStringSearcher;
|
2016-01-27 03:11:28 +00:00
|
|
|
|
|
|
|
|
|
static SearcherInBigHaystack createSearcherInBigHaystack(const char * needle_data, size_t needle_size, size_t haystack_size_hint)
|
2011-10-16 07:11:36 +00:00
|
|
|
|
{
|
2016-01-27 03:11:28 +00:00
|
|
|
|
return SearcherInBigHaystack(needle_data, needle_size, haystack_size_hint);
|
|
|
|
|
}
|
2011-10-16 07:11:36 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
static SearcherInSmallHaystack createSearcherInSmallHaystack(const char * needle_data, size_t needle_size)
|
|
|
|
|
{
|
|
|
|
|
return SearcherInSmallHaystack(needle_data, needle_size);
|
|
|
|
|
}
|
2011-10-16 07:11:36 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
/// Посчитать число символов от begin до end.
|
|
|
|
|
static size_t countChars(const char * begin, const char * end)
|
|
|
|
|
{
|
|
|
|
|
return end - begin;
|
|
|
|
|
}
|
2013-08-29 03:30:47 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
/// Перевести строку в нижний регистр. Только для регистронезависимого поиска. Можно неэффективно, так как вызывается от одиночной строки.
|
|
|
|
|
static void toLowerIfNeed(std::string & s)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
};
|
2011-10-16 07:11:36 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
struct PositionCaseInsensitiveASCII
|
|
|
|
|
{
|
|
|
|
|
/// Здесь не используется Volnitsky, потому что один человек померял, что так лучше. Будет хорошо, если вы подвергнете это сомнению.
|
|
|
|
|
using SearcherInBigHaystack = ASCIICaseInsensitiveStringSearcher;
|
2016-01-27 03:26:36 +00:00
|
|
|
|
using SearcherInSmallHaystack = LibCASCIICaseInsensitiveStringSearcher;
|
2011-10-16 07:11:36 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
static SearcherInBigHaystack createSearcherInBigHaystack(const char * needle_data, size_t needle_size, size_t haystack_size_hint)
|
|
|
|
|
{
|
|
|
|
|
return SearcherInBigHaystack(needle_data, needle_size);
|
|
|
|
|
}
|
2013-12-08 02:29:40 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
static SearcherInSmallHaystack createSearcherInSmallHaystack(const char * needle_data, size_t needle_size)
|
|
|
|
|
{
|
|
|
|
|
return SearcherInSmallHaystack(needle_data, needle_size);
|
2011-10-17 08:28:39 +00:00
|
|
|
|
}
|
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
static size_t countChars(const char * begin, const char * end)
|
2011-10-17 08:28:39 +00:00
|
|
|
|
{
|
2016-01-27 03:11:28 +00:00
|
|
|
|
return end - begin;
|
|
|
|
|
}
|
2015-09-29 14:00:30 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
static void toLowerIfNeed(std::string & s)
|
|
|
|
|
{
|
|
|
|
|
std::transform(std::begin(s), std::end(s), std::begin(s), tolower);
|
2011-10-17 08:28:39 +00:00
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
struct PositionCaseSensitiveUTF8
|
|
|
|
|
{
|
|
|
|
|
using SearcherInBigHaystack = VolnitskyImpl<true, false>;
|
2016-01-27 03:26:36 +00:00
|
|
|
|
using SearcherInSmallHaystack = LibCASCIICaseSensitiveStringSearcher;
|
2016-01-27 03:11:28 +00:00
|
|
|
|
|
|
|
|
|
static SearcherInBigHaystack createSearcherInBigHaystack(const char * needle_data, size_t needle_size, size_t haystack_size_hint)
|
|
|
|
|
{
|
|
|
|
|
return SearcherInBigHaystack(needle_data, needle_size, haystack_size_hint);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static SearcherInSmallHaystack createSearcherInSmallHaystack(const char * needle_data, size_t needle_size)
|
|
|
|
|
{
|
|
|
|
|
return SearcherInSmallHaystack(needle_data, needle_size);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static size_t countChars(const char * begin, const char * end)
|
|
|
|
|
{
|
|
|
|
|
size_t res = 0;
|
|
|
|
|
for (auto it = begin; it != end; ++it)
|
|
|
|
|
if (!UTF8::isContinuationOctet(static_cast<UInt8>(*it)))
|
|
|
|
|
++res;
|
|
|
|
|
return res;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void toLowerIfNeed(std::string & s)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
};
|
2011-10-17 08:28:39 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
struct PositionCaseInsensitiveUTF8
|
2011-10-17 08:28:39 +00:00
|
|
|
|
{
|
2016-01-27 03:11:28 +00:00
|
|
|
|
using SearcherInBigHaystack = VolnitskyImpl<false, false>;
|
2016-01-27 03:26:36 +00:00
|
|
|
|
using SearcherInSmallHaystack = UTF8CaseInsensitiveStringSearcher; /// TODO Очень неоптимально.
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
static SearcherInBigHaystack createSearcherInBigHaystack(const char * needle_data, size_t needle_size, size_t haystack_size_hint)
|
|
|
|
|
{
|
|
|
|
|
return SearcherInBigHaystack(needle_data, needle_size, haystack_size_hint);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static SearcherInSmallHaystack createSearcherInSmallHaystack(const char * needle_data, size_t needle_size)
|
|
|
|
|
{
|
|
|
|
|
return SearcherInSmallHaystack(needle_data, needle_size);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static size_t countChars(const char * begin, const char * end)
|
|
|
|
|
{
|
|
|
|
|
size_t res = 0;
|
|
|
|
|
for (auto it = begin; it != end; ++it)
|
|
|
|
|
if (!UTF8::isContinuationOctet(static_cast<UInt8>(*it)))
|
|
|
|
|
++res;
|
|
|
|
|
return res;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void toLowerIfNeed(std::string & s)
|
|
|
|
|
{
|
|
|
|
|
Poco::UTF8::toLowerInPlace(s);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
template <typename Impl>
|
|
|
|
|
struct PositionImpl
|
|
|
|
|
{
|
|
|
|
|
using ResultType = UInt64;
|
|
|
|
|
|
|
|
|
|
/// Поиск одной подстроки во многих строках.
|
|
|
|
|
static void vector_constant(const ColumnString::Chars_t & data, const ColumnString::Offsets_t & offsets,
|
2011-10-17 08:28:39 +00:00
|
|
|
|
const std::string & needle,
|
2016-04-15 00:33:21 +00:00
|
|
|
|
PaddedPODArray<UInt64> & res)
|
2011-10-17 08:28:39 +00:00
|
|
|
|
{
|
|
|
|
|
const UInt8 * begin = &data[0];
|
|
|
|
|
const UInt8 * pos = begin;
|
|
|
|
|
const UInt8 * end = pos + data.size();
|
2011-10-16 07:11:36 +00:00
|
|
|
|
|
2011-10-17 08:28:39 +00:00
|
|
|
|
/// Текущий индекс в массиве строк.
|
|
|
|
|
size_t i = 0;
|
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
typename Impl::SearcherInBigHaystack searcher = Impl::createSearcherInBigHaystack(needle.data(), needle.size(), end - pos);
|
2013-08-29 03:30:47 +00:00
|
|
|
|
|
2011-10-17 08:28:39 +00:00
|
|
|
|
/// Искать будем следующее вхождение сразу во всех строках.
|
2013-08-29 03:30:47 +00:00
|
|
|
|
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
|
2011-10-17 08:28:39 +00:00
|
|
|
|
{
|
|
|
|
|
/// Определим, к какому индексу оно относится.
|
2015-10-11 04:25:02 +00:00
|
|
|
|
while (begin + offsets[i] <= pos)
|
2013-12-08 02:29:40 +00:00
|
|
|
|
{
|
|
|
|
|
res[i] = 0;
|
2011-10-17 08:28:39 +00:00
|
|
|
|
++i;
|
2013-12-08 02:29:40 +00:00
|
|
|
|
}
|
2011-10-17 08:28:39 +00:00
|
|
|
|
|
|
|
|
|
/// Проверяем, что вхождение не переходит через границы строк.
|
|
|
|
|
if (pos + needle.size() < begin + offsets[i])
|
|
|
|
|
{
|
2016-01-27 03:11:28 +00:00
|
|
|
|
size_t prev_offset = i != 0 ? offsets[i - 1] : 0;
|
|
|
|
|
res[i] = 1 + Impl::countChars(
|
2016-01-27 05:11:51 +00:00
|
|
|
|
reinterpret_cast<const char *>(begin + prev_offset),
|
|
|
|
|
reinterpret_cast<const char *>(pos));
|
2011-10-17 08:28:39 +00:00
|
|
|
|
}
|
2013-12-08 02:29:40 +00:00
|
|
|
|
else
|
|
|
|
|
res[i] = 0;
|
2011-10-17 08:28:39 +00:00
|
|
|
|
|
|
|
|
|
pos = begin + offsets[i];
|
|
|
|
|
++i;
|
|
|
|
|
}
|
2013-12-08 02:29:40 +00:00
|
|
|
|
|
|
|
|
|
memset(&res[i], 0, (res.size() - i) * sizeof(res[0]));
|
2011-10-16 07:11:36 +00:00
|
|
|
|
}
|
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
/// Поиск одной подстроки в одной строке.
|
|
|
|
|
static void constant_constant(std::string data, std::string needle, UInt64 & res)
|
2011-10-16 07:11:36 +00:00
|
|
|
|
{
|
2016-01-27 03:11:28 +00:00
|
|
|
|
Impl::toLowerIfNeed(data);
|
|
|
|
|
Impl::toLowerIfNeed(needle);
|
|
|
|
|
|
|
|
|
|
res = data.find(needle);
|
|
|
|
|
if (res == std::string::npos)
|
|
|
|
|
res = 0;
|
|
|
|
|
else
|
|
|
|
|
res = 1 + Impl::countChars(data.data(), data.data() + res);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Поиск каждой раз разной одной подстроки в каждой раз разной строке.
|
|
|
|
|
static void vector_vector(
|
|
|
|
|
const ColumnString::Chars_t & haystack_data, const ColumnString::Offsets_t & haystack_offsets,
|
|
|
|
|
const ColumnString::Chars_t & needle_data, const ColumnString::Offsets_t & needle_offsets,
|
2016-04-15 00:33:21 +00:00
|
|
|
|
PaddedPODArray<UInt64> & res)
|
2016-01-27 03:11:28 +00:00
|
|
|
|
{
|
|
|
|
|
ColumnString::Offset_t prev_haystack_offset = 0;
|
|
|
|
|
ColumnString::Offset_t prev_needle_offset = 0;
|
|
|
|
|
|
|
|
|
|
size_t size = haystack_offsets.size();
|
|
|
|
|
|
|
|
|
|
for (size_t i = 0; i < size; ++i)
|
2015-09-29 14:00:30 +00:00
|
|
|
|
{
|
2016-01-27 03:11:28 +00:00
|
|
|
|
size_t needle_size = needle_offsets[i] - prev_needle_offset - 1;
|
|
|
|
|
size_t haystack_size = haystack_offsets[i] - prev_haystack_offset - 1;
|
2015-09-29 14:00:30 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
if (0 == needle_size)
|
2015-09-29 14:00:30 +00:00
|
|
|
|
{
|
2016-01-27 03:11:28 +00:00
|
|
|
|
/// Пустая строка всегда находится в самом начале haystack.
|
|
|
|
|
res[i] = 1;
|
2015-09-29 14:00:30 +00:00
|
|
|
|
}
|
2016-01-27 04:34:10 +00:00
|
|
|
|
else
|
2015-09-29 14:00:30 +00:00
|
|
|
|
{
|
2016-01-27 04:34:10 +00:00
|
|
|
|
/// Предполагается, что StringSearcher не очень сложно инициализировать.
|
|
|
|
|
typename Impl::SearcherInSmallHaystack searcher = Impl::createSearcherInSmallHaystack(
|
|
|
|
|
reinterpret_cast<const char *>(&needle_data[prev_needle_offset]),
|
|
|
|
|
needle_offsets[i] - prev_needle_offset - 1); /// нулевой байт на конце
|
|
|
|
|
|
|
|
|
|
/// searcher возвращает указатель на найденную подстроку или на конец haystack.
|
|
|
|
|
size_t pos = searcher.search(&haystack_data[prev_haystack_offset], &haystack_data[haystack_offsets[i] - 1])
|
|
|
|
|
- &haystack_data[prev_haystack_offset];
|
|
|
|
|
|
|
|
|
|
if (pos != haystack_size)
|
|
|
|
|
{
|
|
|
|
|
res[i] = 1 + Impl::countChars(
|
|
|
|
|
reinterpret_cast<const char *>(&haystack_data[prev_haystack_offset]),
|
|
|
|
|
reinterpret_cast<const char *>(&haystack_data[prev_haystack_offset + pos]));
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
res[i] = 0;
|
2015-09-29 14:00:30 +00:00
|
|
|
|
}
|
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
prev_haystack_offset = haystack_offsets[i];
|
|
|
|
|
prev_needle_offset = needle_offsets[i];
|
2015-09-24 11:39:43 +00:00
|
|
|
|
}
|
2011-10-16 07:11:36 +00:00
|
|
|
|
}
|
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
/// Поиск многих подстрок в одной строке.
|
|
|
|
|
static void constant_vector(
|
|
|
|
|
const String & haystack,
|
|
|
|
|
const ColumnString::Chars_t & needle_data, const ColumnString::Offsets_t & needle_offsets,
|
2016-04-15 00:33:21 +00:00
|
|
|
|
PaddedPODArray<UInt64> & res)
|
2015-09-24 14:28:31 +00:00
|
|
|
|
{
|
2016-01-27 03:11:28 +00:00
|
|
|
|
// NOTE Можно было бы использовать индексацию haystack. Но это - редкий случай.
|
2015-09-22 13:43:32 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
ColumnString::Offset_t prev_needle_offset = 0;
|
2015-09-22 13:43:32 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
size_t size = needle_offsets.size();
|
2015-09-22 13:43:32 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
for (size_t i = 0; i < size; ++i)
|
2015-09-22 13:43:32 +00:00
|
|
|
|
{
|
2016-01-27 03:11:28 +00:00
|
|
|
|
size_t needle_size = needle_offsets[i] - prev_needle_offset - 1;
|
|
|
|
|
|
|
|
|
|
if (0 == needle_size)
|
2015-09-22 13:43:32 +00:00
|
|
|
|
{
|
2016-01-27 03:11:28 +00:00
|
|
|
|
res[i] = 1;
|
2015-09-22 13:43:32 +00:00
|
|
|
|
}
|
2016-01-27 04:34:10 +00:00
|
|
|
|
else
|
2016-01-27 03:11:28 +00:00
|
|
|
|
{
|
2016-01-27 04:34:10 +00:00
|
|
|
|
typename Impl::SearcherInSmallHaystack searcher = Impl::createSearcherInSmallHaystack(
|
|
|
|
|
reinterpret_cast<const char *>(&needle_data[prev_needle_offset]),
|
|
|
|
|
needle_offsets[i] - prev_needle_offset - 1);
|
|
|
|
|
|
|
|
|
|
size_t pos = searcher.search(
|
|
|
|
|
reinterpret_cast<const UInt8 *>(haystack.data()),
|
|
|
|
|
reinterpret_cast<const UInt8 *>(haystack.data()) + haystack.size()) - reinterpret_cast<const UInt8 *>(haystack.data());
|
|
|
|
|
|
|
|
|
|
if (pos != haystack.size())
|
|
|
|
|
{
|
|
|
|
|
res[i] = 1 + Impl::countChars(haystack.data(), haystack.data() + pos);
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
res[i] = 0;
|
2016-01-27 03:11:28 +00:00
|
|
|
|
}
|
2015-09-22 13:43:32 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
prev_needle_offset = needle_offsets[i];
|
2015-09-22 13:43:32 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
2011-10-24 12:10:59 +00:00
|
|
|
|
/// Переводит выражение LIKE в regexp re2. Например, abc%def -> ^abc.*def$
|
2011-10-17 08:28:39 +00:00
|
|
|
|
inline String likePatternToRegexp(const String & pattern)
|
|
|
|
|
{
|
2015-03-27 02:12:45 +00:00
|
|
|
|
String res;
|
2011-10-17 08:28:39 +00:00
|
|
|
|
res.reserve(pattern.size() * 2);
|
|
|
|
|
const char * pos = pattern.data();
|
|
|
|
|
const char * end = pos + pattern.size();
|
|
|
|
|
|
2015-03-27 02:12:45 +00:00
|
|
|
|
if (pos < end && *pos == '%')
|
|
|
|
|
++pos;
|
|
|
|
|
else
|
|
|
|
|
res = "^";
|
|
|
|
|
|
2011-10-17 08:28:39 +00:00
|
|
|
|
while (pos < end)
|
|
|
|
|
{
|
|
|
|
|
switch (*pos)
|
|
|
|
|
{
|
|
|
|
|
case '^': case '$': case '.': case '[': case '|': case '(': case ')': case '?': case '*': case '+': case '{':
|
|
|
|
|
res += '\\';
|
|
|
|
|
res += *pos;
|
|
|
|
|
break;
|
|
|
|
|
case '%':
|
2015-03-27 02:12:45 +00:00
|
|
|
|
if (pos + 1 != end)
|
2015-09-03 15:35:34 +00:00
|
|
|
|
res += ".*";
|
2015-03-27 02:12:45 +00:00
|
|
|
|
else
|
|
|
|
|
return res;
|
2011-10-17 08:28:39 +00:00
|
|
|
|
break;
|
|
|
|
|
case '_':
|
|
|
|
|
res += ".";
|
|
|
|
|
break;
|
|
|
|
|
case '\\':
|
|
|
|
|
++pos;
|
|
|
|
|
if (pos == end)
|
|
|
|
|
res += "\\\\";
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
if (*pos == '%' || *pos == '_')
|
|
|
|
|
res += *pos;
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
res += '\\';
|
|
|
|
|
res += *pos;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
res += *pos;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
++pos;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
res += '$';
|
|
|
|
|
return res;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2011-10-24 12:10:59 +00:00
|
|
|
|
/// Сводится ли выражение LIKE к поиску подстроки в строке?
|
|
|
|
|
inline bool likePatternIsStrstr(const String & pattern, String & res)
|
|
|
|
|
{
|
|
|
|
|
res = "";
|
|
|
|
|
|
2014-10-09 23:14:06 +00:00
|
|
|
|
if (pattern.size() < 2 || pattern.front() != '%' || pattern.back() != '%')
|
2011-10-24 12:10:59 +00:00
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
res.reserve(pattern.size() * 2);
|
|
|
|
|
|
|
|
|
|
const char * pos = pattern.data();
|
|
|
|
|
const char * end = pos + pattern.size();
|
|
|
|
|
|
|
|
|
|
++pos;
|
|
|
|
|
--end;
|
|
|
|
|
|
|
|
|
|
while (pos < end)
|
|
|
|
|
{
|
|
|
|
|
switch (*pos)
|
|
|
|
|
{
|
|
|
|
|
case '%': case '_':
|
|
|
|
|
return false;
|
|
|
|
|
case '\\':
|
|
|
|
|
++pos;
|
|
|
|
|
if (pos == end)
|
|
|
|
|
return false;
|
|
|
|
|
else
|
|
|
|
|
res += *pos;
|
|
|
|
|
break;
|
|
|
|
|
default:
|
|
|
|
|
res += *pos;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
++pos;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
2014-07-23 15:25:48 +00:00
|
|
|
|
namespace Regexps
|
2011-10-17 08:28:39 +00:00
|
|
|
|
{
|
2014-07-23 15:25:48 +00:00
|
|
|
|
struct Holder;
|
|
|
|
|
struct Deleter;
|
|
|
|
|
|
2014-07-28 09:49:16 +00:00
|
|
|
|
using Regexp = OptimizedRegularExpressionImpl<false>;
|
2014-07-23 15:25:48 +00:00
|
|
|
|
using KnownRegexps = std::map<String, std::unique_ptr<Holder>>;
|
2014-07-24 13:16:09 +00:00
|
|
|
|
using Pointer = std::unique_ptr<Regexp, Deleter>;
|
2011-10-17 08:28:39 +00:00
|
|
|
|
|
2014-07-23 15:25:48 +00:00
|
|
|
|
/// Container for regular expressions with embedded mutex for safe addition and removal
|
|
|
|
|
struct Holder
|
2011-10-17 08:28:39 +00:00
|
|
|
|
{
|
2014-07-23 15:25:48 +00:00
|
|
|
|
std::mutex mutex;
|
2014-07-24 13:16:09 +00:00
|
|
|
|
std::stack<std::unique_ptr<Regexp>> stack;
|
2014-07-23 15:25:48 +00:00
|
|
|
|
|
|
|
|
|
/** Extracts and returns a pointer from the collection if it's not empty,
|
|
|
|
|
* creates a new one by calling provided f() otherwise.
|
|
|
|
|
*/
|
|
|
|
|
template <typename Factory> Pointer get(Factory && f);
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/** Specialized deleter for std::unique_ptr.
|
|
|
|
|
* Returns underlying pointer back to holder thus reclaiming its ownership.
|
|
|
|
|
*/
|
|
|
|
|
struct Deleter
|
|
|
|
|
{
|
|
|
|
|
Holder * holder;
|
2011-10-17 08:28:39 +00:00
|
|
|
|
|
2014-07-23 15:25:48 +00:00
|
|
|
|
Deleter(Holder * holder = nullptr) : holder{holder} {}
|
|
|
|
|
|
2014-07-24 13:16:09 +00:00
|
|
|
|
void operator()(Regexp * owning_ptr) const
|
2014-07-23 15:25:48 +00:00
|
|
|
|
{
|
|
|
|
|
std::lock_guard<std::mutex> lock{holder->mutex};
|
|
|
|
|
holder->stack.emplace(owning_ptr);
|
|
|
|
|
}
|
2011-10-17 08:28:39 +00:00
|
|
|
|
|
2014-07-23 15:25:48 +00:00
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
template <typename Factory>
|
|
|
|
|
inline Pointer Holder::get(Factory && f)
|
|
|
|
|
{
|
|
|
|
|
std::lock_guard<std::mutex> lock{mutex};
|
|
|
|
|
|
|
|
|
|
if (stack.empty())
|
|
|
|
|
return { f(), this };
|
|
|
|
|
|
|
|
|
|
auto regexp = stack.top().release();
|
|
|
|
|
stack.pop();
|
|
|
|
|
|
|
|
|
|
return { regexp, this };
|
2011-10-17 08:28:39 +00:00
|
|
|
|
}
|
|
|
|
|
|
2014-07-23 15:25:48 +00:00
|
|
|
|
template <bool like>
|
2015-02-11 01:24:51 +00:00
|
|
|
|
inline Regexp createRegexp(const std::string & pattern, int flags) { return {pattern, flags}; }
|
2014-07-23 15:25:48 +00:00
|
|
|
|
template <>
|
2015-02-11 01:24:51 +00:00
|
|
|
|
inline Regexp createRegexp<true>(const std::string & pattern, int flags) { return {likePatternToRegexp(pattern), flags}; }
|
2014-07-23 15:25:48 +00:00
|
|
|
|
|
2015-02-11 01:24:51 +00:00
|
|
|
|
template <bool like, bool no_capture>
|
2014-07-23 15:25:48 +00:00
|
|
|
|
inline Pointer get(const std::string & pattern)
|
2011-10-17 08:28:39 +00:00
|
|
|
|
{
|
2014-07-23 15:25:48 +00:00
|
|
|
|
/// C++11 has thread-safe function-local statics on most modern compilers.
|
2015-02-11 01:24:51 +00:00
|
|
|
|
static KnownRegexps known_regexps; /// Разные переменные для разных параметров шаблона.
|
2014-07-23 15:25:48 +00:00
|
|
|
|
static std::mutex mutex;
|
|
|
|
|
std::lock_guard<std::mutex> lock{mutex};
|
2011-10-17 08:28:39 +00:00
|
|
|
|
|
2014-07-23 15:25:48 +00:00
|
|
|
|
auto it = known_regexps.find(pattern);
|
2011-10-17 08:28:39 +00:00
|
|
|
|
if (known_regexps.end() == it)
|
2015-02-10 21:10:58 +00:00
|
|
|
|
it = known_regexps.emplace(pattern, std::make_unique<Holder>()).first;
|
2011-10-17 08:28:39 +00:00
|
|
|
|
|
2015-09-03 19:19:19 +00:00
|
|
|
|
return it->second->get([&pattern]
|
|
|
|
|
{
|
|
|
|
|
int flags = OptimizedRegularExpression::RE_DOT_NL;
|
2015-09-03 15:35:34 +00:00
|
|
|
|
if (no_capture)
|
|
|
|
|
flags |= OptimizedRegularExpression::RE_NO_CAPTURE;
|
2015-09-03 19:19:19 +00:00
|
|
|
|
|
2015-09-03 15:35:34 +00:00
|
|
|
|
return new Regexp{createRegexp<like>(pattern, flags)};
|
2014-07-23 15:25:48 +00:00
|
|
|
|
});
|
2011-10-17 08:28:39 +00:00
|
|
|
|
}
|
2014-07-23 15:25:48 +00:00
|
|
|
|
}
|
2011-10-17 08:28:39 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** like - использовать выражения LIKE, если true; использовать выражения re2, если false.
|
|
|
|
|
* Замечание: хотелось бы запускать регексп сразу над всем массивом, аналогично функции position,
|
|
|
|
|
* но для этого пришлось бы сделать поддержку символов \0 в движке регулярных выражений,
|
|
|
|
|
* и их интерпретацию как начал и концов строк.
|
|
|
|
|
*/
|
|
|
|
|
template <bool like, bool revert = false>
|
|
|
|
|
struct MatchImpl
|
|
|
|
|
{
|
2016-05-28 10:35:44 +00:00
|
|
|
|
using ResultType = UInt8;
|
2011-10-17 08:28:39 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
static void vector_constant(const ColumnString::Chars_t & data, const ColumnString::Offsets_t & offsets,
|
2011-10-17 08:28:39 +00:00
|
|
|
|
const std::string & pattern,
|
2016-04-15 00:33:21 +00:00
|
|
|
|
PaddedPODArray<UInt8> & res)
|
2011-10-17 08:28:39 +00:00
|
|
|
|
{
|
2011-10-24 12:10:59 +00:00
|
|
|
|
String strstr_pattern;
|
|
|
|
|
/// Простой случай, когда выражение LIKE сводится к поиску подстроки в строке
|
|
|
|
|
if (like && likePatternIsStrstr(pattern, strstr_pattern))
|
|
|
|
|
{
|
|
|
|
|
const UInt8 * begin = &data[0];
|
|
|
|
|
const UInt8 * pos = begin;
|
|
|
|
|
const UInt8 * end = pos + data.size();
|
|
|
|
|
|
|
|
|
|
/// Текущий индекс в массиве строк.
|
|
|
|
|
size_t i = 0;
|
2011-10-17 08:28:39 +00:00
|
|
|
|
|
2015-03-27 02:12:45 +00:00
|
|
|
|
/// TODO Надо сделать так, чтобы searcher был общим на все вызовы функции.
|
2013-08-29 03:30:47 +00:00
|
|
|
|
Volnitsky searcher(strstr_pattern.data(), strstr_pattern.size(), end - pos);
|
|
|
|
|
|
2011-10-24 12:10:59 +00:00
|
|
|
|
/// Искать будем следующее вхождение сразу во всех строках.
|
2013-08-29 03:30:47 +00:00
|
|
|
|
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
|
2011-10-24 12:10:59 +00:00
|
|
|
|
{
|
|
|
|
|
/// Определим, к какому индексу оно относится.
|
2015-10-11 04:25:02 +00:00
|
|
|
|
while (begin + offsets[i] <= pos)
|
2013-12-08 02:29:40 +00:00
|
|
|
|
{
|
|
|
|
|
res[i] = revert;
|
2011-10-24 12:10:59 +00:00
|
|
|
|
++i;
|
2013-12-08 02:29:40 +00:00
|
|
|
|
}
|
2011-10-24 12:10:59 +00:00
|
|
|
|
|
|
|
|
|
/// Проверяем, что вхождение не переходит через границы строк.
|
|
|
|
|
if (pos + strstr_pattern.size() < begin + offsets[i])
|
2012-08-03 18:49:39 +00:00
|
|
|
|
res[i] = !revert;
|
2013-12-08 02:29:40 +00:00
|
|
|
|
else
|
|
|
|
|
res[i] = revert;
|
2011-10-24 12:10:59 +00:00
|
|
|
|
|
|
|
|
|
pos = begin + offsets[i];
|
|
|
|
|
++i;
|
|
|
|
|
}
|
2013-12-08 02:29:40 +00:00
|
|
|
|
|
2015-03-27 02:12:45 +00:00
|
|
|
|
/// Хвостик, в котором не может быть подстрок.
|
2013-12-08 02:29:40 +00:00
|
|
|
|
memset(&res[i], revert, (res.size() - i) * sizeof(res[0]));
|
2011-10-24 12:10:59 +00:00
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
size_t size = offsets.size();
|
2015-03-27 02:12:45 +00:00
|
|
|
|
|
|
|
|
|
const auto & regexp = Regexps::get<like, true>(pattern);
|
|
|
|
|
|
|
|
|
|
std::string required_substring;
|
|
|
|
|
bool is_trivial;
|
|
|
|
|
bool required_substring_is_prefix; /// для anchored выполнения регекспа.
|
|
|
|
|
|
|
|
|
|
regexp->getAnalyzeResult(required_substring, is_trivial, required_substring_is_prefix);
|
|
|
|
|
|
|
|
|
|
if (required_substring.empty())
|
|
|
|
|
{
|
2015-04-03 16:16:02 +00:00
|
|
|
|
if (!regexp->getRE2()) /// Пустой регексп. Всегда матчит.
|
2015-03-27 02:12:45 +00:00
|
|
|
|
{
|
2015-04-03 16:16:02 +00:00
|
|
|
|
memset(&res[0], 1, size * sizeof(res[0]));
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
size_t prev_offset = 0;
|
|
|
|
|
for (size_t i = 0; i < size; ++i)
|
|
|
|
|
{
|
|
|
|
|
res[i] = revert ^ regexp->getRE2()->Match(
|
|
|
|
|
re2_st::StringPiece(reinterpret_cast<const char *>(&data[prev_offset]), offsets[i] - prev_offset - 1),
|
|
|
|
|
0, offsets[i] - prev_offset - 1, re2_st::RE2::UNANCHORED, nullptr, 0);
|
2015-03-27 02:12:45 +00:00
|
|
|
|
|
2015-04-03 16:16:02 +00:00
|
|
|
|
prev_offset = offsets[i];
|
|
|
|
|
}
|
2015-03-27 02:12:45 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
/// NOTE Это почти совпадает со случаем likePatternIsStrstr.
|
|
|
|
|
|
|
|
|
|
const UInt8 * begin = &data[0];
|
|
|
|
|
const UInt8 * pos = begin;
|
|
|
|
|
const UInt8 * end = pos + data.size();
|
|
|
|
|
|
|
|
|
|
/// Текущий индекс в массиве строк.
|
|
|
|
|
size_t i = 0;
|
|
|
|
|
|
|
|
|
|
Volnitsky searcher(required_substring.data(), required_substring.size(), end - pos);
|
|
|
|
|
|
|
|
|
|
/// Искать будем следующее вхождение сразу во всех строках.
|
|
|
|
|
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
|
|
|
|
|
{
|
|
|
|
|
/// Определим, к какому индексу оно относится.
|
2015-10-11 04:25:02 +00:00
|
|
|
|
while (begin + offsets[i] <= pos)
|
2015-03-27 02:12:45 +00:00
|
|
|
|
{
|
|
|
|
|
res[i] = revert;
|
|
|
|
|
++i;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Проверяем, что вхождение не переходит через границы строк.
|
|
|
|
|
if (pos + strstr_pattern.size() < begin + offsets[i])
|
|
|
|
|
{
|
|
|
|
|
/// И если не переходит - при необходимости, проверяем регекспом.
|
|
|
|
|
|
|
|
|
|
if (is_trivial)
|
|
|
|
|
res[i] = !revert;
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
const char * str_data = reinterpret_cast<const char *>(&data[i != 0 ? offsets[i - 1] : 0]);
|
|
|
|
|
size_t str_size = (i != 0 ? offsets[i] - offsets[i - 1] : offsets[0]) - 1;
|
|
|
|
|
|
2015-06-29 18:47:46 +00:00
|
|
|
|
/** Даже в случае required_substring_is_prefix используем UNANCHORED проверку регекспа,
|
|
|
|
|
* чтобы он мог сматчиться, когда required_substring встречается в строке несколько раз,
|
|
|
|
|
* и на первом вхождении регексп не матчит.
|
|
|
|
|
*/
|
|
|
|
|
|
2015-03-27 02:12:45 +00:00
|
|
|
|
if (required_substring_is_prefix)
|
|
|
|
|
res[i] = revert ^ regexp->getRE2()->Match(
|
|
|
|
|
re2_st::StringPiece(str_data, str_size),
|
2015-06-29 18:47:46 +00:00
|
|
|
|
reinterpret_cast<const char *>(pos) - str_data, str_size, re2_st::RE2::UNANCHORED, nullptr, 0);
|
2015-03-27 02:12:45 +00:00
|
|
|
|
else
|
|
|
|
|
res[i] = revert ^ regexp->getRE2()->Match(
|
|
|
|
|
re2_st::StringPiece(str_data, str_size),
|
|
|
|
|
0, str_size, re2_st::RE2::UNANCHORED, nullptr, 0);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
res[i] = revert;
|
|
|
|
|
|
|
|
|
|
pos = begin + offsets[i];
|
|
|
|
|
++i;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
memset(&res[i], revert, (res.size() - i) * sizeof(res[0]));
|
|
|
|
|
}
|
2011-10-24 12:10:59 +00:00
|
|
|
|
}
|
2011-10-17 08:28:39 +00:00
|
|
|
|
}
|
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
static void constant_constant(const std::string & data, const std::string & pattern, UInt8 & res)
|
2011-10-17 08:28:39 +00:00
|
|
|
|
{
|
2015-02-11 01:24:51 +00:00
|
|
|
|
const auto & regexp = Regexps::get<like, true>(pattern);
|
2014-07-23 15:25:48 +00:00
|
|
|
|
res = revert ^ regexp->match(data);
|
2011-10-17 08:28:39 +00:00
|
|
|
|
}
|
2016-01-27 03:11:28 +00:00
|
|
|
|
|
|
|
|
|
static void vector_vector(
|
|
|
|
|
const ColumnString::Chars_t & haystack_data, const ColumnString::Offsets_t & haystack_offsets,
|
|
|
|
|
const ColumnString::Chars_t & needle_data, const ColumnString::Offsets_t & needle_offsets,
|
2016-04-15 00:33:21 +00:00
|
|
|
|
PaddedPODArray<UInt8> & res)
|
2016-01-27 03:11:28 +00:00
|
|
|
|
{
|
|
|
|
|
throw Exception("Functions 'like' and 'match' doesn't support non-constant needle argument", ErrorCodes::ILLEGAL_COLUMN);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Поиск многих подстрок в одной строке.
|
|
|
|
|
static void constant_vector(
|
|
|
|
|
const String & haystack,
|
|
|
|
|
const ColumnString::Chars_t & needle_data, const ColumnString::Offsets_t & needle_offsets,
|
2016-04-15 00:33:21 +00:00
|
|
|
|
PaddedPODArray<UInt8> & res)
|
2016-01-27 03:11:28 +00:00
|
|
|
|
{
|
|
|
|
|
throw Exception("Functions 'like' and 'match' doesn't support non-constant needle argument", ErrorCodes::ILLEGAL_COLUMN);
|
|
|
|
|
}
|
2011-10-17 08:28:39 +00:00
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
2013-02-28 13:01:07 +00:00
|
|
|
|
struct ExtractImpl
|
|
|
|
|
{
|
2013-09-15 05:51:43 +00:00
|
|
|
|
static void vector(const ColumnString::Chars_t & data, const ColumnString::Offsets_t & offsets,
|
2013-02-28 13:01:07 +00:00
|
|
|
|
const std::string & pattern,
|
2013-09-15 05:51:43 +00:00
|
|
|
|
ColumnString::Chars_t & res_data, ColumnString::Offsets_t & res_offsets)
|
2013-02-28 13:01:07 +00:00
|
|
|
|
{
|
2013-10-03 19:23:08 +00:00
|
|
|
|
res_data.reserve(data.size() / 5);
|
2013-02-28 13:01:07 +00:00
|
|
|
|
res_offsets.resize(offsets.size());
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2015-02-11 01:24:51 +00:00
|
|
|
|
const auto & regexp = Regexps::get<false, false>(pattern);
|
2013-02-28 13:01:07 +00:00
|
|
|
|
|
2014-07-23 15:25:48 +00:00
|
|
|
|
unsigned capture = regexp->getNumberOfSubpatterns() > 0 ? 1 : 0;
|
2013-02-28 13:01:07 +00:00
|
|
|
|
OptimizedRegularExpression::MatchVec matches;
|
|
|
|
|
matches.reserve(capture + 1);
|
|
|
|
|
size_t prev_offset = 0;
|
|
|
|
|
size_t res_offset = 0;
|
|
|
|
|
|
|
|
|
|
for (size_t i = 0; i < offsets.size(); ++i)
|
|
|
|
|
{
|
|
|
|
|
size_t cur_offset = offsets[i];
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2014-07-23 15:25:48 +00:00
|
|
|
|
unsigned count = regexp->match(reinterpret_cast<const char *>(&data[prev_offset]), cur_offset - prev_offset - 1, matches, capture + 1);
|
2013-03-04 10:10:19 +00:00
|
|
|
|
if (count > capture && matches[capture].offset != std::string::npos)
|
2013-02-28 13:01:07 +00:00
|
|
|
|
{
|
2014-07-28 09:49:16 +00:00
|
|
|
|
const auto & match = matches[capture];
|
2013-02-28 13:01:07 +00:00
|
|
|
|
res_data.resize(res_offset + match.length + 1);
|
2016-04-15 00:33:21 +00:00
|
|
|
|
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], &data[prev_offset + match.offset], match.length);
|
2013-02-28 13:01:07 +00:00
|
|
|
|
res_offset += match.length;
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
res_data.resize(res_offset + 1);
|
|
|
|
|
}
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-02-28 13:01:07 +00:00
|
|
|
|
res_data[res_offset] = 0;
|
|
|
|
|
++res_offset;
|
|
|
|
|
res_offsets[i] = res_offset;
|
|
|
|
|
|
|
|
|
|
prev_offset = cur_offset;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
2014-01-27 16:01:53 +00:00
|
|
|
|
/** Заменить все вхождения регекспа needle на строку replacement. needle и replacement - константы.
|
2014-01-28 13:02:37 +00:00
|
|
|
|
* Replacement может содержать подстановки, например '\2-\3-\1'
|
2014-01-27 16:01:53 +00:00
|
|
|
|
*/
|
|
|
|
|
template <bool replaceOne = false>
|
|
|
|
|
struct ReplaceRegexpImpl
|
|
|
|
|
{
|
2014-02-06 16:54:41 +00:00
|
|
|
|
/// Последовательность инструкций, описывает как получить конечную строку. Каждый элемент
|
|
|
|
|
/// либо подстановка, тогда первое число в паре ее id,
|
|
|
|
|
/// либо строка, которую необходимо вставить, записана второй в паре. (id = -1)
|
2016-05-28 10:35:44 +00:00
|
|
|
|
using Instructions = std::vector< std::pair<int, std::string> >;
|
2014-02-06 16:54:41 +00:00
|
|
|
|
|
2014-01-28 13:02:37 +00:00
|
|
|
|
static void split(const std::string & s, Instructions & instructions)
|
|
|
|
|
{
|
|
|
|
|
instructions.clear();
|
|
|
|
|
String now = "";
|
|
|
|
|
for (size_t i = 0; i < s.size(); ++i)
|
|
|
|
|
{
|
2014-02-06 16:54:41 +00:00
|
|
|
|
if (s[i] == '\\' && i + 1 < s.size())
|
2014-01-28 13:02:37 +00:00
|
|
|
|
{
|
2014-02-06 16:54:41 +00:00
|
|
|
|
if (isdigit(s[i+1])) /// Подстановка
|
2014-01-28 13:02:37 +00:00
|
|
|
|
{
|
2014-02-06 16:54:41 +00:00
|
|
|
|
if (!now.empty())
|
|
|
|
|
{
|
|
|
|
|
instructions.push_back(std::make_pair(-1, now));
|
|
|
|
|
now = "";
|
|
|
|
|
}
|
|
|
|
|
instructions.push_back(std::make_pair(s[i+1] - '0', ""));
|
2014-01-28 13:02:37 +00:00
|
|
|
|
}
|
2014-02-06 16:54:41 +00:00
|
|
|
|
else
|
|
|
|
|
now += s[i+1]; /// Экранирование
|
2014-01-28 13:02:37 +00:00
|
|
|
|
++i;
|
|
|
|
|
}
|
2014-02-06 16:54:41 +00:00
|
|
|
|
else
|
|
|
|
|
now += s[i]; /// Обычный символ
|
2014-01-28 13:02:37 +00:00
|
|
|
|
}
|
|
|
|
|
if (!now.empty())
|
|
|
|
|
{
|
|
|
|
|
instructions.push_back(std::make_pair(-1, now));
|
|
|
|
|
now = "";
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2014-01-27 16:01:53 +00:00
|
|
|
|
static void vector(const ColumnString::Chars_t & data, const ColumnString::Offsets_t & offsets,
|
|
|
|
|
const std::string & needle, const std::string & replacement,
|
|
|
|
|
ColumnString::Chars_t & res_data, ColumnString::Offsets_t & res_offsets)
|
|
|
|
|
{
|
|
|
|
|
ColumnString::Offset_t res_offset = 0;
|
|
|
|
|
res_data.reserve(data.size());
|
|
|
|
|
size_t size = offsets.size();
|
|
|
|
|
res_offsets.resize(size);
|
|
|
|
|
|
2014-02-06 16:54:41 +00:00
|
|
|
|
RE2 searcher(needle);
|
|
|
|
|
int capture = std::min(searcher.NumberOfCapturingGroups() + 1, 10);
|
|
|
|
|
re2::StringPiece matches[10];
|
2014-01-27 16:01:53 +00:00
|
|
|
|
|
2014-01-28 13:02:37 +00:00
|
|
|
|
Instructions instructions;
|
|
|
|
|
split(replacement, instructions);
|
|
|
|
|
|
|
|
|
|
for (const auto & it : instructions)
|
2014-02-06 17:02:32 +00:00
|
|
|
|
if (it.first >= capture)
|
2014-01-28 13:02:37 +00:00
|
|
|
|
throw Exception("Invalid replace instruction in replacement string. Id: " + toString(it.first) +
|
2014-02-06 17:02:32 +00:00
|
|
|
|
", but regexp has only " + toString(capture - 1) + " subpatterns",
|
2014-01-28 13:02:37 +00:00
|
|
|
|
ErrorCodes::BAD_ARGUMENTS);
|
|
|
|
|
|
2014-02-06 16:54:41 +00:00
|
|
|
|
/// Искать вхождение сразу во всех сроках нельзя, будем двигаться вдоль каждой независимо
|
|
|
|
|
for (size_t id = 0; id < size; ++id)
|
2014-01-27 16:01:53 +00:00
|
|
|
|
{
|
2016-04-15 00:33:21 +00:00
|
|
|
|
int from = id > 0 ? offsets[id - 1] : 0;
|
2014-02-06 16:54:41 +00:00
|
|
|
|
int start_pos = 0;
|
|
|
|
|
re2::StringPiece input(reinterpret_cast<const char*>(&data[0] + from), offsets[id] - from - 1);
|
2014-01-27 16:01:53 +00:00
|
|
|
|
|
2014-02-06 16:54:41 +00:00
|
|
|
|
while (start_pos < input.length())
|
2014-01-27 16:01:53 +00:00
|
|
|
|
{
|
2014-02-06 16:54:41 +00:00
|
|
|
|
/// Правда ли, что с этой строкой больше не надо выполнять преобразования
|
|
|
|
|
bool can_finish_current_string = false;
|
2014-01-27 16:01:53 +00:00
|
|
|
|
|
2014-02-06 16:54:41 +00:00
|
|
|
|
if (searcher.Match(input, start_pos, input.length(), re2::RE2::Anchor::UNANCHORED, matches, capture))
|
|
|
|
|
{
|
2014-07-28 09:49:16 +00:00
|
|
|
|
const auto & match = matches[0];
|
2014-02-06 16:54:41 +00:00
|
|
|
|
size_t char_to_copy = (match.data() - input.data()) - start_pos;
|
2014-01-27 16:01:53 +00:00
|
|
|
|
|
2014-02-06 16:54:41 +00:00
|
|
|
|
/// Копируем данные без изменения
|
|
|
|
|
res_data.resize(res_data.size() + char_to_copy);
|
2016-04-15 00:33:21 +00:00
|
|
|
|
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], input.data() + start_pos, char_to_copy);
|
2014-02-06 16:54:41 +00:00
|
|
|
|
res_offset += char_to_copy;
|
|
|
|
|
start_pos += char_to_copy + match.length();
|
2014-01-27 16:01:53 +00:00
|
|
|
|
|
2014-02-06 16:54:41 +00:00
|
|
|
|
/// Выполняем инструкции подстановки
|
|
|
|
|
for (const auto & it : instructions)
|
2014-01-28 13:02:37 +00:00
|
|
|
|
{
|
2014-02-06 16:54:41 +00:00
|
|
|
|
if (it.first >= 0)
|
|
|
|
|
{
|
|
|
|
|
res_data.resize(res_data.size() + matches[it.first].length());
|
2016-04-15 00:33:21 +00:00
|
|
|
|
memcpySmallAllowReadWriteOverflow15(
|
|
|
|
|
&res_data[res_offset], matches[it.first].data(), matches[it.first].length());
|
2014-02-06 16:54:41 +00:00
|
|
|
|
res_offset += matches[it.first].length();
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
res_data.resize(res_data.size() + it.second.size());
|
2016-04-15 00:33:21 +00:00
|
|
|
|
memcpySmallAllowReadWriteOverflow15(
|
|
|
|
|
&res_data[res_offset], it.second.data(), it.second.size());
|
2014-02-06 16:54:41 +00:00
|
|
|
|
res_offset += it.second.size();
|
|
|
|
|
}
|
2014-01-28 13:02:37 +00:00
|
|
|
|
}
|
2014-02-07 12:26:25 +00:00
|
|
|
|
if (replaceOne || match.length() == 0)
|
2014-02-06 16:54:41 +00:00
|
|
|
|
can_finish_current_string = true;
|
|
|
|
|
} else
|
2014-01-27 16:01:53 +00:00
|
|
|
|
can_finish_current_string = true;
|
|
|
|
|
|
2014-02-06 16:54:41 +00:00
|
|
|
|
/// Если пора, копируем все символы до конца строки
|
|
|
|
|
if (can_finish_current_string)
|
|
|
|
|
{
|
|
|
|
|
res_data.resize(res_data.size() + input.length() - start_pos);
|
2016-04-15 00:33:21 +00:00
|
|
|
|
memcpySmallAllowReadWriteOverflow15(
|
|
|
|
|
&res_data[res_offset], input.data() + start_pos, input.length() - start_pos);
|
2014-02-06 16:54:41 +00:00
|
|
|
|
res_offset += input.length() - start_pos;
|
|
|
|
|
res_offsets[id] = res_offset;
|
|
|
|
|
start_pos = input.length();
|
|
|
|
|
}
|
2014-01-27 16:01:53 +00:00
|
|
|
|
}
|
2014-02-06 16:54:41 +00:00
|
|
|
|
res_data.resize(res_data.size() + 1);
|
|
|
|
|
res_data[res_offset++] = 0;
|
|
|
|
|
res_offsets[id] = res_offset;
|
2014-01-27 16:01:53 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void vector_fixed(const ColumnString::Chars_t & data, size_t n,
|
|
|
|
|
const std::string & needle, const std::string & replacement,
|
|
|
|
|
ColumnString::Chars_t & res_data, ColumnString::Offsets_t & res_offsets)
|
|
|
|
|
{
|
|
|
|
|
ColumnString::Offset_t res_offset = 0;
|
|
|
|
|
size_t size = data.size() / n;
|
|
|
|
|
res_data.reserve(data.size());
|
|
|
|
|
res_offsets.resize(size);
|
|
|
|
|
|
2014-02-06 16:54:41 +00:00
|
|
|
|
RE2 searcher(needle);
|
|
|
|
|
int capture = std::min(searcher.NumberOfCapturingGroups() + 1, 10);
|
|
|
|
|
re2::StringPiece matches[10];
|
2014-01-27 16:01:53 +00:00
|
|
|
|
|
2014-01-28 13:02:37 +00:00
|
|
|
|
Instructions instructions;
|
|
|
|
|
split(replacement, instructions);
|
|
|
|
|
|
|
|
|
|
for (const auto & it : instructions)
|
2014-02-06 17:02:32 +00:00
|
|
|
|
if (it.first >= capture)
|
2014-01-28 13:02:37 +00:00
|
|
|
|
throw Exception("Invalid replace instruction in replacement string. Id: " + toString(it.first) +
|
2014-02-06 17:02:32 +00:00
|
|
|
|
", but regexp has only " + toString(capture - 1) + " subpatterns",
|
2014-01-28 13:02:37 +00:00
|
|
|
|
ErrorCodes::BAD_ARGUMENTS);
|
|
|
|
|
|
2014-02-06 16:54:41 +00:00
|
|
|
|
/// Искать вхождение сразу во всех сроках нельзя, будем двигаться вдоль каждой независимо.
|
|
|
|
|
for (size_t id = 0; id < size; ++id)
|
2014-01-27 16:01:53 +00:00
|
|
|
|
{
|
2014-02-06 16:54:41 +00:00
|
|
|
|
int from = id * n;
|
|
|
|
|
int start_pos = 0;
|
|
|
|
|
re2::StringPiece input(reinterpret_cast<const char*>(&data[0] + from), (id + 1) * n - from);
|
2014-01-27 16:01:53 +00:00
|
|
|
|
|
2014-02-06 16:54:41 +00:00
|
|
|
|
while (start_pos < input.length())
|
2014-01-27 16:01:53 +00:00
|
|
|
|
{
|
2014-02-06 16:54:41 +00:00
|
|
|
|
/// Правда ли, что с этой строкой больше не надо выполнять преобразования.
|
|
|
|
|
bool can_finish_current_string = false;
|
2014-01-27 16:01:53 +00:00
|
|
|
|
|
2014-02-06 16:54:41 +00:00
|
|
|
|
if (searcher.Match(input, start_pos, input.length(), re2::RE2::Anchor::UNANCHORED, matches, capture))
|
|
|
|
|
{
|
2014-07-28 09:49:16 +00:00
|
|
|
|
const auto & match = matches[0];
|
2014-02-06 16:54:41 +00:00
|
|
|
|
size_t char_to_copy = (match.data() - input.data()) - start_pos;
|
2014-01-27 16:01:53 +00:00
|
|
|
|
|
2014-02-06 16:54:41 +00:00
|
|
|
|
/// Копируем данные без изменения
|
|
|
|
|
res_data.resize(res_data.size() + char_to_copy);
|
2016-04-15 00:33:21 +00:00
|
|
|
|
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], input.data() + start_pos, char_to_copy);
|
2014-02-06 16:54:41 +00:00
|
|
|
|
res_offset += char_to_copy;
|
|
|
|
|
start_pos += char_to_copy + match.length();
|
2014-01-27 16:01:53 +00:00
|
|
|
|
|
2014-02-06 16:54:41 +00:00
|
|
|
|
/// Выполняем инструкции подстановки
|
|
|
|
|
for (const auto & it : instructions)
|
2014-01-28 13:02:37 +00:00
|
|
|
|
{
|
2014-02-06 16:54:41 +00:00
|
|
|
|
if (it.first >= 0)
|
|
|
|
|
{
|
|
|
|
|
res_data.resize(res_data.size() + matches[it.first].length());
|
2016-04-15 00:33:21 +00:00
|
|
|
|
memcpySmallAllowReadWriteOverflow15(
|
|
|
|
|
&res_data[res_offset], matches[it.first].data(), matches[it.first].length());
|
2014-02-06 16:54:41 +00:00
|
|
|
|
res_offset += matches[it.first].length();
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
res_data.resize(res_data.size() + it.second.size());
|
2016-04-15 00:33:21 +00:00
|
|
|
|
memcpySmallAllowReadWriteOverflow15(
|
|
|
|
|
&res_data[res_offset], it.second.data(), it.second.size());
|
2014-02-06 16:54:41 +00:00
|
|
|
|
res_offset += it.second.size();
|
|
|
|
|
}
|
2014-01-28 13:02:37 +00:00
|
|
|
|
}
|
2014-02-07 12:26:25 +00:00
|
|
|
|
if (replaceOne || match.length() == 0)
|
2014-02-06 16:54:41 +00:00
|
|
|
|
can_finish_current_string = true;
|
|
|
|
|
} else
|
2014-01-27 16:01:53 +00:00
|
|
|
|
can_finish_current_string = true;
|
|
|
|
|
|
2014-02-06 16:54:41 +00:00
|
|
|
|
/// Если пора, копируем все символы до конца строки
|
|
|
|
|
if (can_finish_current_string)
|
|
|
|
|
{
|
|
|
|
|
res_data.resize(res_data.size() + input.length() - start_pos);
|
2016-04-15 00:33:21 +00:00
|
|
|
|
memcpySmallAllowReadWriteOverflow15(
|
|
|
|
|
&res_data[res_offset], input.data() + start_pos, input.length() - start_pos);
|
2014-02-06 16:54:41 +00:00
|
|
|
|
res_offset += input.length() - start_pos;
|
|
|
|
|
res_offsets[id] = res_offset;
|
|
|
|
|
start_pos = input.length();
|
|
|
|
|
}
|
2014-01-27 16:01:53 +00:00
|
|
|
|
}
|
2014-02-06 16:54:41 +00:00
|
|
|
|
res_data.resize(res_data.size() + 1);
|
|
|
|
|
res_data[res_offset++] = 0;
|
|
|
|
|
res_offsets[id] = res_offset;
|
|
|
|
|
|
2014-01-27 16:01:53 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void constant(const std::string & data, const std::string & needle, const std::string & replacement,
|
|
|
|
|
std::string & res_data)
|
|
|
|
|
{
|
2014-02-06 16:54:41 +00:00
|
|
|
|
RE2 searcher(needle);
|
|
|
|
|
int capture = std::min(searcher.NumberOfCapturingGroups() + 1, 10);
|
|
|
|
|
re2::StringPiece matches[10];
|
2014-01-27 16:01:53 +00:00
|
|
|
|
|
2014-01-28 13:02:37 +00:00
|
|
|
|
Instructions instructions;
|
|
|
|
|
split(replacement, instructions);
|
|
|
|
|
|
|
|
|
|
for (const auto & it : instructions)
|
2014-02-06 16:54:41 +00:00
|
|
|
|
if (it.first >= capture)
|
2014-01-28 13:02:37 +00:00
|
|
|
|
throw Exception("Invalid replace instruction in replacement string. Id: " + toString(it.first) +
|
2014-02-06 17:02:32 +00:00
|
|
|
|
", but regexp has only " + toString(capture - 1) + " subpatterns",
|
2014-01-28 13:02:37 +00:00
|
|
|
|
ErrorCodes::BAD_ARGUMENTS);
|
|
|
|
|
|
2014-02-06 16:54:41 +00:00
|
|
|
|
int start_pos = 0;
|
|
|
|
|
re2::StringPiece input(data);
|
|
|
|
|
res_data = "";
|
|
|
|
|
|
|
|
|
|
while (start_pos < input.length())
|
2014-01-27 16:01:53 +00:00
|
|
|
|
{
|
2014-02-06 16:54:41 +00:00
|
|
|
|
/// Правда ли, что с этой строкой больше не надо выполнять преобразования.
|
|
|
|
|
bool can_finish_current_string = false;
|
|
|
|
|
|
|
|
|
|
if (searcher.Match(input, start_pos, input.length(), re2::RE2::Anchor::UNANCHORED, matches, capture))
|
2014-01-27 16:01:53 +00:00
|
|
|
|
{
|
2014-07-28 09:49:16 +00:00
|
|
|
|
const auto & match = matches[0];
|
2014-02-06 16:54:41 +00:00
|
|
|
|
size_t char_to_copy = (match.data() - input.data()) - start_pos;
|
2014-01-28 13:02:37 +00:00
|
|
|
|
|
2014-02-06 16:54:41 +00:00
|
|
|
|
/// Копируем данные без изменения
|
|
|
|
|
res_data += data.substr(start_pos, char_to_copy);
|
|
|
|
|
start_pos += char_to_copy + match.length();
|
2014-01-28 13:02:37 +00:00
|
|
|
|
|
2014-02-06 16:54:41 +00:00
|
|
|
|
/// Выполняем инструкции подстановки
|
|
|
|
|
for (const auto & it : instructions)
|
|
|
|
|
{
|
|
|
|
|
if (it.first >= 0)
|
|
|
|
|
res_data += matches[it.first].ToString();
|
|
|
|
|
else
|
|
|
|
|
res_data += it.second;
|
|
|
|
|
}
|
|
|
|
|
|
2014-02-07 12:26:25 +00:00
|
|
|
|
if (replaceOne || match.length() == 0)
|
2014-02-06 16:54:41 +00:00
|
|
|
|
can_finish_current_string = true;
|
|
|
|
|
} else
|
|
|
|
|
can_finish_current_string = true;
|
|
|
|
|
|
|
|
|
|
/// Если пора, копируем все символы до конца строки
|
|
|
|
|
if (can_finish_current_string)
|
2014-01-27 16:01:53 +00:00
|
|
|
|
{
|
2014-02-06 16:54:41 +00:00
|
|
|
|
res_data += data.substr(start_pos);
|
|
|
|
|
start_pos = input.length();
|
2014-01-27 16:01:53 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
2014-01-27 13:49:06 +00:00
|
|
|
|
/** Заменить все вхождения подстроки needle на строку replacement. needle и replacement - константы.
|
|
|
|
|
*/
|
|
|
|
|
template <bool replaceOne = false>
|
2014-01-27 16:01:53 +00:00
|
|
|
|
struct ReplaceStringImpl
|
2014-01-27 13:49:06 +00:00
|
|
|
|
{
|
|
|
|
|
static void vector(const ColumnString::Chars_t & data, const ColumnString::Offsets_t & offsets,
|
|
|
|
|
const std::string & needle, const std::string & replacement,
|
|
|
|
|
ColumnString::Chars_t & res_data, ColumnString::Offsets_t & res_offsets)
|
|
|
|
|
{
|
|
|
|
|
const UInt8 * begin = &data[0];
|
|
|
|
|
const UInt8 * pos = begin;
|
|
|
|
|
const UInt8 * end = pos + data.size();
|
|
|
|
|
|
|
|
|
|
ColumnString::Offset_t res_offset = 0;
|
|
|
|
|
res_data.reserve(data.size());
|
|
|
|
|
size_t size = offsets.size();
|
|
|
|
|
res_offsets.resize(size);
|
|
|
|
|
|
|
|
|
|
/// Текущий индекс в массиве строк.
|
|
|
|
|
size_t i = 0;
|
|
|
|
|
|
|
|
|
|
Volnitsky searcher(needle.data(), needle.size(), end - pos);
|
|
|
|
|
|
|
|
|
|
/// Искать будем следующее вхождение сразу во всех строках.
|
|
|
|
|
while (pos < end)
|
|
|
|
|
{
|
|
|
|
|
const UInt8 * match = searcher.search(pos, end - pos);
|
|
|
|
|
|
|
|
|
|
/// Копируем данные без изменения
|
|
|
|
|
res_data.resize(res_data.size() + (match - pos));
|
|
|
|
|
memcpy(&res_data[res_offset], pos, match - pos);
|
|
|
|
|
|
|
|
|
|
/// Определим, к какому индексу оно относится.
|
2015-10-11 04:25:02 +00:00
|
|
|
|
while (i < offsets.size() && begin + offsets[i] <= match)
|
2014-01-27 13:49:06 +00:00
|
|
|
|
{
|
|
|
|
|
res_offsets[i] = res_offset + ((begin + offsets[i]) - pos);
|
|
|
|
|
++i;
|
|
|
|
|
}
|
|
|
|
|
res_offset += (match - pos);
|
|
|
|
|
|
|
|
|
|
/// Если дошли до конца, пора остановиться
|
|
|
|
|
if (i == offsets.size())
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
/// Правда ли, что с этой строкой больше не надо выполнять преобразования.
|
|
|
|
|
bool can_finish_current_string = false;
|
|
|
|
|
|
|
|
|
|
/// Проверяем, что вхождение не переходит через границы строк.
|
|
|
|
|
if (match + needle.size() < begin + offsets[i])
|
|
|
|
|
{
|
|
|
|
|
res_data.resize(res_data.size() + replacement.size());
|
|
|
|
|
memcpy(&res_data[res_offset], replacement.data(), replacement.size());
|
|
|
|
|
res_offset += replacement.size();
|
|
|
|
|
pos = match + needle.size();
|
|
|
|
|
if (replaceOne)
|
|
|
|
|
can_finish_current_string = true;
|
|
|
|
|
}
|
|
|
|
|
else
|
2014-01-27 16:01:53 +00:00
|
|
|
|
{
|
|
|
|
|
pos = match;
|
2014-01-27 13:49:06 +00:00
|
|
|
|
can_finish_current_string = true;
|
2014-01-27 16:01:53 +00:00
|
|
|
|
}
|
2014-01-27 13:49:06 +00:00
|
|
|
|
|
|
|
|
|
if (can_finish_current_string)
|
|
|
|
|
{
|
2014-01-27 16:01:53 +00:00
|
|
|
|
res_data.resize(res_data.size() + (begin + offsets[i] - pos));
|
|
|
|
|
memcpy(&res_data[res_offset], pos, (begin + offsets[i] - pos));
|
|
|
|
|
res_offset += (begin + offsets[i] - pos);
|
|
|
|
|
res_offsets[i] = res_offset;
|
2014-01-27 13:49:06 +00:00
|
|
|
|
pos = begin + offsets[i];
|
2015-10-11 04:25:02 +00:00
|
|
|
|
++i;
|
2014-01-27 13:49:06 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void vector_fixed(const ColumnString::Chars_t & data, size_t n,
|
|
|
|
|
const std::string & needle, const std::string & replacement,
|
|
|
|
|
ColumnString::Chars_t & res_data, ColumnString::Offsets_t & res_offsets)
|
|
|
|
|
{
|
|
|
|
|
const UInt8 * begin = &data[0];
|
|
|
|
|
const UInt8 * pos = begin;
|
|
|
|
|
const UInt8 * end = pos + data.size();
|
|
|
|
|
|
|
|
|
|
ColumnString::Offset_t res_offset = 0;
|
|
|
|
|
size_t size = data.size() / n;
|
|
|
|
|
res_data.reserve(data.size());
|
|
|
|
|
res_offsets.resize(size);
|
|
|
|
|
|
|
|
|
|
/// Текущий индекс в массиве строк.
|
|
|
|
|
size_t i = 0;
|
|
|
|
|
|
|
|
|
|
Volnitsky searcher(needle.data(), needle.size(), end - pos);
|
|
|
|
|
|
|
|
|
|
/// Искать будем следующее вхождение сразу во всех строках.
|
|
|
|
|
while (pos < end)
|
|
|
|
|
{
|
|
|
|
|
const UInt8 * match = searcher.search(pos, end - pos);
|
|
|
|
|
|
|
|
|
|
/// Копируем данные без изменения
|
|
|
|
|
res_data.resize(res_data.size() + (match - pos));
|
|
|
|
|
memcpy(&res_data[res_offset], pos, match - pos);
|
|
|
|
|
|
|
|
|
|
/// Определим, к какому индексу оно относится.
|
2015-10-11 04:25:02 +00:00
|
|
|
|
while (i < size && begin + n * (i + 1) <= match)
|
2014-01-27 13:49:06 +00:00
|
|
|
|
{
|
|
|
|
|
res_offsets[i] = res_offset + ((begin + n * (i + 1)) - pos);
|
|
|
|
|
++i;
|
|
|
|
|
}
|
|
|
|
|
res_offset += (match - pos);
|
|
|
|
|
|
|
|
|
|
/// Если дошли до конца, пора остановиться
|
|
|
|
|
if (i == size)
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
/// Правда ли, что с этой строкой больше не надо выполнять преобразования.
|
|
|
|
|
bool can_finish_current_string = false;
|
|
|
|
|
|
|
|
|
|
/// Проверяем, что вхождение не переходит через границы строк.
|
|
|
|
|
if (match + needle.size() < begin + n * (i + 1))
|
|
|
|
|
{
|
|
|
|
|
res_data.resize(res_data.size() + replacement.size());
|
|
|
|
|
memcpy(&res_data[res_offset], replacement.data(), replacement.size());
|
|
|
|
|
res_offset += replacement.size();
|
|
|
|
|
pos = match + needle.size();
|
|
|
|
|
if (replaceOne)
|
|
|
|
|
can_finish_current_string = true;
|
|
|
|
|
}
|
|
|
|
|
else
|
2014-01-27 16:01:53 +00:00
|
|
|
|
{
|
|
|
|
|
pos = match;
|
2014-01-27 13:49:06 +00:00
|
|
|
|
can_finish_current_string = true;
|
2014-01-27 16:01:53 +00:00
|
|
|
|
}
|
2014-01-27 13:49:06 +00:00
|
|
|
|
|
|
|
|
|
if (can_finish_current_string)
|
|
|
|
|
{
|
2014-01-27 16:01:53 +00:00
|
|
|
|
res_data.resize(res_data.size() + (begin + n * (i + 1) - pos));
|
|
|
|
|
memcpy(&res_data[res_offset], pos, (begin + n * (i + 1) - pos));
|
|
|
|
|
res_offset += (begin + n * (i + 1) - pos);
|
|
|
|
|
res_offsets[i] = res_offset;
|
2014-01-27 13:49:06 +00:00
|
|
|
|
pos = begin + n * (i + 1);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void constant(const std::string & data, const std::string & needle, const std::string & replacement,
|
|
|
|
|
std::string & res_data)
|
|
|
|
|
{
|
|
|
|
|
res_data = "";
|
|
|
|
|
int replace_cnt = 0;
|
|
|
|
|
for (size_t i = 0; i < data.size(); ++i)
|
|
|
|
|
{
|
|
|
|
|
bool match = true;
|
|
|
|
|
if (i + needle.size() > data.size() || (replaceOne && replace_cnt > 0))
|
|
|
|
|
match = false;
|
|
|
|
|
for (size_t j = 0; match && j < needle.size(); ++j)
|
|
|
|
|
if (data[i + j] != needle[j])
|
|
|
|
|
match = false;
|
|
|
|
|
if (match)
|
|
|
|
|
{
|
2014-03-27 19:09:23 +00:00
|
|
|
|
++replace_cnt;
|
2014-01-27 13:49:06 +00:00
|
|
|
|
res_data += replacement;
|
|
|
|
|
i = i + needle.size() - 1;
|
|
|
|
|
} else
|
|
|
|
|
res_data += data[i];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
template <typename Impl, typename Name>
|
|
|
|
|
class FunctionStringReplace : public IFunction
|
|
|
|
|
{
|
|
|
|
|
public:
|
2014-11-12 17:23:26 +00:00
|
|
|
|
static constexpr auto name = Name::name;
|
2016-05-28 15:42:22 +00:00
|
|
|
|
static FunctionPtr create(const Context & context) { return std::make_shared<FunctionStringReplace>(); }
|
2014-11-12 17:23:26 +00:00
|
|
|
|
|
2014-01-27 13:49:06 +00:00
|
|
|
|
/// Получить имя функции.
|
2015-10-11 23:36:45 +00:00
|
|
|
|
String getName() const override
|
2014-01-27 13:49:06 +00:00
|
|
|
|
{
|
2014-11-12 17:23:26 +00:00
|
|
|
|
return name;
|
2014-01-27 13:49:06 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Получить тип результата по типам аргументов. Если функция неприменима для данных аргументов - кинуть исключение.
|
2015-10-11 23:36:45 +00:00
|
|
|
|
DataTypePtr getReturnType(const DataTypes & arguments) const override
|
2014-01-27 13:49:06 +00:00
|
|
|
|
{
|
|
|
|
|
if (arguments.size() != 3)
|
|
|
|
|
throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
|
|
|
|
|
+ toString(arguments.size()) + ", should be 3.",
|
|
|
|
|
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
|
|
|
|
|
2014-06-26 00:58:14 +00:00
|
|
|
|
if (!typeid_cast<const DataTypeString *>(&*arguments[0]) && !typeid_cast<const DataTypeFixedString *>(&*arguments[0]))
|
2014-01-27 13:49:06 +00:00
|
|
|
|
throw Exception("Illegal type " + arguments[0]->getName() + " of first argument of function " + getName(),
|
|
|
|
|
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
if (!typeid_cast<const DataTypeString *>(&*arguments[1]) && !typeid_cast<const DataTypeFixedString *>(&*arguments[1]))
|
2014-01-27 13:49:06 +00:00
|
|
|
|
throw Exception("Illegal type " + arguments[1]->getName() + " of second argument of function " + getName(),
|
|
|
|
|
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
if (!typeid_cast<const DataTypeString *>(&*arguments[2]) && !typeid_cast<const DataTypeFixedString *>(&*arguments[2]))
|
2014-01-27 13:49:06 +00:00
|
|
|
|
throw Exception("Illegal type " + arguments[2]->getName() + " of third argument of function " + getName(),
|
|
|
|
|
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
|
|
|
|
|
2016-05-28 07:48:40 +00:00
|
|
|
|
return std::make_shared<DataTypeString>();
|
2014-01-27 13:49:06 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Выполнить функцию над блоком.
|
2015-10-11 23:36:45 +00:00
|
|
|
|
void execute(Block & block, const ColumnNumbers & arguments, size_t result) override
|
2014-01-27 13:49:06 +00:00
|
|
|
|
{
|
|
|
|
|
const ColumnPtr column_src = block.getByPosition(arguments[0]).column;
|
|
|
|
|
const ColumnPtr column_needle = block.getByPosition(arguments[1]).column;
|
|
|
|
|
const ColumnPtr column_replacement = block.getByPosition(arguments[2]).column;
|
|
|
|
|
|
|
|
|
|
if (!column_needle->isConst() || !column_replacement->isConst())
|
|
|
|
|
throw Exception("2nd and 3rd arguments of function " + getName() + " must be constants.");
|
|
|
|
|
|
2016-07-10 02:33:50 +00:00
|
|
|
|
const IColumn * c1 = block.getByPosition(arguments[1]).column.get();
|
|
|
|
|
const IColumn * c2 = block.getByPosition(arguments[2]).column.get();
|
2014-06-26 00:58:14 +00:00
|
|
|
|
const ColumnConstString * c1_const = typeid_cast<const ColumnConstString *>(c1);
|
|
|
|
|
const ColumnConstString * c2_const = typeid_cast<const ColumnConstString *>(c2);
|
2014-01-27 13:49:06 +00:00
|
|
|
|
String needle = c1_const->getData();
|
|
|
|
|
String replacement = c2_const->getData();
|
|
|
|
|
|
|
|
|
|
if (needle.size() == 0)
|
|
|
|
|
throw Exception("Length of the second argument of function replace must be greater than 0.", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
|
|
|
|
|
2014-06-26 00:58:14 +00:00
|
|
|
|
if (const ColumnString * col = typeid_cast<const ColumnString *>(&*column_src))
|
2014-01-27 13:49:06 +00:00
|
|
|
|
{
|
2016-05-28 05:31:36 +00:00
|
|
|
|
std::shared_ptr<ColumnString> col_res = std::make_shared<ColumnString>();
|
2014-01-27 13:49:06 +00:00
|
|
|
|
block.getByPosition(result).column = col_res;
|
|
|
|
|
Impl::vector(col->getChars(), col->getOffsets(),
|
|
|
|
|
needle, replacement,
|
|
|
|
|
col_res->getChars(), col_res->getOffsets());
|
|
|
|
|
}
|
2014-06-26 00:58:14 +00:00
|
|
|
|
else if (const ColumnFixedString * col = typeid_cast<const ColumnFixedString *>(&*column_src))
|
2014-01-27 13:49:06 +00:00
|
|
|
|
{
|
2016-05-28 05:31:36 +00:00
|
|
|
|
std::shared_ptr<ColumnString> col_res = std::make_shared<ColumnString>();
|
2014-01-27 13:49:06 +00:00
|
|
|
|
block.getByPosition(result).column = col_res;
|
|
|
|
|
Impl::vector_fixed(col->getChars(), col->getN(),
|
|
|
|
|
needle, replacement,
|
|
|
|
|
col_res->getChars(), col_res->getOffsets());
|
|
|
|
|
}
|
2014-06-26 00:58:14 +00:00
|
|
|
|
else if (const ColumnConstString * col = typeid_cast<const ColumnConstString *>(&*column_src))
|
2014-01-27 13:49:06 +00:00
|
|
|
|
{
|
|
|
|
|
String res;
|
|
|
|
|
Impl::constant(col->getData(), needle, replacement, res);
|
2016-05-28 05:31:36 +00:00
|
|
|
|
auto col_res = std::make_shared<ColumnConstString>(col->size(), res);
|
2014-01-27 13:49:06 +00:00
|
|
|
|
block.getByPosition(result).column = col_res;
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName()
|
|
|
|
|
+ " of first argument of function " + getName(),
|
|
|
|
|
ErrorCodes::ILLEGAL_COLUMN);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
2011-10-17 08:28:39 +00:00
|
|
|
|
template <typename Impl, typename Name>
|
|
|
|
|
class FunctionsStringSearch : public IFunction
|
2011-10-16 07:11:36 +00:00
|
|
|
|
{
|
|
|
|
|
public:
|
2014-11-12 17:23:26 +00:00
|
|
|
|
static constexpr auto name = Name::name;
|
2016-05-28 15:42:22 +00:00
|
|
|
|
static FunctionPtr create(const Context & context) { return std::make_shared<FunctionsStringSearch>(); }
|
2014-11-12 17:23:26 +00:00
|
|
|
|
|
2011-10-16 07:11:36 +00:00
|
|
|
|
/// Получить имя функции.
|
2015-10-11 23:36:45 +00:00
|
|
|
|
String getName() const override
|
2011-10-16 07:11:36 +00:00
|
|
|
|
{
|
2014-11-12 17:23:26 +00:00
|
|
|
|
return name;
|
2011-10-16 07:11:36 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Получить тип результата по типам аргументов. Если функция неприменима для данных аргументов - кинуть исключение.
|
2015-10-11 23:36:45 +00:00
|
|
|
|
DataTypePtr getReturnType(const DataTypes & arguments) const override
|
2011-10-16 07:11:36 +00:00
|
|
|
|
{
|
|
|
|
|
if (arguments.size() != 2)
|
|
|
|
|
throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
|
2013-06-21 20:34:19 +00:00
|
|
|
|
+ toString(arguments.size()) + ", should be 2.",
|
2011-10-16 07:11:36 +00:00
|
|
|
|
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
|
|
|
|
|
2014-06-26 00:58:14 +00:00
|
|
|
|
if (!typeid_cast<const DataTypeString *>(&*arguments[0]))
|
2011-10-16 07:11:36 +00:00
|
|
|
|
throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(),
|
|
|
|
|
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
|
|
|
|
|
2014-06-26 00:58:14 +00:00
|
|
|
|
if (!typeid_cast<const DataTypeString *>(&*arguments[1]))
|
2011-10-16 07:11:36 +00:00
|
|
|
|
throw Exception("Illegal type " + arguments[1]->getName() + " of argument of function " + getName(),
|
|
|
|
|
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
|
|
|
|
|
2016-05-28 07:48:40 +00:00
|
|
|
|
return std::make_shared<typename DataTypeFromFieldType<typename Impl::ResultType>::Type>();
|
2011-10-16 07:11:36 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Выполнить функцию над блоком.
|
2015-10-11 23:36:45 +00:00
|
|
|
|
void execute(Block & block, const ColumnNumbers & arguments, size_t result) override
|
2011-10-16 07:11:36 +00:00
|
|
|
|
{
|
2016-05-28 10:35:44 +00:00
|
|
|
|
using ResultType = typename Impl::ResultType;
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
const ColumnPtr & column_haystack = block.getByPosition(arguments[0]).column;
|
|
|
|
|
const ColumnPtr & column_needle = block.getByPosition(arguments[1]).column;
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
const ColumnConstString * col_haystack_const = typeid_cast<const ColumnConstString *>(&*column_haystack);
|
|
|
|
|
const ColumnConstString * col_needle_const = typeid_cast<const ColumnConstString *>(&*column_needle);
|
2011-10-16 07:11:36 +00:00
|
|
|
|
|
2016-01-27 03:11:28 +00:00
|
|
|
|
if (col_haystack_const && col_needle_const)
|
2011-10-16 07:11:36 +00:00
|
|
|
|
{
|
2015-09-22 13:43:32 +00:00
|
|
|
|
ResultType res{};
|
2016-01-27 03:11:28 +00:00
|
|
|
|
Impl::constant_constant(col_haystack_const->getData(), col_needle_const->getData(), res);
|
2016-05-28 05:31:36 +00:00
|
|
|
|
block.getByPosition(result).column = std::make_shared<ColumnConst<ResultType>>(col_haystack_const->size(), res);
|
2016-01-27 03:11:28 +00:00
|
|
|
|
return;
|
2011-10-16 07:11:36 +00:00
|
|
|
|
}
|
2016-01-27 03:11:28 +00:00
|
|
|
|
|
2016-05-28 05:31:36 +00:00
|
|
|
|
auto col_res = std::make_shared<ColumnVector<ResultType>>();
|
2016-01-27 03:11:28 +00:00
|
|
|
|
block.getByPosition(result).column = col_res;
|
|
|
|
|
|
|
|
|
|
typename ColumnVector<ResultType>::Container_t & vec_res = col_res->getData();
|
|
|
|
|
vec_res.resize(column_haystack->size());
|
|
|
|
|
|
|
|
|
|
const ColumnString * col_haystack_vector = typeid_cast<const ColumnString *>(&*column_haystack);
|
|
|
|
|
const ColumnString * col_needle_vector = typeid_cast<const ColumnString *>(&*column_needle);
|
|
|
|
|
|
|
|
|
|
if (col_haystack_vector && col_needle_vector)
|
|
|
|
|
Impl::vector_vector(
|
|
|
|
|
col_haystack_vector->getChars(), col_haystack_vector->getOffsets(),
|
|
|
|
|
col_needle_vector->getChars(), col_needle_vector->getOffsets(),
|
|
|
|
|
vec_res);
|
|
|
|
|
else if (col_haystack_vector && col_needle_const)
|
|
|
|
|
Impl::vector_constant(
|
|
|
|
|
col_haystack_vector->getChars(), col_haystack_vector->getOffsets(),
|
|
|
|
|
col_needle_const->getData(),
|
|
|
|
|
vec_res);
|
|
|
|
|
else if (col_haystack_const && col_needle_vector)
|
|
|
|
|
Impl::constant_vector(
|
|
|
|
|
col_haystack_const->getData(),
|
|
|
|
|
col_needle_vector->getChars(), col_needle_vector->getOffsets(),
|
|
|
|
|
vec_res);
|
2011-10-16 07:11:36 +00:00
|
|
|
|
else
|
2016-01-27 03:11:28 +00:00
|
|
|
|
throw Exception("Illegal columns "
|
|
|
|
|
+ block.getByPosition(arguments[0]).column->getName()
|
|
|
|
|
+ " and " + block.getByPosition(arguments[1]).column->getName()
|
|
|
|
|
+ " of arguments of function " + getName(),
|
2011-10-16 07:11:36 +00:00
|
|
|
|
ErrorCodes::ILLEGAL_COLUMN);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
2011-10-17 08:28:39 +00:00
|
|
|
|
|
2013-02-28 13:01:07 +00:00
|
|
|
|
template <typename Impl, typename Name>
|
|
|
|
|
class FunctionsStringSearchToString : public IFunction
|
|
|
|
|
{
|
|
|
|
|
public:
|
2014-11-12 17:23:26 +00:00
|
|
|
|
static constexpr auto name = Name::name;
|
2016-05-28 15:42:22 +00:00
|
|
|
|
static FunctionPtr create(const Context & context) { return std::make_shared<FunctionsStringSearchToString>(); }
|
2014-11-12 17:23:26 +00:00
|
|
|
|
|
2013-02-28 13:01:07 +00:00
|
|
|
|
/// Получить имя функции.
|
2015-10-11 23:36:45 +00:00
|
|
|
|
String getName() const override
|
2013-02-28 13:01:07 +00:00
|
|
|
|
{
|
2014-11-12 17:23:26 +00:00
|
|
|
|
return name;
|
2013-02-28 13:01:07 +00:00
|
|
|
|
}
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-02-28 13:01:07 +00:00
|
|
|
|
/// Получить тип результата по типам аргументов. Если функция неприменима для данных аргументов - кинуть исключение.
|
2015-10-11 23:36:45 +00:00
|
|
|
|
DataTypePtr getReturnType(const DataTypes & arguments) const override
|
2013-02-28 13:01:07 +00:00
|
|
|
|
{
|
|
|
|
|
if (arguments.size() != 2)
|
|
|
|
|
throw Exception("Number of arguments for function " + getName() + " doesn't match: passed "
|
2013-06-21 20:34:19 +00:00
|
|
|
|
+ toString(arguments.size()) + ", should be 2.",
|
2013-02-28 13:01:07 +00:00
|
|
|
|
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
|
|
|
|
if (!typeid_cast<const DataTypeString *>(&*arguments[0]))
|
2013-03-04 10:38:44 +00:00
|
|
|
|
throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(),
|
|
|
|
|
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
|
|
|
|
if (!typeid_cast<const DataTypeString *>(&*arguments[1]))
|
2013-03-04 10:38:44 +00:00
|
|
|
|
throw Exception("Illegal type " + arguments[1]->getName() + " of argument of function " + getName(),
|
|
|
|
|
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2016-05-28 07:48:40 +00:00
|
|
|
|
return std::make_shared<DataTypeString>();
|
2013-02-28 13:01:07 +00:00
|
|
|
|
}
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-02-28 13:01:07 +00:00
|
|
|
|
/// Выполнить функцию над блоком.
|
2015-10-11 23:36:45 +00:00
|
|
|
|
void execute(Block & block, const ColumnNumbers & arguments, size_t result) override
|
2013-02-28 13:01:07 +00:00
|
|
|
|
{
|
|
|
|
|
const ColumnPtr column = block.getByPosition(arguments[0]).column;
|
|
|
|
|
const ColumnPtr column_needle = block.getByPosition(arguments[1]).column;
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
|
|
|
|
const ColumnConstString * col_needle = typeid_cast<const ColumnConstString *>(&*column_needle);
|
2013-02-28 13:01:07 +00:00
|
|
|
|
if (!col_needle)
|
|
|
|
|
throw Exception("Second argument of function " + getName() + " must be constant string.", ErrorCodes::ILLEGAL_COLUMN);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
|
|
|
|
if (const ColumnString * col = typeid_cast<const ColumnString *>(&*column))
|
2013-02-28 13:01:07 +00:00
|
|
|
|
{
|
2016-05-28 05:31:36 +00:00
|
|
|
|
std::shared_ptr<ColumnString> col_res = std::make_shared<ColumnString>();
|
2013-02-28 13:01:07 +00:00
|
|
|
|
block.getByPosition(result).column = col_res;
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-05-05 15:25:25 +00:00
|
|
|
|
ColumnString::Chars_t & vec_res = col_res->getChars();
|
2013-02-28 13:01:07 +00:00
|
|
|
|
ColumnString::Offsets_t & offsets_res = col_res->getOffsets();
|
2013-05-05 15:25:25 +00:00
|
|
|
|
Impl::vector(col->getChars(), col->getOffsets(), col_needle->getData(), vec_res, offsets_res);
|
2013-02-28 13:01:07 +00:00
|
|
|
|
}
|
2014-06-26 00:58:14 +00:00
|
|
|
|
else if (const ColumnConstString * col = typeid_cast<const ColumnConstString *>(&*column))
|
2013-02-28 13:01:07 +00:00
|
|
|
|
{
|
2013-03-05 12:12:47 +00:00
|
|
|
|
const std::string & data = col->getData();
|
2013-09-15 05:51:43 +00:00
|
|
|
|
ColumnString::Chars_t vdata(
|
|
|
|
|
reinterpret_cast<const ColumnString::Chars_t::value_type *>(data.c_str()),
|
|
|
|
|
reinterpret_cast<const ColumnString::Chars_t::value_type *>(data.c_str() + data.size() + 1));
|
2013-05-05 15:25:25 +00:00
|
|
|
|
ColumnString::Offsets_t offsets(1, vdata.size());
|
2013-09-15 05:51:43 +00:00
|
|
|
|
ColumnString::Chars_t res_vdata;
|
2013-05-05 15:25:25 +00:00
|
|
|
|
ColumnString::Offsets_t res_offsets;
|
2013-03-05 12:12:47 +00:00
|
|
|
|
Impl::vector(vdata, offsets, col_needle->getData(), res_vdata, res_offsets);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2013-10-03 19:23:08 +00:00
|
|
|
|
std::string res;
|
|
|
|
|
|
|
|
|
|
if (!res_offsets.empty())
|
|
|
|
|
res.assign(&res_vdata[0], &res_vdata[res_vdata.size() - 1]);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
|
2016-05-28 05:31:36 +00:00
|
|
|
|
block.getByPosition(result).column = std::make_shared<ColumnConstString>(col->size(), res);
|
2013-02-28 13:01:07 +00:00
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName()
|
|
|
|
|
+ " of argument of function " + getName(),
|
|
|
|
|
ErrorCodes::ILLEGAL_COLUMN);
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
2015-09-22 13:43:32 +00:00
|
|
|
|
struct NamePosition { static constexpr auto name = "position"; };
|
|
|
|
|
struct NamePositionUTF8 { static constexpr auto name = "positionUTF8"; };
|
|
|
|
|
struct NamePositionCaseInsensitive { static constexpr auto name = "positionCaseInsensitive"; };
|
2015-09-23 16:49:10 +00:00
|
|
|
|
struct NamePositionCaseInsensitiveUTF8 { static constexpr auto name = "positionCaseInsensitiveUTF8"; };
|
2015-09-22 13:43:32 +00:00
|
|
|
|
struct NameMatch { static constexpr auto name = "match"; };
|
|
|
|
|
struct NameLike { static constexpr auto name = "like"; };
|
|
|
|
|
struct NameNotLike { static constexpr auto name = "notLike"; };
|
|
|
|
|
struct NameExtract { static constexpr auto name = "extract"; };
|
|
|
|
|
struct NameReplaceOne { static constexpr auto name = "replaceOne"; };
|
|
|
|
|
struct NameReplaceAll { static constexpr auto name = "replaceAll"; };
|
|
|
|
|
struct NameReplaceRegexpOne { static constexpr auto name = "replaceRegexpOne"; };
|
|
|
|
|
struct NameReplaceRegexpAll { static constexpr auto name = "replaceRegexpAll"; };
|
|
|
|
|
|
2016-05-28 10:35:44 +00:00
|
|
|
|
using FunctionPosition = FunctionsStringSearch<PositionImpl<PositionCaseSensitiveASCII>, NamePosition> ;
|
|
|
|
|
using FunctionPositionUTF8 = FunctionsStringSearch<PositionImpl<PositionCaseSensitiveUTF8>, NamePositionUTF8> ;
|
|
|
|
|
using FunctionPositionCaseInsensitive = FunctionsStringSearch<PositionImpl<PositionCaseInsensitiveASCII>, NamePositionCaseInsensitive> ;
|
|
|
|
|
using FunctionPositionCaseInsensitiveUTF8 = FunctionsStringSearch<PositionImpl<PositionCaseInsensitiveUTF8>, NamePositionCaseInsensitiveUTF8>;
|
|
|
|
|
|
|
|
|
|
using FunctionMatch = FunctionsStringSearch<MatchImpl<false>, NameMatch> ;
|
|
|
|
|
using FunctionLike = FunctionsStringSearch<MatchImpl<true>, NameLike> ;
|
|
|
|
|
using FunctionNotLike = FunctionsStringSearch<MatchImpl<true, true>, NameNotLike> ;
|
|
|
|
|
using FunctionExtract = FunctionsStringSearchToString<ExtractImpl, NameExtract> ;
|
|
|
|
|
using FunctionReplaceOne = FunctionStringReplace<ReplaceStringImpl<true>, NameReplaceOne> ;
|
|
|
|
|
using FunctionReplaceAll = FunctionStringReplace<ReplaceStringImpl<false>, NameReplaceAll> ;
|
|
|
|
|
using FunctionReplaceRegexpOne = FunctionStringReplace<ReplaceRegexpImpl<true>, NameReplaceRegexpOne> ;
|
|
|
|
|
using FunctionReplaceRegexpAll = FunctionStringReplace<ReplaceRegexpImpl<false>, NameReplaceRegexpAll> ;
|
2011-10-17 08:28:39 +00:00
|
|
|
|
|
2011-10-16 07:11:36 +00:00
|
|
|
|
}
|