2020-05-06 23:21:13 +00:00
|
|
|
#pragma once
|
|
|
|
|
2021-10-02 07:13:14 +00:00
|
|
|
#include <base/types.h>
|
2020-05-06 23:21:13 +00:00
|
|
|
#include <Common/Volnitsky.h>
|
|
|
|
#include <Columns/ColumnString.h>
|
|
|
|
#include <IO/WriteHelpers.h>
|
|
|
|
|
2021-10-27 23:10:39 +00:00
|
|
|
#include "config_functions.h"
|
|
|
|
#include <Common/config.h>
|
2020-05-06 23:21:13 +00:00
|
|
|
|
|
|
|
#if USE_RE2_ST
|
|
|
|
# include <re2_st/re2.h>
|
|
|
|
#else
|
|
|
|
# include <re2/re2.h>
|
|
|
|
# define re2_st re2
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
|
|
|
extern const int BAD_ARGUMENTS;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/** Replace all matches of regexp 'needle' to string 'replacement'. 'needle' and 'replacement' are constants.
|
|
|
|
* 'replacement' could contain substitutions, for example: '\2-\3-\1'
|
|
|
|
*/
|
|
|
|
template <bool replace_one = false>
|
|
|
|
struct ReplaceRegexpImpl
|
|
|
|
{
|
|
|
|
/// Sequence of instructions, describing how to get resulting string.
|
2021-12-19 09:44:42 +00:00
|
|
|
struct Instruction
|
|
|
|
{
|
|
|
|
/// If non zero - perform substitution of n-th subpattern from the regexp match.
|
|
|
|
int substitution_num = -1;
|
|
|
|
/// Otherwise - paste this string verbatim.
|
|
|
|
std::string literal;
|
|
|
|
|
|
|
|
Instruction(int substitution_num_) : substitution_num(substitution_num_) {}
|
|
|
|
Instruction(std::string literal_) : literal(std::move(literal_)) {}
|
|
|
|
};
|
|
|
|
|
|
|
|
using Instructions = std::vector<Instruction>;
|
2020-05-06 23:21:13 +00:00
|
|
|
|
|
|
|
static const size_t max_captures = 10;
|
|
|
|
|
|
|
|
|
|
|
|
static Instructions createInstructions(const std::string & s, int num_captures)
|
|
|
|
{
|
|
|
|
Instructions instructions;
|
|
|
|
|
|
|
|
String now;
|
|
|
|
for (size_t i = 0; i < s.size(); ++i)
|
|
|
|
{
|
|
|
|
if (s[i] == '\\' && i + 1 < s.size())
|
|
|
|
{
|
|
|
|
if (isNumericASCII(s[i + 1])) /// Substitution
|
|
|
|
{
|
|
|
|
if (!now.empty())
|
|
|
|
{
|
2021-12-19 09:44:42 +00:00
|
|
|
instructions.emplace_back(now);
|
2020-05-06 23:21:13 +00:00
|
|
|
now = "";
|
|
|
|
}
|
2021-12-19 09:44:42 +00:00
|
|
|
instructions.emplace_back(s[i + 1] - '0');
|
2020-05-06 23:21:13 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
now += s[i + 1]; /// Escaping
|
|
|
|
++i;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
now += s[i]; /// Plain character
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!now.empty())
|
|
|
|
{
|
2021-12-19 09:44:42 +00:00
|
|
|
instructions.emplace_back(now);
|
2020-05-06 23:21:13 +00:00
|
|
|
now = "";
|
|
|
|
}
|
|
|
|
|
|
|
|
for (const auto & it : instructions)
|
2021-12-19 09:44:42 +00:00
|
|
|
if (it.substitution_num >= num_captures)
|
|
|
|
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
|
|
|
"Invalid replace instruction in replacement string. Id: {}, but regexp has only {} subpatterns",
|
|
|
|
it.substitution_num, num_captures - 1);
|
2020-05-06 23:21:13 +00:00
|
|
|
|
|
|
|
return instructions;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static void processString(
|
|
|
|
const re2_st::StringPiece & input,
|
|
|
|
ColumnString::Chars & res_data,
|
|
|
|
ColumnString::Offset & res_offset,
|
|
|
|
re2_st::RE2 & searcher,
|
|
|
|
int num_captures,
|
|
|
|
const Instructions & instructions)
|
|
|
|
{
|
|
|
|
re2_st::StringPiece matches[max_captures];
|
|
|
|
|
2021-12-19 09:44:42 +00:00
|
|
|
size_t copy_pos = 0;
|
|
|
|
size_t match_pos = 0;
|
|
|
|
|
|
|
|
while (match_pos < static_cast<size_t>(input.length()))
|
2020-05-06 23:21:13 +00:00
|
|
|
{
|
|
|
|
/// If no more replacements possible for current string
|
|
|
|
bool can_finish_current_string = false;
|
|
|
|
|
2021-12-19 09:44:42 +00:00
|
|
|
if (searcher.Match(input, match_pos, input.length(), re2_st::RE2::Anchor::UNANCHORED, matches, num_captures))
|
2020-05-06 23:21:13 +00:00
|
|
|
{
|
|
|
|
const auto & match = matches[0];
|
2021-12-19 09:44:42 +00:00
|
|
|
size_t bytes_to_copy = (match.data() - input.data()) - copy_pos;
|
2020-05-06 23:21:13 +00:00
|
|
|
|
|
|
|
/// Copy prefix before matched regexp without modification
|
|
|
|
res_data.resize(res_data.size() + bytes_to_copy);
|
2021-12-19 09:44:42 +00:00
|
|
|
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], input.data() + copy_pos, bytes_to_copy);
|
2020-05-06 23:21:13 +00:00
|
|
|
res_offset += bytes_to_copy;
|
2021-12-19 09:44:42 +00:00
|
|
|
copy_pos += bytes_to_copy + match.length();
|
|
|
|
match_pos = copy_pos;
|
2020-05-06 23:21:13 +00:00
|
|
|
|
|
|
|
/// Do substitution instructions
|
|
|
|
for (const auto & it : instructions)
|
|
|
|
{
|
2021-12-19 09:44:42 +00:00
|
|
|
if (it.substitution_num >= 0)
|
2020-05-06 23:21:13 +00:00
|
|
|
{
|
2021-12-19 09:44:42 +00:00
|
|
|
const auto & substitution = matches[it.substitution_num];
|
|
|
|
|
|
|
|
res_data.resize(res_data.size() + substitution.length());
|
|
|
|
memcpy(&res_data[res_offset], substitution.data(), substitution.length());
|
|
|
|
res_offset += substitution.length();
|
2020-05-06 23:21:13 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2021-12-19 09:44:42 +00:00
|
|
|
const auto & literal = it.literal;
|
|
|
|
|
|
|
|
res_data.resize(res_data.size() + literal.size());
|
|
|
|
memcpy(&res_data[res_offset], literal.data(), literal.size());
|
|
|
|
res_offset += literal.size();
|
2020-05-06 23:21:13 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-12-19 09:44:42 +00:00
|
|
|
if (replace_one)
|
2020-05-06 23:21:13 +00:00
|
|
|
can_finish_current_string = true;
|
2021-12-19 09:44:42 +00:00
|
|
|
else if (match.length() == 0)
|
|
|
|
++match_pos; /// Step one character to avoid infinite loop.
|
2020-05-06 23:21:13 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
can_finish_current_string = true;
|
|
|
|
|
|
|
|
/// If ready, append suffix after match to end of string.
|
|
|
|
if (can_finish_current_string)
|
|
|
|
{
|
2021-12-19 09:44:42 +00:00
|
|
|
res_data.resize(res_data.size() + input.length() - copy_pos);
|
|
|
|
memcpySmallAllowReadWriteOverflow15(&res_data[res_offset], input.data() + copy_pos, input.length() - copy_pos);
|
|
|
|
res_offset += input.length() - copy_pos;
|
|
|
|
copy_pos = input.length();
|
|
|
|
match_pos = copy_pos;
|
2020-05-06 23:21:13 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
res_data.resize(res_data.size() + 1);
|
|
|
|
res_data[res_offset] = 0;
|
|
|
|
++res_offset;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static void vector(
|
|
|
|
const ColumnString::Chars & data,
|
|
|
|
const ColumnString::Offsets & offsets,
|
|
|
|
const std::string & needle,
|
|
|
|
const std::string & replacement,
|
|
|
|
ColumnString::Chars & res_data,
|
|
|
|
ColumnString::Offsets & res_offsets)
|
|
|
|
{
|
|
|
|
ColumnString::Offset res_offset = 0;
|
|
|
|
res_data.reserve(data.size());
|
|
|
|
size_t size = offsets.size();
|
|
|
|
res_offsets.resize(size);
|
|
|
|
|
2021-02-23 09:00:22 +00:00
|
|
|
typename re2_st::RE2::Options regexp_options;
|
|
|
|
/// Never write error messages to stderr. It's ignorant to do it from library code.
|
|
|
|
regexp_options.set_log_errors(false);
|
|
|
|
re2_st::RE2 searcher(needle, regexp_options);
|
2020-05-06 23:21:13 +00:00
|
|
|
int num_captures = std::min(searcher.NumberOfCapturingGroups() + 1, static_cast<int>(max_captures));
|
|
|
|
|
|
|
|
Instructions instructions = createInstructions(replacement, num_captures);
|
|
|
|
|
2020-10-14 14:04:50 +00:00
|
|
|
/// Cannot perform search for whole columns. Will process each string separately.
|
2020-05-06 23:21:13 +00:00
|
|
|
for (size_t i = 0; i < size; ++i)
|
|
|
|
{
|
|
|
|
int from = i > 0 ? offsets[i - 1] : 0;
|
|
|
|
re2_st::StringPiece input(reinterpret_cast<const char *>(data.data() + from), offsets[i] - from - 1);
|
|
|
|
|
|
|
|
processString(input, res_data, res_offset, searcher, num_captures, instructions);
|
|
|
|
res_offsets[i] = res_offset;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void vectorFixed(
|
|
|
|
const ColumnString::Chars & data,
|
|
|
|
size_t n,
|
|
|
|
const std::string & needle,
|
|
|
|
const std::string & replacement,
|
|
|
|
ColumnString::Chars & res_data,
|
|
|
|
ColumnString::Offsets & res_offsets)
|
|
|
|
{
|
|
|
|
ColumnString::Offset res_offset = 0;
|
|
|
|
size_t size = data.size() / n;
|
|
|
|
res_data.reserve(data.size());
|
|
|
|
res_offsets.resize(size);
|
|
|
|
|
2021-02-23 09:00:22 +00:00
|
|
|
typename re2_st::RE2::Options regexp_options;
|
|
|
|
/// Never write error messages to stderr. It's ignorant to do it from library code.
|
|
|
|
regexp_options.set_log_errors(false);
|
|
|
|
re2_st::RE2 searcher(needle, regexp_options);
|
2020-05-06 23:21:13 +00:00
|
|
|
int num_captures = std::min(searcher.NumberOfCapturingGroups() + 1, static_cast<int>(max_captures));
|
|
|
|
|
|
|
|
Instructions instructions = createInstructions(replacement, num_captures);
|
|
|
|
|
|
|
|
for (size_t i = 0; i < size; ++i)
|
|
|
|
{
|
|
|
|
int from = i * n;
|
|
|
|
re2_st::StringPiece input(reinterpret_cast<const char *>(data.data() + from), n);
|
|
|
|
|
|
|
|
processString(input, res_data, res_offset, searcher, num_captures, instructions);
|
|
|
|
res_offsets[i] = res_offset;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
}
|