2013-06-10 15:19:37 +00:00
|
|
|
#pragma once
|
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <DataTypes/DataTypesNumber.h>
|
|
|
|
#include <DataTypes/DataTypeString.h>
|
|
|
|
#include <DataTypes/DataTypeFixedString.h>
|
2023-01-23 22:27:48 +00:00
|
|
|
#include <Columns/ColumnsNumber.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Columns/ColumnString.h>
|
|
|
|
#include <Common/Volnitsky.h>
|
2021-05-17 07:30:42 +00:00
|
|
|
#include <Functions/IFunction.h>
|
2017-07-21 06:35:58 +00:00
|
|
|
#include <Functions/FunctionHelpers.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <IO/ReadBufferFromMemory.h>
|
|
|
|
#include <IO/ReadHelpers.h>
|
2017-03-12 10:13:45 +00:00
|
|
|
|
2013-06-10 15:19:37 +00:00
|
|
|
|
2017-06-01 13:41:58 +00:00
|
|
|
/** Functions for retrieving "visit parameters".
|
2022-04-15 22:20:47 +00:00
|
|
|
* Visit parameters in Metrica web analytics system are a special kind of JSONs.
|
2017-06-01 13:41:58 +00:00
|
|
|
* These functions are applicable to almost any JSONs.
|
|
|
|
* Implemented via templates from FunctionsStringSearch.h.
|
2014-11-12 17:23:26 +00:00
|
|
|
*
|
2017-05-27 15:45:25 +00:00
|
|
|
* Check if there is a parameter
|
2017-04-01 07:20:54 +00:00
|
|
|
* visitParamHas
|
2014-11-12 17:23:26 +00:00
|
|
|
*
|
2017-05-27 15:45:25 +00:00
|
|
|
* Retrieve the numeric value of the parameter
|
2017-04-01 07:20:54 +00:00
|
|
|
* visitParamExtractUInt
|
|
|
|
* visitParamExtractInt
|
|
|
|
* visitParamExtractFloat
|
|
|
|
* visitParamExtractBool
|
2014-11-12 17:23:26 +00:00
|
|
|
*
|
2017-05-27 15:45:25 +00:00
|
|
|
* Retrieve the string value of the parameter
|
|
|
|
* visitParamExtractString - unescape value
|
2017-04-01 07:20:54 +00:00
|
|
|
* visitParamExtractRaw
|
2013-06-10 15:19:37 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2019-03-14 23:10:51 +00:00
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
|
|
|
extern const int ILLEGAL_COLUMN;
|
2020-08-04 07:05:16 +00:00
|
|
|
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
2019-03-14 23:10:51 +00:00
|
|
|
}
|
|
|
|
|
2013-06-10 15:19:37 +00:00
|
|
|
|
2017-09-15 12:16:12 +00:00
|
|
|
template <typename NumericType>
|
2013-06-11 12:31:08 +00:00
|
|
|
struct ExtractNumericType
|
2013-06-10 15:19:37 +00:00
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
using ResultType = NumericType;
|
2014-11-12 17:23:26 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
static ResultType extract(const UInt8 * begin, const UInt8 * end)
|
|
|
|
{
|
|
|
|
ReadBufferFromMemory in(begin, end - begin);
|
2014-11-12 17:23:26 +00:00
|
|
|
|
2017-05-27 15:45:25 +00:00
|
|
|
/// Read numbers in double quotes
|
2017-04-01 07:20:54 +00:00
|
|
|
if (!in.eof() && *in.position() == '"')
|
|
|
|
++in.position();
|
2014-11-12 17:23:26 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
ResultType x = 0;
|
|
|
|
if (!in.eof())
|
2017-12-22 02:25:03 +00:00
|
|
|
{
|
|
|
|
if constexpr (std::is_floating_point_v<NumericType>)
|
|
|
|
tryReadFloatText(x, in);
|
|
|
|
else
|
|
|
|
tryReadIntText(x, in);
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
return x;
|
|
|
|
}
|
2013-06-10 15:19:37 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
2017-05-27 15:45:25 +00:00
|
|
|
/** Searches for occurrences of a field in the visit parameter and calls ParamExtractor
|
|
|
|
* for each occurrence of the field, passing it a pointer to the part of the string,
|
|
|
|
* where the occurrence of the field value begins.
|
|
|
|
* ParamExtractor must parse and return the value of the desired type.
|
2014-11-12 17:23:26 +00:00
|
|
|
*
|
2017-05-27 15:45:25 +00:00
|
|
|
* If a field was not found or an incorrect value is associated with the field,
|
|
|
|
* then the default value used - 0.
|
2013-06-10 15:19:37 +00:00
|
|
|
*/
|
2021-09-21 16:43:46 +00:00
|
|
|
template <typename Name, typename ParamExtractor>
|
2013-06-10 15:19:37 +00:00
|
|
|
struct ExtractParamImpl
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
using ResultType = typename ParamExtractor::ResultType;
|
|
|
|
|
2020-02-17 18:53:59 +00:00
|
|
|
static constexpr bool use_default_implementation_for_constants = true;
|
2020-08-01 21:14:23 +00:00
|
|
|
static constexpr bool supports_start_pos = false;
|
2021-09-21 16:43:46 +00:00
|
|
|
static constexpr auto name = Name::name;
|
2020-02-17 18:53:59 +00:00
|
|
|
|
2022-05-16 20:23:51 +00:00
|
|
|
static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {1, 2};}
|
|
|
|
|
2017-05-27 15:45:25 +00:00
|
|
|
/// It is assumed that `res` is the correct size and initialized with zeros.
|
2020-08-01 21:14:23 +00:00
|
|
|
static void vectorConstant(
|
2022-05-13 08:52:25 +00:00
|
|
|
const ColumnString::Chars & haystack_data,
|
|
|
|
const ColumnString::Offsets & haystack_offsets,
|
2017-04-01 07:20:54 +00:00
|
|
|
std::string needle,
|
2020-08-01 21:14:23 +00:00
|
|
|
const ColumnPtr & start_pos,
|
2023-01-23 22:27:48 +00:00
|
|
|
PaddedPODArray<ResultType> & res,
|
|
|
|
[[maybe_unused]] ColumnUInt8 * res_null)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2023-01-23 22:27:48 +00:00
|
|
|
/// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
|
|
|
|
assert(!res_null);
|
|
|
|
|
2020-08-02 14:24:39 +00:00
|
|
|
if (start_pos != nullptr)
|
2021-09-21 16:43:46 +00:00
|
|
|
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' doesn't support start_pos argument", name);
|
2020-08-02 14:24:39 +00:00
|
|
|
|
2017-05-27 15:45:25 +00:00
|
|
|
/// We are looking for a parameter simply as a substring of the form "name"
|
2017-04-01 07:20:54 +00:00
|
|
|
needle = "\"" + needle + "\":";
|
|
|
|
|
2022-05-13 08:52:25 +00:00
|
|
|
const UInt8 * const begin = haystack_data.data();
|
|
|
|
const UInt8 * const end = haystack_data.data() + haystack_data.size();
|
2017-04-01 07:20:54 +00:00
|
|
|
const UInt8 * pos = begin;
|
|
|
|
|
2017-05-27 15:45:25 +00:00
|
|
|
/// The current index in the string array.
|
2017-04-01 07:20:54 +00:00
|
|
|
size_t i = 0;
|
|
|
|
|
|
|
|
Volnitsky searcher(needle.data(), needle.size(), end - pos);
|
|
|
|
|
2017-05-27 15:45:25 +00:00
|
|
|
/// We will search for the next occurrence in all strings at once.
|
2017-04-01 07:20:54 +00:00
|
|
|
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
|
|
|
|
{
|
2017-05-27 15:45:25 +00:00
|
|
|
/// Let's determine which index it belongs to.
|
2022-05-13 08:52:25 +00:00
|
|
|
while (begin + haystack_offsets[i] <= pos)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
|
|
|
res[i] = 0;
|
|
|
|
++i;
|
|
|
|
}
|
|
|
|
|
2017-05-27 15:45:25 +00:00
|
|
|
/// We check that the entry does not pass through the boundaries of strings.
|
2022-05-13 08:52:25 +00:00
|
|
|
if (pos + needle.size() < begin + haystack_offsets[i])
|
|
|
|
res[i] = ParamExtractor::extract(pos + needle.size(), begin + haystack_offsets[i] - 1); /// don't include terminating zero
|
2017-04-01 07:20:54 +00:00
|
|
|
else
|
|
|
|
res[i] = 0;
|
|
|
|
|
2022-05-13 08:52:25 +00:00
|
|
|
pos = begin + haystack_offsets[i];
|
2017-04-01 07:20:54 +00:00
|
|
|
++i;
|
|
|
|
}
|
|
|
|
|
2018-09-02 03:52:04 +00:00
|
|
|
if (res.size() > i)
|
|
|
|
memset(&res[i], 0, (res.size() - i) * sizeof(res[0]));
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
2020-03-23 02:12:31 +00:00
|
|
|
template <typename... Args> static void vectorVector(Args &&...)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2021-09-21 16:43:46 +00:00
|
|
|
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support non-constant needle argument", name);
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
2020-03-23 02:12:31 +00:00
|
|
|
template <typename... Args> static void constantVector(Args &&...)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2021-09-21 16:43:46 +00:00
|
|
|
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support non-constant needle argument", name);
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2020-03-26 18:55:41 +00:00
|
|
|
|
|
|
|
template <typename... Args>
|
|
|
|
static void vectorFixedConstant(Args &&...)
|
|
|
|
{
|
2021-09-21 16:43:46 +00:00
|
|
|
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support FixedString haystack argument", name);
|
2020-03-26 18:55:41 +00:00
|
|
|
}
|
Implement SQL functions (NOT) (I)LIKE() + MATCH() with non-const needles
With this commit, SQL functions LIKE and MATCH and their variants can
work with non-const needle arguments. E.g.
create table tab
(id UInt32, haystack String, needle String)
engine = MergeTree()
order by id;
insert into tab values
(1, 'Hello', '%ell%')
(2, 'World', '%orl%')
select id, haystack, needle, like(haystack, needle)
from tab;
For that, methods vectorVector() and vectorFixedVector() were added to
MatchImpl. The existing code for const needles has an optimization where
the compiled regexp is cached. The new code expects a different needle
per row and consequently does not cache the regexp.
2022-05-16 20:37:31 +00:00
|
|
|
|
|
|
|
template <typename... Args>
|
|
|
|
static void vectorFixedVector(Args &&...)
|
|
|
|
{
|
|
|
|
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Function '{}' doesn't support FixedString haystack argument", name);
|
|
|
|
}
|
2013-06-10 15:19:37 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
2017-05-27 15:45:25 +00:00
|
|
|
/** For the case where the type of field to extract is a string.
|
2013-06-10 15:19:37 +00:00
|
|
|
*/
|
2017-09-15 12:16:12 +00:00
|
|
|
template <typename ParamExtractor>
|
2013-06-10 15:19:37 +00:00
|
|
|
struct ExtractParamToStringImpl
|
|
|
|
{
|
2022-05-13 08:52:25 +00:00
|
|
|
static void vector(const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets,
|
2017-04-01 07:20:54 +00:00
|
|
|
std::string needle,
|
2018-11-25 00:08:50 +00:00
|
|
|
ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2017-05-27 15:45:25 +00:00
|
|
|
/// Constant 5 is taken from a function that performs a similar task FunctionsStringSearch.h::ExtractImpl
|
2022-05-13 08:52:25 +00:00
|
|
|
res_data.reserve(haystack_data.size() / 5);
|
|
|
|
res_offsets.resize(haystack_offsets.size());
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2017-05-27 15:45:25 +00:00
|
|
|
/// We are looking for a parameter simply as a substring of the form "name"
|
2017-04-01 07:20:54 +00:00
|
|
|
needle = "\"" + needle + "\":";
|
|
|
|
|
2022-05-13 08:52:25 +00:00
|
|
|
const UInt8 * const begin = haystack_data.data();
|
|
|
|
const UInt8 * const end = haystack_data.data() + haystack_data.size();
|
2017-04-01 07:20:54 +00:00
|
|
|
const UInt8 * pos = begin;
|
|
|
|
|
2017-05-27 15:45:25 +00:00
|
|
|
/// The current index in the string array.
|
2017-04-01 07:20:54 +00:00
|
|
|
size_t i = 0;
|
|
|
|
|
|
|
|
Volnitsky searcher(needle.data(), needle.size(), end - pos);
|
|
|
|
|
2017-05-27 15:45:25 +00:00
|
|
|
/// We will search for the next occurrence in all strings at once.
|
2017-04-01 07:20:54 +00:00
|
|
|
while (pos < end && end != (pos = searcher.search(pos, end - pos)))
|
|
|
|
{
|
2017-05-27 15:45:25 +00:00
|
|
|
/// Determine which index it belongs to.
|
2022-05-13 08:52:25 +00:00
|
|
|
while (begin + haystack_offsets[i] <= pos)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
|
|
|
res_data.push_back(0);
|
|
|
|
res_offsets[i] = res_data.size();
|
|
|
|
++i;
|
|
|
|
}
|
|
|
|
|
2017-05-27 15:45:25 +00:00
|
|
|
/// We check that the entry does not pass through the boundaries of strings.
|
2022-05-13 08:52:25 +00:00
|
|
|
if (pos + needle.size() < begin + haystack_offsets[i])
|
|
|
|
ParamExtractor::extract(pos + needle.size(), begin + haystack_offsets[i], res_data);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2022-05-13 08:52:25 +00:00
|
|
|
pos = begin + haystack_offsets[i];
|
2017-04-01 07:20:54 +00:00
|
|
|
|
|
|
|
res_data.push_back(0);
|
|
|
|
res_offsets[i] = res_data.size();
|
|
|
|
++i;
|
|
|
|
}
|
|
|
|
|
|
|
|
while (i < res_offsets.size())
|
|
|
|
{
|
|
|
|
res_data.push_back(0);
|
|
|
|
res_offsets[i] = res_data.size();
|
|
|
|
++i;
|
|
|
|
}
|
|
|
|
}
|
2013-06-10 15:19:37 +00:00
|
|
|
};
|
|
|
|
|
2014-11-12 17:23:26 +00:00
|
|
|
}
|