ClickHouse/src/Storages/MergeTree/MergeTreeIndexGin.cpp

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

787 lines
29 KiB
C++
Raw Normal View History

2022-06-24 01:56:15 +00:00
#include <algorithm>
#include <Columns/ColumnArray.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeArray.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadHelpers.h>
#include <Interpreters/GinFilter.h>
#include <Interpreters/ExpressionActions.h>
#include <Interpreters/ExpressionAnalyzer.h>
#include <Interpreters/TreeRewriter.h>
#include <Interpreters/misc.h>
#include <Storages/MergeTree/MergeTreeData.h>
#include <Storages/MergeTree/RPNBuilder.h>
#include <Storages/MergeTree/MergeTreeIndexGin.h>
#include <Storages/MergeTree/MergeTreeIndexUtils.h>
2022-06-24 01:56:15 +00:00
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTSubquery.h>
#include <Core/Defines.h>
#include <Poco/Logger.h>
#include <Columns/ColumnNullable.h>
#include <Columns/ColumnLowCardinality.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypeLowCardinality.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
extern const int INCORRECT_QUERY;
}
MergeTreeIndexGranuleGinFilter::MergeTreeIndexGranuleGinFilter(
const String & index_name_,
size_t columns_number,
const GinFilterParameters & params_)
: index_name(index_name_)
, params(params_)
, gin_filters(
columns_number, GinFilter(params))
, has_elems(false)
{
}
void MergeTreeIndexGranuleGinFilter::serializeBinary(WriteBuffer & ostr) const
{
if (empty())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to write empty fulltext index {}.", backQuote(index_name));
const auto & size_type = std::make_shared<DataTypeUInt32>();
2022-06-24 01:56:15 +00:00
auto size_serialization = size_type->getDefaultSerialization();
for (const auto & gin_filter : gin_filters)
{
2022-07-19 03:07:57 +00:00
size_t filter_size = gin_filter.getFilter().size();
size_serialization->serializeBinary(filter_size, ostr, {});
2023-01-10 16:26:27 +00:00
ostr.write(reinterpret_cast<const char*>(gin_filter.getFilter().data()), filter_size * sizeof(GinFilter::GinSegmentWithRowIDRanges::value_type));
2022-06-24 01:56:15 +00:00
}
}
void MergeTreeIndexGranuleGinFilter::deserializeBinary(ReadBuffer & istr, MergeTreeIndexVersion version)
{
if (version != 1)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown index version {}.", version);
Field field_rows;
const auto & size_type = std::make_shared<DataTypeUInt32>();
2022-06-24 01:56:15 +00:00
auto size_serialization = size_type->getDefaultSerialization();
2022-06-24 01:56:15 +00:00
for (auto & gin_filter : gin_filters)
{
size_serialization->deserializeBinary(field_rows, istr, {});
2022-07-19 20:15:59 +00:00
size_t filter_size = field_rows.get<size_t>();
2022-06-24 01:56:15 +00:00
2022-07-19 20:15:59 +00:00
if (filter_size == 0)
2022-06-24 01:56:15 +00:00
continue;
2022-07-19 20:15:59 +00:00
gin_filter.getFilter().assign(filter_size, {});
2023-01-10 16:26:27 +00:00
istr.readStrict(reinterpret_cast<char*>(gin_filter.getFilter().data()), filter_size * sizeof(GinFilter::GinSegmentWithRowIDRanges::value_type));
2022-06-24 01:56:15 +00:00
}
has_elems = true;
}
MergeTreeIndexAggregatorGinFilter::MergeTreeIndexAggregatorGinFilter(
GinIndexStorePtr store_,
const Names & index_columns_,
const String & index_name_,
const GinFilterParameters & params_,
TokenExtractorPtr token_extractor_)
: store(store_)
, index_columns(index_columns_)
, index_name (index_name_)
, params(params_)
, token_extractor(token_extractor_)
, granule(
std::make_shared<MergeTreeIndexGranuleGinFilter>(
index_name, index_columns.size(), params))
{
}
MergeTreeIndexGranulePtr MergeTreeIndexAggregatorGinFilter::getGranuleAndReset()
{
auto new_granule = std::make_shared<MergeTreeIndexGranuleGinFilter>(
index_name, index_columns.size(), params);
new_granule.swap(granule);
return new_granule;
}
void MergeTreeIndexAggregatorGinFilter::addToGinFilter(UInt32 rowID, const char* data, size_t length, GinFilter& gin_filter, UInt64 limit)
2022-06-24 01:56:15 +00:00
{
2022-07-03 12:18:51 +00:00
size_t cur = 0;
size_t token_start = 0;
size_t token_len = 0;
while (cur < length && token_extractor->nextInStringPadded(data, length, &cur, &token_start, &token_len))
{
gin_filter.add(data + token_start, token_len, rowID, store, limit);
2022-07-03 12:18:51 +00:00
}
2022-06-24 01:56:15 +00:00
}
void MergeTreeIndexAggregatorGinFilter::update(const Block & block, size_t * pos, size_t limit)
{
if (*pos >= block.rows())
throw Exception(
"The provided position is not less than the number of block rows. Position: "
+ toString(*pos) + ", Block rows: " + toString(block.rows()) + ".", ErrorCodes::LOGICAL_ERROR);
size_t rows_read = std::min(limit, block.rows() - *pos);
2022-07-19 03:07:57 +00:00
auto row_id = store->getNextRowIDRange(rows_read);
auto start_row_id = row_id;
2022-06-24 01:56:15 +00:00
for (size_t col = 0; col < index_columns.size(); ++col)
{
const auto & column_with_type = block.getByName(index_columns[col]);
const auto & column = column_with_type.column;
size_t current_position = *pos;
bool need_to_write = false;
if (isArray(column_with_type.type))
{
const auto & column_array = assert_cast<const ColumnArray &>(*column);
const auto & column_offsets = column_array.getOffsets();
const auto & column_key = column_array.getData();
for (size_t i = 0; i < rows_read; ++i)
{
size_t element_start_row = column_offsets[current_position - 1];
size_t elements_size = column_offsets[current_position] - element_start_row;
for (size_t row_num = 0; row_num < elements_size; ++row_num)
{
auto ref = column_key.getDataAt(element_start_row + row_num);
addToGinFilter(row_id, ref.data, ref.size, granule->gin_filters[col], rows_read);
2023-01-10 16:26:27 +00:00
store->incrementCurrentSizeBy(ref.size);
2022-06-24 01:56:15 +00:00
}
current_position += 1;
2022-07-19 03:07:57 +00:00
row_id++;
2022-06-24 01:56:15 +00:00
if (store->needToWrite())
need_to_write = true;
}
}
else
{
for (size_t i = 0; i < rows_read; ++i)
{
auto ref = column->getDataAt(current_position + i);
addToGinFilter(row_id, ref.data, ref.size, granule->gin_filters[col], rows_read);
2023-01-10 16:26:27 +00:00
store->incrementCurrentSizeBy(ref.size);
2022-07-19 03:07:57 +00:00
row_id++;
2022-06-24 01:56:15 +00:00
if (store->needToWrite())
need_to_write = true;
}
}
granule->gin_filters[col].addRowRangeToGinFilter(store->getCurrentSegmentID(), start_row_id, static_cast<UInt32>(start_row_id + rows_read - 1));
2022-06-24 01:56:15 +00:00
if (need_to_write)
{
store->writeSegment();
}
}
granule->has_elems = true;
*pos += rows_read;
}
MergeTreeConditionGinFilter::MergeTreeConditionGinFilter(
const SelectQueryInfo & query_info,
ContextPtr context_,
2022-06-24 01:56:15 +00:00
const Block & index_sample_block,
const GinFilterParameters & params_,
TokenExtractorPtr token_extactor_)
2022-12-31 17:06:56 +00:00
: WithContext(context_), header(index_sample_block)
2022-06-24 01:56:15 +00:00
, params(params_)
, token_extractor(token_extactor_)
, prepared_sets(query_info.prepared_sets)
2022-06-24 01:56:15 +00:00
{
if (context_->getSettingsRef().allow_experimental_analyzer)
{
if (!query_info.filter_actions_dag)
{
rpn.push_back(RPNElement::FUNCTION_UNKNOWN);
return;
}
rpn = std::move(
RPNBuilder<RPNElement>(
query_info.filter_actions_dag->getOutputs().at(0), context_,
[&](const RPNBuilderTreeNode & node, RPNElement & out)
{
return this->traverseAtomAST(node, out);
}).extractRPN());
}
ASTPtr filter_node = buildFilterNode(query_info.query);
if (!filter_node)
{
rpn.push_back(RPNElement::FUNCTION_UNKNOWN);
return;
}
auto block_with_constants = KeyCondition::getBlockWithConstants(query_info.query, query_info.syntax_analyzer_result, context_);
RPNBuilder<RPNElement> builder(
filter_node,
context_,
std::move(block_with_constants),
query_info.prepared_sets,
[&](const RPNBuilderTreeNode & node, RPNElement & out) { return traverseAtomAST(node, out); });
rpn = std::move(builder).extractRPN();
2022-06-24 01:56:15 +00:00
}
2023-01-10 16:26:27 +00:00
/// Keep in-sync with MergeTreeConditionFullText::alwaysUnknownOrTrue
2022-06-24 01:56:15 +00:00
bool MergeTreeConditionGinFilter::alwaysUnknownOrTrue() const
{
/// Check like in KeyCondition.
std::vector<bool> rpn_stack;
for (const auto & element : rpn)
{
if (element.function == RPNElement::FUNCTION_UNKNOWN
|| element.function == RPNElement::ALWAYS_TRUE)
{
rpn_stack.push_back(true);
}
else if (element.function == RPNElement::FUNCTION_EQUALS
|| element.function == RPNElement::FUNCTION_NOT_EQUALS
|| element.function == RPNElement::FUNCTION_HAS
|| element.function == RPNElement::FUNCTION_IN
|| element.function == RPNElement::FUNCTION_NOT_IN
|| element.function == RPNElement::FUNCTION_MULTI_SEARCH
2022-06-25 18:20:07 +00:00
|| element.function == RPNElement::ALWAYS_FALSE)
2022-06-24 01:56:15 +00:00
{
rpn_stack.push_back(false);
}
else if (element.function == RPNElement::FUNCTION_NOT)
{
// do nothing
}
else if (element.function == RPNElement::FUNCTION_AND)
{
auto arg1 = rpn_stack.back();
rpn_stack.pop_back();
auto arg2 = rpn_stack.back();
rpn_stack.back() = arg1 && arg2;
}
else if (element.function == RPNElement::FUNCTION_OR)
{
auto arg1 = rpn_stack.back();
rpn_stack.pop_back();
auto arg2 = rpn_stack.back();
rpn_stack.back() = arg1 || arg2;
}
else
throw Exception("Unexpected function type in KeyCondition::RPNElement", ErrorCodes::LOGICAL_ERROR);
}
return rpn_stack[0];
}
2022-07-19 03:07:57 +00:00
bool MergeTreeConditionGinFilter::mayBeTrueOnGranuleInPart(MergeTreeIndexGranulePtr idx_granule,[[maybe_unused]] PostingsCacheForStore &cache_store) const
2022-06-24 01:56:15 +00:00
{
std::shared_ptr<MergeTreeIndexGranuleGinFilter> granule
= std::dynamic_pointer_cast<MergeTreeIndexGranuleGinFilter>(idx_granule);
if (!granule)
throw Exception(
"GinFilter index condition got a granule with the wrong type.", ErrorCodes::LOGICAL_ERROR);
/// Check like in KeyCondition.
std::vector<BoolMask> rpn_stack;
for (const auto & element : rpn)
{
if (element.function == RPNElement::FUNCTION_UNKNOWN)
{
rpn_stack.emplace_back(true, true);
}
else if (element.function == RPNElement::FUNCTION_EQUALS
|| element.function == RPNElement::FUNCTION_NOT_EQUALS
2022-06-25 18:20:07 +00:00
|| element.function == RPNElement::FUNCTION_HAS)
2022-06-24 01:56:15 +00:00
{
2022-07-19 03:07:57 +00:00
rpn_stack.emplace_back(granule->gin_filters[element.key_column].contains(*element.gin_filter, cache_store), true);
2022-06-24 01:56:15 +00:00
if (element.function == RPNElement::FUNCTION_NOT_EQUALS)
rpn_stack.back() = !rpn_stack.back();
}
else if (element.function == RPNElement::FUNCTION_IN
|| element.function == RPNElement::FUNCTION_NOT_IN)
{
std::vector<bool> result(element.set_gin_filters.back().size(), true);
for (size_t column = 0; column < element.set_key_position.size(); ++column)
{
const size_t key_idx = element.set_key_position[column];
const auto & gin_filters = element.set_gin_filters[column];
for (size_t row = 0; row < gin_filters.size(); ++row)
2022-07-19 03:07:57 +00:00
result[row] = result[row] && granule->gin_filters[key_idx].contains(gin_filters[row], cache_store);
2022-06-24 01:56:15 +00:00
}
rpn_stack.emplace_back(
std::find(std::cbegin(result), std::cend(result), true) != std::end(result), true);
if (element.function == RPNElement::FUNCTION_NOT_IN)
rpn_stack.back() = !rpn_stack.back();
}
else if (element.function == RPNElement::FUNCTION_MULTI_SEARCH)
{
std::vector<bool> result(element.set_gin_filters.back().size(), true);
const auto & gin_filters = element.set_gin_filters[0];
for (size_t row = 0; row < gin_filters.size(); ++row)
2022-07-19 03:07:57 +00:00
result[row] = result[row] && granule->gin_filters[element.key_column].contains(gin_filters[row], cache_store);
2022-06-24 01:56:15 +00:00
rpn_stack.emplace_back(
std::find(std::cbegin(result), std::cend(result), true) != std::end(result), true);
}
else if (element.function == RPNElement::FUNCTION_NOT)
{
rpn_stack.back() = !rpn_stack.back();
}
else if (element.function == RPNElement::FUNCTION_AND)
{
auto arg1 = rpn_stack.back();
rpn_stack.pop_back();
auto arg2 = rpn_stack.back();
rpn_stack.back() = arg1 & arg2;
}
else if (element.function == RPNElement::FUNCTION_OR)
{
auto arg1 = rpn_stack.back();
rpn_stack.pop_back();
auto arg2 = rpn_stack.back();
rpn_stack.back() = arg1 | arg2;
}
else if (element.function == RPNElement::ALWAYS_FALSE)
{
rpn_stack.emplace_back(false, true);
}
else if (element.function == RPNElement::ALWAYS_TRUE)
{
rpn_stack.emplace_back(true, false);
}
else
throw Exception("Unexpected function type in GinFilterCondition::RPNElement", ErrorCodes::LOGICAL_ERROR);
}
if (rpn_stack.size() != 1)
throw Exception("Unexpected stack size in GinFilterCondition::mayBeTrueOnGranule", ErrorCodes::LOGICAL_ERROR);
return rpn_stack[0].can_be_true;
}
bool MergeTreeConditionGinFilter::traverseAtomAST(const RPNBuilderTreeNode & node, RPNElement & out)
2022-06-24 01:56:15 +00:00
{
{
Field const_value;
DataTypePtr const_type;
if (node.tryGetConstant(const_value, const_type))
2022-06-24 01:56:15 +00:00
{
/// Check constant like in KeyCondition
if (const_value.getType() == Field::Types::UInt64
|| const_value.getType() == Field::Types::Int64
|| const_value.getType() == Field::Types::Float64)
{
/// Zero in all types is represented in memory the same way as in UInt64.
out.function = const_value.get<UInt64>()
? RPNElement::ALWAYS_TRUE
: RPNElement::ALWAYS_FALSE;
return true;
}
}
}
if (node.isFunction())
2022-06-24 01:56:15 +00:00
{
const auto function = node.toFunctionNode();
// auto arguments_size = function.getArgumentsSize();
auto function_name = function.getFunctionName();
2022-06-24 01:56:15 +00:00
size_t function_arguments_size = function.getArgumentsSize();
if (function_arguments_size != 2)
2022-06-24 01:56:15 +00:00
return false;
auto lhs_argument = function.getArgumentAt(0);
auto rhs_argument = function.getArgumentAt(1);
2022-06-24 01:56:15 +00:00
if (functionIsInOrGlobalInOperator(function_name))
2022-06-24 01:56:15 +00:00
{
if (tryPrepareSetGinFilter(lhs_argument, rhs_argument, out))
2022-06-24 01:56:15 +00:00
{
if (function_name == "notIn")
2022-06-24 01:56:15 +00:00
{
out.function = RPNElement::FUNCTION_NOT_IN;
return true;
}
else if (function_name == "in")
2022-06-24 01:56:15 +00:00
{
out.function = RPNElement::FUNCTION_IN;
return true;
}
}
}
else if (function_name == "equals" ||
function_name == "notEquals" ||
function_name == "has" ||
function_name == "mapContains" ||
function_name == "like" ||
function_name == "notLike" ||
function_name == "hasToken" ||
function_name == "startsWith" ||
function_name == "endsWith" ||
function_name == "multiSearchAny")
2022-06-24 01:56:15 +00:00
{
Field const_value;
DataTypePtr const_type;
if (rhs_argument.tryGetConstant(const_value, const_type))
2022-06-24 01:56:15 +00:00
{
if (traverseASTEquals(function_name, lhs_argument, const_type, const_value, out))
2022-06-24 01:56:15 +00:00
return true;
}
else if (lhs_argument.tryGetConstant(const_value, const_type) && (function_name == "equals" || function_name == "notEquals"))
2022-06-24 01:56:15 +00:00
{
if (traverseASTEquals(function_name, rhs_argument, const_type, const_value, out))
2022-06-24 01:56:15 +00:00
return true;
}
}
}
return false;
}
bool MergeTreeConditionGinFilter::traverseASTEquals(
const String & function_name,
const RPNBuilderTreeNode & key_ast,
2022-06-24 01:56:15 +00:00
const DataTypePtr & value_type,
const Field & value_field,
RPNElement & out)
{
auto value_data_type = WhichDataType(value_type);
if (!value_data_type.isStringOrFixedString() && !value_data_type.isArray())
return false;
Field const_value = value_field;
size_t key_column_num = 0;
2022-12-31 17:06:56 +00:00
bool key_exists = header.has(key_ast.getColumnName());
bool map_key_exists = header.has(fmt::format("mapKeys({})", key_ast.getColumnName()));
2022-06-24 01:56:15 +00:00
if (key_ast.isFunction())
2022-06-24 01:56:15 +00:00
{
const auto function = key_ast.toFunctionNode();
if (function.getFunctionName() == "arrayElement")
2022-06-24 01:56:15 +00:00
{
/** Try to parse arrayElement for mapKeys index.
* It is important to ignore keys like column_map['Key'] = '' because if key does not exists in map
* we return default value for arrayElement.
*
* We cannot skip keys that does not exist in map if comparison is with default type value because
* that way we skip necessary granules where map key does not exists.
*/
if (value_field == value_type->getDefault())
return false;
auto first_argument = function.getArgumentAt(0);
const auto map_column_name = first_argument.getColumnName();
2022-06-24 01:56:15 +00:00
auto map_keys_index_column_name = fmt::format("mapKeys({})", map_column_name);
auto map_values_index_column_name = fmt::format("mapValues({})", map_column_name);
2022-12-31 17:06:56 +00:00
if (header.has(map_keys_index_column_name))
2022-06-24 01:56:15 +00:00
{
auto argument = function.getArgumentAt(1);
DataTypePtr const_type;
if (argument.tryGetConstant(const_value, const_type))
2022-06-24 01:56:15 +00:00
{
2022-12-31 17:06:56 +00:00
key_column_num = header.getPositionByName(map_keys_index_column_name);
2022-06-24 01:56:15 +00:00
key_exists = true;
}
else
{
return false;
}
}
2022-12-31 17:06:56 +00:00
else if (header.has(map_values_index_column_name))
2022-06-24 01:56:15 +00:00
{
2022-12-31 17:06:56 +00:00
key_column_num = header.getPositionByName(map_values_index_column_name);
2022-06-24 01:56:15 +00:00
key_exists = true;
}
else
{
return false;
}
}
}
if (!key_exists && !map_key_exists)
return false;
if (map_key_exists && (function_name == "has" || function_name == "mapContains"))
{
out.key_column = key_column_num;
out.function = RPNElement::FUNCTION_HAS;
out.gin_filter = std::make_unique<GinFilter>(params);
auto & value = const_value.get<String>();
token_extractor->stringToGinFilter(value.data(), value.size(), *out.gin_filter);
return true;
}
else if (function_name == "has")
{
out.key_column = key_column_num;
out.function = RPNElement::FUNCTION_HAS;
out.gin_filter = std::make_unique<GinFilter>(params);
auto & value = const_value.get<String>();
token_extractor->stringToGinFilter(value.data(), value.size(), *out.gin_filter);
return true;
}
if (function_name == "notEquals")
{
out.key_column = key_column_num;
out.function = RPNElement::FUNCTION_NOT_EQUALS;
out.gin_filter = std::make_unique<GinFilter>(params);
const auto & value = const_value.get<String>();
token_extractor->stringToGinFilter(value.data(), value.size(), *out.gin_filter);
return true;
}
else if (function_name == "equals")
{
out.key_column = key_column_num;
out.function = RPNElement::FUNCTION_EQUALS;
out.gin_filter = std::make_unique<GinFilter>(params);
const auto & value = const_value.get<String>();
token_extractor->stringToGinFilter(value.data(), value.size(), *out.gin_filter);
return true;
}
else if (function_name == "like")
{
out.key_column = key_column_num;
2022-06-25 18:20:07 +00:00
out.function = RPNElement::FUNCTION_EQUALS;
2022-06-24 01:56:15 +00:00
out.gin_filter = std::make_unique<GinFilter>(params);
const auto & value = const_value.get<String>();
token_extractor->stringLikeToGinFilter(value.data(), value.size(), *out.gin_filter);
return true;
}
else if (function_name == "notLike")
{
out.key_column = key_column_num;
out.function = RPNElement::FUNCTION_NOT_EQUALS;
out.gin_filter = std::make_unique<GinFilter>(params);
const auto & value = const_value.get<String>();
token_extractor->stringLikeToGinFilter(value.data(), value.size(), *out.gin_filter);
return true;
}
else if (function_name == "hasToken")
{
out.key_column = key_column_num;
out.function = RPNElement::FUNCTION_EQUALS;
out.gin_filter = std::make_unique<GinFilter>(params);
const auto & value = const_value.get<String>();
token_extractor->stringToGinFilter(value.data(), value.size(), *out.gin_filter);
return true;
}
else if (function_name == "startsWith")
{
out.key_column = key_column_num;
out.function = RPNElement::FUNCTION_EQUALS;
out.gin_filter = std::make_unique<GinFilter>(params);
const auto & value = const_value.get<String>();
token_extractor->stringToGinFilter(value.data(), value.size(), *out.gin_filter);
return true;
}
else if (function_name == "endsWith")
{
out.key_column = key_column_num;
out.function = RPNElement::FUNCTION_EQUALS;
out.gin_filter = std::make_unique<GinFilter>(params);
const auto & value = const_value.get<String>();
token_extractor->stringToGinFilter(value.data(), value.size(), *out.gin_filter);
return true;
}
else if (function_name == "multiSearchAny")
{
out.key_column = key_column_num;
out.function = RPNElement::FUNCTION_MULTI_SEARCH;
/// 2d vector is not needed here but is used because already exists for FUNCTION_IN
std::vector<std::vector<GinFilter>> gin_filters;
gin_filters.emplace_back();
for (const auto & element : const_value.get<Array>())
{
if (element.getType() != Field::Types::String)
return false;
gin_filters.back().emplace_back(params);
const auto & value = element.get<String>();
token_extractor->stringToGinFilter(value.data(), value.size(), gin_filters.back().back());
}
out.set_gin_filters = std::move(gin_filters);
return true;
}
return false;
}
bool MergeTreeConditionGinFilter::tryPrepareSetGinFilter(
const RPNBuilderTreeNode & lhs,
const RPNBuilderTreeNode & rhs,
2022-06-24 01:56:15 +00:00
RPNElement & out)
{
std::vector<KeyTuplePositionMapping> key_tuple_mapping;
DataTypes data_types;
if (lhs.isFunction() && lhs.toFunctionNode().getFunctionName() == "tuple")
2022-06-24 01:56:15 +00:00
{
const auto function = lhs.toFunctionNode();
auto arguments_size = function.getArgumentsSize();
for (size_t i = 0; i < arguments_size; ++i)
2022-06-24 01:56:15 +00:00
{
2022-12-31 17:06:56 +00:00
if (header.has(function.getArgumentAt(i).getColumnName()))
2022-06-24 01:56:15 +00:00
{
2022-12-31 17:06:56 +00:00
auto key = header.getPositionByName(function.getArgumentAt(i).getColumnName());
2022-06-24 01:56:15 +00:00
key_tuple_mapping.emplace_back(i, key);
2022-12-31 17:06:56 +00:00
data_types.push_back(header.getByPosition(key).type);
2022-06-24 01:56:15 +00:00
}
}
}
else
{
2022-12-31 17:06:56 +00:00
if (header.has(lhs.getColumnName()))
2022-06-24 01:56:15 +00:00
{
2022-12-31 17:06:56 +00:00
auto key = header.getPositionByName(lhs.getColumnName());
2022-06-24 01:56:15 +00:00
key_tuple_mapping.emplace_back(0, key);
2023-01-05 04:08:28 +00:00
data_types.push_back(header.getByPosition(key).type);
2022-06-24 01:56:15 +00:00
}
}
if (key_tuple_mapping.empty())
return false;
ConstSetPtr prepared_set = rhs.tryGetPreparedSet();
if (!prepared_set && !prepared_set->hasExplicitSetElements())
2022-06-24 01:56:15 +00:00
return false;
for (const auto & data_type : prepared_set->getDataTypes())
if (data_type->getTypeId() != TypeIndex::String && data_type->getTypeId() != TypeIndex::FixedString)
return false;
std::vector<std::vector<GinFilter>> gin_filters;
std::vector<size_t> key_position;
Columns columns = prepared_set->getSetElements();
for (const auto & elem : key_tuple_mapping)
{
gin_filters.emplace_back();
gin_filters.back().reserve(prepared_set->getTotalRowCount());
2022-06-24 01:56:15 +00:00
key_position.push_back(elem.key_index);
size_t tuple_idx = elem.tuple_index;
const auto & column = columns[tuple_idx];
for (size_t row = 0; row < prepared_set->getTotalRowCount(); ++row)
{
gin_filters.back().emplace_back(params);
auto ref = column->getDataAt(row);
2022-12-31 17:06:56 +00:00
token_extractor->stringToGinFilter(ref.data, ref.size, gin_filters.back().back());
2022-06-24 01:56:15 +00:00
}
}
out.set_key_position = std::move(key_position);
out.set_gin_filters = std::move(gin_filters);
return true;
}
MergeTreeIndexGranulePtr MergeTreeIndexGinFilter::createIndexGranule() const
{
return std::make_shared<MergeTreeIndexGranuleGinFilter>(index.name, index.column_names.size(), params);
}
MergeTreeIndexAggregatorPtr MergeTreeIndexGinFilter::createIndexAggregator() const
{
/// should not be called: createIndexAggregatorForPart should be used
assert(false);
return nullptr;
}
MergeTreeIndexAggregatorPtr MergeTreeIndexGinFilter::createIndexAggregatorForPart(const GinIndexStorePtr &store) const
{
return std::make_shared<MergeTreeIndexAggregatorGinFilter>(store, index.column_names, index.name, params, token_extractor.get());
}
MergeTreeIndexConditionPtr MergeTreeIndexGinFilter::createIndexCondition(
const SelectQueryInfo & query, ContextPtr context) const
{
return std::make_shared<MergeTreeConditionGinFilter>(query, context, index.sample_block, params, token_extractor.get());
};
bool MergeTreeIndexGinFilter::mayBenefitFromIndexForIn(const ASTPtr & node) const
{
return std::find(std::cbegin(index.column_names), std::cend(index.column_names), node->getColumnName()) != std::cend(index.column_names);
}
MergeTreeIndexPtr ginIndexCreator(
const IndexDescription & index)
{
2023-01-10 16:26:27 +00:00
size_t n = index.arguments.empty() ? 0 : index.arguments[0].get<size_t>();
Float64 density = index.arguments.size() < 2 ? 1.0f : index.arguments[1].get<Float64>();
GinFilterParameters params(n, density);
2022-06-24 01:56:15 +00:00
2023-01-10 16:26:27 +00:00
/// Use SplitTokenExtractor when n is 0, otherwise use NgramTokenExtractor
if (n > 0)
{
auto tokenizer = std::make_unique<NgramTokenExtractor>(n);
return std::make_shared<MergeTreeIndexGinFilter>(index, params, std::move(tokenizer));
2022-06-24 01:56:15 +00:00
}
else
{
2023-01-10 16:26:27 +00:00
auto tokenizer = std::make_unique<SplitTokenExtractor>();
return std::make_shared<MergeTreeIndexGinFilter>(index, params, std::move(tokenizer));
2022-06-24 01:56:15 +00:00
}
}
void ginIndexValidator(const IndexDescription & index, bool /*attach*/)
{
for (const auto & index_data_type : index.data_types)
{
WhichDataType data_type(index_data_type);
if (data_type.isArray())
{
const auto & gin_type = assert_cast<const DataTypeArray &>(*index_data_type);
data_type = WhichDataType(gin_type.getNestedType());
}
else if (data_type.isLowCarnality())
{
const auto & low_cardinality = assert_cast<const DataTypeLowCardinality &>(*index_data_type);
data_type = WhichDataType(low_cardinality.getDictionaryType());
}
if (!data_type.isString() && !data_type.isFixedString())
throw Exception("Inverted index can be used only with `String`, `FixedString`, `LowCardinality(String)`, `LowCardinality(FixedString)` column or Array with `String` or `FixedString` values column.", ErrorCodes::INCORRECT_QUERY);
2022-06-24 01:56:15 +00:00
}
2022-07-03 12:18:51 +00:00
if (index.type != GinFilter::getName())
2022-06-24 01:56:15 +00:00
throw Exception("Unknown index type: " + backQuote(index.name), ErrorCodes::LOGICAL_ERROR);
if (index.arguments.size() > 2)
throw Exception("Inverted index must have less than two arguments.", ErrorCodes::INCORRECT_QUERY);
2022-06-24 01:56:15 +00:00
if (index.arguments.size() >= 1 && index.arguments[0].getType() != Field::Types::UInt64)
throw Exception("The first Inverted index argument must be positive integer.", ErrorCodes::INCORRECT_QUERY);
if (index.arguments.size() == 2 && (index.arguments[1].getType() != Field::Types::Float64 || index.arguments[1].get<Float64>() <= 0 || index.arguments[1].get<Float64>() > 1))
throw Exception("The second Inverted index argument must be a float between 0 and 1.", ErrorCodes::INCORRECT_QUERY);
2022-06-24 01:56:15 +00:00
2022-07-19 03:07:57 +00:00
size_t ngrams = index.arguments.empty() ? 0 : index.arguments[0].get<size_t>();
Float64 density = index.arguments.size() < 2 ? 1.0f : index.arguments[1].get<Float64>();
2022-06-24 01:56:15 +00:00
/// Just validate
GinFilterParameters params(ngrams, density);
2022-06-24 01:56:15 +00:00
}
}