2021-02-10 14:12:49 +00:00
|
|
|
#include <Interpreters/ExpressionAnalyzer.h>
|
|
|
|
#include <Interpreters/TreeRewriter.h>
|
|
|
|
#include <Storages/ProjectionsDescription.h>
|
|
|
|
|
|
|
|
#include <Parsers/ASTProjectionDeclaration.h>
|
|
|
|
#include <Parsers/ParserCreateQuery.h>
|
|
|
|
#include <Parsers/parseQuery.h>
|
|
|
|
#include <Parsers/queryToString.h>
|
|
|
|
|
|
|
|
#include <Core/Defines.h>
|
|
|
|
#include <Interpreters/InterpreterSelectQuery.h>
|
|
|
|
#include <Parsers/ASTProjectionSelectQuery.h>
|
|
|
|
#include <Parsers/ASTSubquery.h>
|
|
|
|
#include <Processors/Pipe.h>
|
|
|
|
#include <Processors/Sources/SourceFromSingleChunk.h>
|
|
|
|
|
2021-08-26 11:01:15 +00:00
|
|
|
#include <DataStreams/SquashingBlockInputStream.h>
|
2021-02-10 14:12:49 +00:00
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
|
|
|
extern const int INCORRECT_QUERY;
|
|
|
|
extern const int NO_SUCH_PROJECTION_IN_TABLE;
|
|
|
|
extern const int ILLEGAL_PROJECTION;
|
|
|
|
extern const int NOT_IMPLEMENTED;
|
2021-08-26 11:01:15 +00:00
|
|
|
extern const int LOGICAL_ERROR;
|
2021-02-10 14:12:49 +00:00
|
|
|
};
|
|
|
|
|
2021-04-26 11:48:51 +00:00
|
|
|
const char * ProjectionDescription::typeToString(Type type)
|
|
|
|
{
|
|
|
|
switch (type)
|
|
|
|
{
|
|
|
|
case Type::Normal:
|
|
|
|
return "normal";
|
|
|
|
case Type::Aggregate:
|
|
|
|
return "aggregate";
|
|
|
|
}
|
|
|
|
|
|
|
|
__builtin_unreachable();
|
|
|
|
}
|
|
|
|
|
2021-02-10 14:12:49 +00:00
|
|
|
|
|
|
|
bool ProjectionDescription::isPrimaryKeyColumnPossiblyWrappedInFunctions(const ASTPtr & node) const
|
|
|
|
{
|
|
|
|
const String column_name = node->getColumnName();
|
|
|
|
|
|
|
|
for (const auto & key_name : metadata->getPrimaryKeyColumns())
|
|
|
|
if (column_name == key_name)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
if (const auto * func = node->as<ASTFunction>())
|
|
|
|
if (func->arguments->children.size() == 1)
|
|
|
|
return isPrimaryKeyColumnPossiblyWrappedInFunctions(func->arguments->children.front());
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2021-04-26 16:06:18 +00:00
|
|
|
ProjectionDescription ProjectionDescription::clone() const
|
2021-02-10 14:12:49 +00:00
|
|
|
{
|
2021-04-26 16:06:18 +00:00
|
|
|
ProjectionDescription other;
|
|
|
|
if (definition_ast)
|
|
|
|
other.definition_ast = definition_ast->clone();
|
|
|
|
if (query_ast)
|
|
|
|
other.query_ast = query_ast->clone();
|
|
|
|
|
|
|
|
other.name = name;
|
|
|
|
other.type = type;
|
|
|
|
other.required_columns = required_columns;
|
|
|
|
other.column_names = column_names;
|
|
|
|
other.data_types = data_types;
|
|
|
|
other.sample_block = sample_block;
|
2021-04-29 07:38:47 +00:00
|
|
|
other.sample_block_for_keys = sample_block_for_keys;
|
2021-04-26 16:06:18 +00:00
|
|
|
other.metadata = metadata;
|
|
|
|
other.key_size = key_size;
|
2021-08-27 18:35:13 +00:00
|
|
|
other.is_minmax_count_projection = is_minmax_count_projection;
|
2021-04-26 16:06:18 +00:00
|
|
|
|
|
|
|
return other;
|
|
|
|
}
|
2021-02-10 14:12:49 +00:00
|
|
|
|
2021-04-26 16:06:18 +00:00
|
|
|
ProjectionsDescription ProjectionsDescription::clone() const
|
|
|
|
{
|
|
|
|
ProjectionsDescription other;
|
|
|
|
for (const auto & projection : projections)
|
|
|
|
other.add(projection.clone());
|
2021-02-10 14:12:49 +00:00
|
|
|
|
2021-04-26 16:06:18 +00:00
|
|
|
return other;
|
2021-02-10 14:12:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
bool ProjectionDescription::operator==(const ProjectionDescription & other) const
|
|
|
|
{
|
|
|
|
return name == other.name && queryToString(definition_ast) == queryToString(other.definition_ast);
|
|
|
|
}
|
|
|
|
|
|
|
|
ProjectionDescription
|
|
|
|
ProjectionDescription::getProjectionFromAST(const ASTPtr & definition_ast, const ColumnsDescription & columns, ContextPtr query_context)
|
|
|
|
{
|
|
|
|
const auto * projection_definition = definition_ast->as<ASTProjectionDeclaration>();
|
|
|
|
|
|
|
|
if (!projection_definition)
|
|
|
|
throw Exception("Cannot create projection from non ASTProjectionDeclaration AST", ErrorCodes::INCORRECT_QUERY);
|
|
|
|
|
|
|
|
if (projection_definition->name.empty())
|
|
|
|
throw Exception("Projection must have name in definition.", ErrorCodes::INCORRECT_QUERY);
|
|
|
|
|
2021-08-27 18:35:13 +00:00
|
|
|
if (startsWith(projection_definition->name, "tmp_"))
|
|
|
|
throw Exception("Projection's name cannot start with 'tmp_'", ErrorCodes::INCORRECT_QUERY);
|
|
|
|
|
2021-02-10 14:12:49 +00:00
|
|
|
if (!projection_definition->query)
|
|
|
|
throw Exception("QUERY is required for projection", ErrorCodes::INCORRECT_QUERY);
|
|
|
|
|
|
|
|
ProjectionDescription result;
|
|
|
|
result.definition_ast = projection_definition->clone();
|
|
|
|
result.name = projection_definition->name;
|
2021-04-26 11:48:51 +00:00
|
|
|
|
2021-02-10 14:12:49 +00:00
|
|
|
auto query = projection_definition->query->as<ASTProjectionSelectQuery &>();
|
|
|
|
result.query_ast = query.cloneToASTSelect();
|
|
|
|
|
|
|
|
auto external_storage_holder = std::make_shared<TemporaryTableHolder>(query_context, columns, ConstraintsDescription{});
|
|
|
|
StoragePtr storage = external_storage_holder->getTable();
|
|
|
|
InterpreterSelectQuery select(
|
2021-05-05 18:52:20 +00:00
|
|
|
result.query_ast, query_context, storage, {}, SelectQueryOptions{QueryProcessingStage::WithMergeableState}.modify().ignoreAlias());
|
2021-02-10 14:12:49 +00:00
|
|
|
|
|
|
|
result.required_columns = select.getRequiredColumns();
|
|
|
|
result.sample_block = select.getSampleBlock();
|
|
|
|
|
2021-04-29 07:38:47 +00:00
|
|
|
const auto & analysis_result = select.getAnalysisResult();
|
|
|
|
if (analysis_result.need_aggregate)
|
|
|
|
{
|
|
|
|
for (const auto & key : select.getQueryAnalyzer()->aggregationKeys())
|
|
|
|
result.sample_block_for_keys.insert({nullptr, key.type, key.name});
|
|
|
|
}
|
|
|
|
|
2021-02-10 14:12:49 +00:00
|
|
|
for (size_t i = 0; i < result.sample_block.columns(); ++i)
|
|
|
|
{
|
|
|
|
const auto & column_with_type_name = result.sample_block.getByPosition(i);
|
|
|
|
|
|
|
|
if (column_with_type_name.column && isColumnConst(*column_with_type_name.column))
|
|
|
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Projections cannot contain constant columns: {}", column_with_type_name.name);
|
|
|
|
|
|
|
|
result.column_names.emplace_back(column_with_type_name.name);
|
|
|
|
result.data_types.emplace_back(column_with_type_name.type);
|
|
|
|
}
|
|
|
|
|
|
|
|
StorageInMemoryMetadata metadata;
|
|
|
|
metadata.setColumns(ColumnsDescription(result.sample_block.getNamesAndTypesList()));
|
|
|
|
metadata.partition_key = KeyDescription::getSortingKeyFromAST({}, metadata.columns, query_context, {});
|
|
|
|
|
|
|
|
const auto & query_select = result.query_ast->as<const ASTSelectQuery &>();
|
|
|
|
if (select.hasAggregation())
|
|
|
|
{
|
2021-05-05 18:52:20 +00:00
|
|
|
result.type = ProjectionDescription::Type::Aggregate;
|
2021-02-10 14:12:49 +00:00
|
|
|
if (const auto & group_expression_list = query_select.groupBy())
|
|
|
|
{
|
|
|
|
ASTPtr order_expression;
|
|
|
|
if (group_expression_list->children.size() == 1)
|
|
|
|
{
|
|
|
|
result.key_size = 1;
|
|
|
|
order_expression = std::make_shared<ASTIdentifier>(group_expression_list->children.front()->getColumnName());
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
auto function_node = std::make_shared<ASTFunction>();
|
|
|
|
function_node->name = "tuple";
|
|
|
|
function_node->arguments = group_expression_list->clone();
|
|
|
|
result.key_size = function_node->arguments->children.size();
|
|
|
|
for (auto & child : function_node->arguments->children)
|
|
|
|
child = std::make_shared<ASTIdentifier>(child->getColumnName());
|
|
|
|
function_node->children.push_back(function_node->arguments);
|
|
|
|
order_expression = function_node;
|
|
|
|
}
|
|
|
|
metadata.sorting_key = KeyDescription::getSortingKeyFromAST(order_expression, metadata.columns, query_context, {});
|
|
|
|
metadata.primary_key = KeyDescription::getKeyFromAST(order_expression, metadata.columns, query_context);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
metadata.sorting_key = KeyDescription::getSortingKeyFromAST({}, metadata.columns, query_context, {});
|
|
|
|
metadata.primary_key = KeyDescription::getKeyFromAST({}, metadata.columns, query_context);
|
|
|
|
}
|
2021-05-31 04:24:09 +00:00
|
|
|
if (query.orderBy())
|
2021-02-10 14:12:49 +00:00
|
|
|
throw Exception(
|
|
|
|
"When aggregation is used in projection, ORDER BY cannot be specified", ErrorCodes::ILLEGAL_PROJECTION);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2021-05-05 18:52:20 +00:00
|
|
|
result.type = ProjectionDescription::Type::Normal;
|
2021-05-31 04:24:09 +00:00
|
|
|
metadata.sorting_key = KeyDescription::getSortingKeyFromAST(query.orderBy(), metadata.columns, query_context, {});
|
|
|
|
metadata.primary_key = KeyDescription::getKeyFromAST(query.orderBy(), metadata.columns, query_context);
|
2021-02-10 14:12:49 +00:00
|
|
|
}
|
|
|
|
metadata.primary_key.definition_ast = nullptr;
|
|
|
|
result.metadata = std::make_shared<StorageInMemoryMetadata>(metadata);
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2021-08-27 18:35:13 +00:00
|
|
|
ProjectionDescription
|
|
|
|
ProjectionDescription::getMinMaxCountProjection(const ColumnsDescription & columns, const Names & minmax_columns, ContextPtr query_context)
|
|
|
|
{
|
|
|
|
auto select_query = std::make_shared<ASTProjectionSelectQuery>();
|
|
|
|
ASTPtr select_expression_list = std::make_shared<ASTExpressionList>();
|
|
|
|
for (const auto & column : minmax_columns)
|
|
|
|
{
|
|
|
|
select_expression_list->children.push_back(makeASTFunction("min", std::make_shared<ASTIdentifier>(column)));
|
|
|
|
select_expression_list->children.push_back(makeASTFunction("max", std::make_shared<ASTIdentifier>(column)));
|
|
|
|
}
|
|
|
|
select_expression_list->children.push_back(makeASTFunction("count"));
|
|
|
|
select_query->setExpression(ASTProjectionSelectQuery::Expression::SELECT, std::move(select_expression_list));
|
|
|
|
|
|
|
|
ProjectionDescription result;
|
|
|
|
result.definition_ast = select_query;
|
|
|
|
result.name = MINMAX_COUNT_PROJECTION_NAME;
|
|
|
|
result.query_ast = select_query->cloneToASTSelect();
|
|
|
|
|
|
|
|
auto external_storage_holder = std::make_shared<TemporaryTableHolder>(query_context, columns, ConstraintsDescription{});
|
|
|
|
StoragePtr storage = external_storage_holder->getTable();
|
|
|
|
InterpreterSelectQuery select(
|
|
|
|
result.query_ast, query_context, storage, {}, SelectQueryOptions{QueryProcessingStage::WithMergeableState}.modify().ignoreAlias());
|
|
|
|
result.required_columns = select.getRequiredColumns();
|
|
|
|
result.sample_block = select.getSampleBlock();
|
|
|
|
|
|
|
|
for (size_t i = 0; i < result.sample_block.columns(); ++i)
|
|
|
|
{
|
|
|
|
const auto & column_with_type_name = result.sample_block.getByPosition(i);
|
|
|
|
|
|
|
|
if (column_with_type_name.column && isColumnConst(*column_with_type_name.column))
|
|
|
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Projections cannot contain constant columns: {}", column_with_type_name.name);
|
|
|
|
|
|
|
|
result.column_names.emplace_back(column_with_type_name.name);
|
|
|
|
result.data_types.emplace_back(column_with_type_name.type);
|
|
|
|
}
|
|
|
|
result.type = ProjectionDescription::Type::Aggregate;
|
|
|
|
StorageInMemoryMetadata metadata;
|
|
|
|
metadata.setColumns(ColumnsDescription(result.sample_block.getNamesAndTypesList()));
|
|
|
|
metadata.partition_key = KeyDescription::getSortingKeyFromAST({}, metadata.columns, query_context, {});
|
|
|
|
metadata.sorting_key = KeyDescription::getSortingKeyFromAST({}, metadata.columns, query_context, {});
|
|
|
|
metadata.primary_key = KeyDescription::getKeyFromAST({}, metadata.columns, query_context);
|
|
|
|
metadata.primary_key.definition_ast = nullptr;
|
|
|
|
result.metadata = std::make_shared<StorageInMemoryMetadata>(metadata);
|
|
|
|
result.is_minmax_count_projection = true;
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2021-02-10 14:12:49 +00:00
|
|
|
void ProjectionDescription::recalculateWithNewColumns(const ColumnsDescription & new_columns, ContextPtr query_context)
|
|
|
|
{
|
|
|
|
*this = getProjectionFromAST(definition_ast, new_columns, query_context);
|
|
|
|
}
|
|
|
|
|
2021-08-26 11:01:15 +00:00
|
|
|
|
|
|
|
Block ProjectionDescription::calculate(const Block & block, ContextPtr context) const
|
|
|
|
{
|
|
|
|
auto in = InterpreterSelectQuery(
|
|
|
|
query_ast,
|
|
|
|
context,
|
|
|
|
Pipe(std::make_shared<SourceFromSingleChunk>(block, Chunk(block.getColumns(), block.rows()))),
|
|
|
|
SelectQueryOptions{
|
|
|
|
type == ProjectionDescription::Type::Normal ? QueryProcessingStage::FetchColumns
|
|
|
|
: QueryProcessingStage::WithMergeableState})
|
|
|
|
.execute()
|
|
|
|
.getInputStream();
|
|
|
|
in = std::make_shared<SquashingBlockInputStream>(in, block.rows(), 0);
|
|
|
|
in->readPrefix();
|
|
|
|
auto ret = in->read();
|
|
|
|
if (in->read())
|
|
|
|
throw Exception("Projection cannot increase the number of rows in a block", ErrorCodes::LOGICAL_ERROR);
|
|
|
|
in->readSuffix();
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2021-02-10 14:12:49 +00:00
|
|
|
String ProjectionsDescription::toString() const
|
|
|
|
{
|
|
|
|
if (empty())
|
|
|
|
return {};
|
|
|
|
|
|
|
|
ASTExpressionList list;
|
|
|
|
for (const auto & projection : projections)
|
|
|
|
list.children.push_back(projection.definition_ast);
|
|
|
|
|
|
|
|
return serializeAST(list, true);
|
|
|
|
}
|
|
|
|
|
|
|
|
ProjectionsDescription ProjectionsDescription::parse(const String & str, const ColumnsDescription & columns, ContextPtr query_context)
|
|
|
|
{
|
|
|
|
ProjectionsDescription result;
|
|
|
|
if (str.empty())
|
|
|
|
return result;
|
|
|
|
|
|
|
|
ParserProjectionDeclarationList parser;
|
|
|
|
ASTPtr list = parseQuery(parser, str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
|
|
|
|
|
|
|
|
for (const auto & projection_ast : list->children)
|
|
|
|
{
|
|
|
|
auto projection = ProjectionDescription::getProjectionFromAST(projection_ast, columns, query_context);
|
|
|
|
result.add(std::move(projection));
|
|
|
|
}
|
|
|
|
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool ProjectionsDescription::has(const String & projection_name) const
|
|
|
|
{
|
2021-04-26 16:06:18 +00:00
|
|
|
return map.count(projection_name) > 0;
|
2021-02-10 14:12:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
const ProjectionDescription & ProjectionsDescription::get(const String & projection_name) const
|
|
|
|
{
|
2021-04-26 16:06:18 +00:00
|
|
|
auto it = map.find(projection_name);
|
|
|
|
if (it == map.end())
|
2021-02-10 14:12:49 +00:00
|
|
|
throw Exception("There is no projection " + projection_name + " in table", ErrorCodes::NO_SUCH_PROJECTION_IN_TABLE);
|
|
|
|
|
2021-04-26 16:06:18 +00:00
|
|
|
return *(it->second);
|
2021-02-10 14:12:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void ProjectionsDescription::add(ProjectionDescription && projection, const String & after_projection, bool first, bool if_not_exists)
|
|
|
|
{
|
|
|
|
if (has(projection.name))
|
|
|
|
{
|
|
|
|
if (if_not_exists)
|
|
|
|
return;
|
|
|
|
throw Exception(
|
|
|
|
"Cannot add projection " + projection.name + ": projection with this name already exists", ErrorCodes::ILLEGAL_PROJECTION);
|
|
|
|
}
|
|
|
|
|
|
|
|
auto insert_it = projections.cend();
|
|
|
|
|
|
|
|
if (first)
|
|
|
|
insert_it = projections.cbegin();
|
|
|
|
else if (!after_projection.empty())
|
|
|
|
{
|
|
|
|
auto it = std::find_if(projections.cbegin(), projections.cend(), [&after_projection](const auto & projection_)
|
|
|
|
{
|
|
|
|
return projection_.name == after_projection;
|
|
|
|
});
|
|
|
|
if (it != projections.cend())
|
|
|
|
++it;
|
|
|
|
insert_it = it;
|
|
|
|
}
|
|
|
|
|
2021-04-26 16:06:18 +00:00
|
|
|
auto it = projections.insert(insert_it, std::move(projection));
|
|
|
|
map[it->name] = it;
|
2021-02-10 14:12:49 +00:00
|
|
|
}
|
|
|
|
|
2021-05-28 13:27:36 +00:00
|
|
|
void ProjectionsDescription::remove(const String & projection_name, bool if_exists)
|
2021-02-10 14:12:49 +00:00
|
|
|
{
|
2021-04-26 16:06:18 +00:00
|
|
|
auto it = map.find(projection_name);
|
|
|
|
if (it == map.end())
|
2021-05-28 13:27:36 +00:00
|
|
|
{
|
|
|
|
if (if_exists)
|
|
|
|
return;
|
2021-02-10 14:12:49 +00:00
|
|
|
throw Exception("There is no projection " + projection_name + " in table.", ErrorCodes::NO_SUCH_PROJECTION_IN_TABLE);
|
2021-05-28 13:27:36 +00:00
|
|
|
}
|
2021-04-26 16:06:18 +00:00
|
|
|
|
|
|
|
projections.erase(it->second);
|
|
|
|
map.erase(it);
|
2021-02-10 14:12:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
ExpressionActionsPtr
|
|
|
|
ProjectionsDescription::getSingleExpressionForProjections(const ColumnsDescription & columns, ContextPtr query_context) const
|
|
|
|
{
|
|
|
|
ASTPtr combined_expr_list = std::make_shared<ASTExpressionList>();
|
|
|
|
for (const auto & projection : projections)
|
|
|
|
for (const auto & projection_expr : projection.query_ast->children)
|
|
|
|
combined_expr_list->children.push_back(projection_expr->clone());
|
|
|
|
|
|
|
|
auto syntax_result = TreeRewriter(query_context).analyze(combined_expr_list, columns.getAllPhysical());
|
|
|
|
return ExpressionAnalyzer(combined_expr_list, syntax_result, query_context).getActions(false);
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|