2024-01-09 06:33:48 +00:00
|
|
|
#include <Interpreters/InterpreterFactory.h>
|
2022-07-14 11:20:16 +00:00
|
|
|
#include <Interpreters/InterpreterSelectQueryAnalyzer.h>
|
|
|
|
|
|
|
|
#include <Parsers/ASTSelectWithUnionQuery.h>
|
|
|
|
#include <Parsers/ASTSelectQuery.h>
|
|
|
|
#include <Parsers/ASTExpressionList.h>
|
2022-07-18 17:20:28 +00:00
|
|
|
#include <Parsers/ASTSubquery.h>
|
2022-07-14 11:20:16 +00:00
|
|
|
|
2023-02-02 18:25:32 +00:00
|
|
|
#include <DataTypes/DataTypesNumber.h>
|
|
|
|
|
2022-10-07 13:20:45 +00:00
|
|
|
#include <Processors/QueryPlan/IQueryPlanStep.h>
|
2022-07-14 11:20:16 +00:00
|
|
|
#include <Processors/QueryPlan/QueryPlan.h>
|
|
|
|
#include <Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h>
|
|
|
|
#include <QueryPipeline/QueryPipelineBuilder.h>
|
|
|
|
|
2023-02-11 11:10:53 +00:00
|
|
|
#include <Storages/IStorage.h>
|
|
|
|
|
|
|
|
#include <Analyzer/QueryTreeBuilder.h>
|
|
|
|
#include <Analyzer/QueryTreePassManager.h>
|
2023-02-15 14:03:52 +00:00
|
|
|
#include <Analyzer/IdentifierNode.h>
|
2023-02-11 11:10:53 +00:00
|
|
|
#include <Analyzer/QueryNode.h>
|
|
|
|
#include <Analyzer/UnionNode.h>
|
|
|
|
#include <Analyzer/TableNode.h>
|
2023-02-15 14:03:52 +00:00
|
|
|
#include <Analyzer/TableFunctionNode.h>
|
2023-02-11 11:10:53 +00:00
|
|
|
#include <Analyzer/Utils.h>
|
|
|
|
|
2022-07-14 11:20:16 +00:00
|
|
|
#include <Interpreters/Context.h>
|
2022-10-12 11:26:02 +00:00
|
|
|
#include <Interpreters/QueryLog.h>
|
2022-07-14 11:20:16 +00:00
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
|
|
|
extern const int UNSUPPORTED_METHOD;
|
|
|
|
}
|
|
|
|
|
2022-08-24 09:21:03 +00:00
|
|
|
namespace
|
2022-08-15 16:34:10 +00:00
|
|
|
{
|
|
|
|
|
2022-08-24 09:21:03 +00:00
|
|
|
ASTPtr normalizeAndValidateQuery(const ASTPtr & query)
|
2022-08-15 16:34:10 +00:00
|
|
|
{
|
2022-08-24 09:21:03 +00:00
|
|
|
if (query->as<ASTSelectWithUnionQuery>() || query->as<ASTSelectQuery>())
|
2022-08-15 16:34:10 +00:00
|
|
|
{
|
2022-08-24 09:21:03 +00:00
|
|
|
return query;
|
2022-08-15 16:34:10 +00:00
|
|
|
}
|
2022-08-24 09:21:03 +00:00
|
|
|
else if (auto * subquery = query->as<ASTSubquery>())
|
2022-08-15 16:34:10 +00:00
|
|
|
{
|
2022-08-24 09:21:03 +00:00
|
|
|
return subquery->children[0];
|
2022-08-15 16:34:10 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
|
2022-08-24 09:21:03 +00:00
|
|
|
"Expected ASTSelectWithUnionQuery or ASTSelectQuery. Actual {}",
|
|
|
|
query->formatForErrorMessage());
|
2022-08-20 14:01:50 +00:00
|
|
|
}
|
2022-08-15 16:34:10 +00:00
|
|
|
}
|
|
|
|
|
2023-02-02 18:25:32 +00:00
|
|
|
ContextMutablePtr buildContext(const ContextPtr & context, const SelectQueryOptions & select_query_options)
|
|
|
|
{
|
|
|
|
auto result_context = Context::createCopy(context);
|
|
|
|
|
|
|
|
if (select_query_options.shard_num)
|
|
|
|
result_context->addSpecialScalar(
|
|
|
|
"_shard_num",
|
|
|
|
Block{{DataTypeUInt32().createColumnConst(1, *select_query_options.shard_num), std::make_shared<DataTypeUInt32>(), "_shard_num"}});
|
|
|
|
if (select_query_options.shard_count)
|
|
|
|
result_context->addSpecialScalar(
|
|
|
|
"_shard_count",
|
|
|
|
Block{{DataTypeUInt32().createColumnConst(1, *select_query_options.shard_count), std::make_shared<DataTypeUInt32>(), "_shard_count"}});
|
|
|
|
|
|
|
|
return result_context;
|
|
|
|
}
|
|
|
|
|
2023-02-14 11:20:01 +00:00
|
|
|
void replaceStorageInQueryTree(QueryTreeNodePtr & query_tree, const ContextPtr & context, const StoragePtr & storage)
|
2023-02-11 11:10:53 +00:00
|
|
|
{
|
2024-01-23 10:59:41 +00:00
|
|
|
auto nodes = extractAllTableReferences(query_tree);
|
2024-01-18 17:59:23 +00:00
|
|
|
IQueryTreeNode::ReplacementMap replacement_map;
|
2024-01-18 18:10:54 +00:00
|
|
|
|
2024-01-18 17:59:23 +00:00
|
|
|
for (auto & node : nodes)
|
2023-02-11 11:10:53 +00:00
|
|
|
{
|
2024-01-18 17:59:23 +00:00
|
|
|
auto & table_node = node->as<TableNode &>();
|
2023-02-14 11:20:01 +00:00
|
|
|
|
2024-01-18 17:59:23 +00:00
|
|
|
/// Don't replace storage if table name differs
|
|
|
|
if (table_node.getStorageID().getFullNameNotQuoted() != storage->getStorageID().getFullNameNotQuoted())
|
|
|
|
continue;
|
2023-06-11 23:58:49 +00:00
|
|
|
|
2024-01-18 17:59:23 +00:00
|
|
|
auto replacement_table_expression = std::make_shared<TableNode>(storage, context);
|
2023-02-15 14:03:52 +00:00
|
|
|
|
2024-01-18 17:59:23 +00:00
|
|
|
if (auto table_expression_modifiers = table_node.getTableExpressionModifiers())
|
|
|
|
replacement_table_expression->setTableExpressionModifiers(*table_expression_modifiers);
|
2023-02-15 14:03:52 +00:00
|
|
|
|
2024-01-18 17:59:23 +00:00
|
|
|
replacement_map.emplace(node.get(), std::move(replacement_table_expression));
|
|
|
|
}
|
|
|
|
query_tree = query_tree->cloneAndReplace(replacement_map);
|
2023-02-11 11:10:53 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
QueryTreeNodePtr buildQueryTreeAndRunPasses(const ASTPtr & query,
|
|
|
|
const SelectQueryOptions & select_query_options,
|
|
|
|
const ContextPtr & context,
|
|
|
|
const StoragePtr & storage)
|
2022-08-21 11:46:07 +00:00
|
|
|
{
|
2022-09-27 15:04:03 +00:00
|
|
|
auto query_tree = buildQueryTree(query, context);
|
2022-08-21 11:46:07 +00:00
|
|
|
|
2022-08-24 09:21:03 +00:00
|
|
|
QueryTreePassManager query_tree_pass_manager(context);
|
2023-06-14 21:34:43 +00:00
|
|
|
addQueryTreePasses(query_tree_pass_manager);
|
2022-12-23 17:45:28 +00:00
|
|
|
|
2023-06-12 13:51:46 +00:00
|
|
|
/// We should not apply any query tree level optimizations on shards
|
|
|
|
/// because it can lead to a changed header.
|
2023-06-05 12:32:08 +00:00
|
|
|
if (select_query_options.ignore_ast_optimizations
|
|
|
|
|| context->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY)
|
2024-01-15 16:02:37 +00:00
|
|
|
query_tree_pass_manager.runOnlyResolve(query_tree);
|
2022-12-23 17:45:28 +00:00
|
|
|
else
|
|
|
|
query_tree_pass_manager.run(query_tree);
|
2024-01-18 18:10:54 +00:00
|
|
|
|
|
|
|
if (storage)
|
2023-02-14 11:20:01 +00:00
|
|
|
replaceStorageInQueryTree(query_tree, context, storage);
|
2024-01-18 18:10:54 +00:00
|
|
|
|
2022-08-24 09:21:03 +00:00
|
|
|
return query_tree;
|
2022-08-15 16:34:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
2022-07-18 17:20:28 +00:00
|
|
|
InterpreterSelectQueryAnalyzer::InterpreterSelectQueryAnalyzer(
|
|
|
|
const ASTPtr & query_,
|
2022-12-23 17:45:28 +00:00
|
|
|
const ContextPtr & context_,
|
|
|
|
const SelectQueryOptions & select_query_options_)
|
2022-12-06 09:44:38 +00:00
|
|
|
: query(normalizeAndValidateQuery(query_))
|
2023-02-02 18:25:32 +00:00
|
|
|
, context(buildContext(context_, select_query_options_))
|
2022-07-14 11:20:16 +00:00
|
|
|
, select_query_options(select_query_options_)
|
2023-02-11 11:10:53 +00:00
|
|
|
, query_tree(buildQueryTreeAndRunPasses(query, select_query_options, context, nullptr /*storage*/))
|
2023-02-15 14:03:52 +00:00
|
|
|
, planner(query_tree, select_query_options)
|
2023-02-11 11:10:53 +00:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
InterpreterSelectQueryAnalyzer::InterpreterSelectQueryAnalyzer(
|
|
|
|
const ASTPtr & query_,
|
|
|
|
const ContextPtr & context_,
|
2023-02-18 16:06:00 +00:00
|
|
|
const StoragePtr & storage_,
|
|
|
|
const SelectQueryOptions & select_query_options_)
|
2023-02-11 11:10:53 +00:00
|
|
|
: query(normalizeAndValidateQuery(query_))
|
|
|
|
, context(buildContext(context_, select_query_options_))
|
|
|
|
, select_query_options(select_query_options_)
|
2023-02-14 11:20:01 +00:00
|
|
|
, query_tree(buildQueryTreeAndRunPasses(query, select_query_options, context, storage_))
|
2023-02-15 14:03:52 +00:00
|
|
|
, planner(query_tree, select_query_options)
|
2022-07-14 11:20:16 +00:00
|
|
|
{
|
2022-07-18 17:20:28 +00:00
|
|
|
}
|
|
|
|
|
2022-08-31 15:21:17 +00:00
|
|
|
InterpreterSelectQueryAnalyzer::InterpreterSelectQueryAnalyzer(
|
|
|
|
const QueryTreeNodePtr & query_tree_,
|
2022-12-23 17:45:28 +00:00
|
|
|
const ContextPtr & context_,
|
|
|
|
const SelectQueryOptions & select_query_options_)
|
2022-12-06 09:44:38 +00:00
|
|
|
: query(query_tree_->toAST())
|
2023-02-02 18:25:32 +00:00
|
|
|
, context(buildContext(context_, select_query_options_))
|
2022-08-31 15:21:17 +00:00
|
|
|
, select_query_options(select_query_options_)
|
2022-12-23 17:45:28 +00:00
|
|
|
, query_tree(query_tree_)
|
2023-08-11 23:04:08 +00:00
|
|
|
, planner(query_tree_, select_query_options)
|
2023-01-26 10:52:40 +00:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
Block InterpreterSelectQueryAnalyzer::getSampleBlock(const ASTPtr & query,
|
|
|
|
const ContextPtr & context,
|
|
|
|
const SelectQueryOptions & select_query_options)
|
2022-08-31 15:21:17 +00:00
|
|
|
{
|
2023-01-26 10:52:40 +00:00
|
|
|
auto select_query_options_copy = select_query_options;
|
|
|
|
select_query_options_copy.only_analyze = true;
|
|
|
|
InterpreterSelectQueryAnalyzer interpreter(query, context, select_query_options_copy);
|
|
|
|
|
|
|
|
return interpreter.getSampleBlock();
|
|
|
|
}
|
|
|
|
|
2023-02-18 16:06:00 +00:00
|
|
|
Block InterpreterSelectQueryAnalyzer::getSampleBlock(const QueryTreeNodePtr & query_tree,
|
2023-01-26 10:52:40 +00:00
|
|
|
const ContextPtr & context,
|
|
|
|
const SelectQueryOptions & select_query_options)
|
|
|
|
{
|
|
|
|
auto select_query_options_copy = select_query_options;
|
|
|
|
select_query_options_copy.only_analyze = true;
|
2023-11-19 14:28:33 +00:00
|
|
|
InterpreterSelectQueryAnalyzer interpreter(query_tree, context, select_query_options_copy);
|
2023-01-26 10:52:40 +00:00
|
|
|
|
|
|
|
return interpreter.getSampleBlock();
|
2022-08-31 15:21:17 +00:00
|
|
|
}
|
|
|
|
|
2022-08-24 09:21:03 +00:00
|
|
|
Block InterpreterSelectQueryAnalyzer::getSampleBlock()
|
2022-07-18 17:20:28 +00:00
|
|
|
{
|
2022-08-24 14:50:53 +00:00
|
|
|
planner.buildQueryPlanIfNeeded();
|
2022-08-24 09:21:03 +00:00
|
|
|
return planner.getQueryPlan().getCurrentDataStream().header;
|
2022-07-14 11:20:16 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
BlockIO InterpreterSelectQueryAnalyzer::execute()
|
|
|
|
{
|
2023-02-02 18:25:32 +00:00
|
|
|
auto pipeline_builder = buildQueryPipeline();
|
2022-07-14 11:20:16 +00:00
|
|
|
|
2022-10-12 11:26:02 +00:00
|
|
|
BlockIO result;
|
2023-02-02 18:25:32 +00:00
|
|
|
result.pipeline = QueryPipelineBuilder::getPipeline(std::move(pipeline_builder));
|
2022-10-12 11:26:02 +00:00
|
|
|
|
2023-02-14 11:20:01 +00:00
|
|
|
if (!select_query_options.ignore_quota && select_query_options.to_stage == QueryProcessingStage::Complete)
|
2022-12-06 09:44:38 +00:00
|
|
|
result.pipeline.setQuota(context->getQuota());
|
2022-10-12 11:26:02 +00:00
|
|
|
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2023-03-04 17:46:40 +00:00
|
|
|
QueryPlan & InterpreterSelectQueryAnalyzer::getQueryPlan()
|
|
|
|
{
|
|
|
|
planner.buildQueryPlanIfNeeded();
|
|
|
|
return planner.getQueryPlan();
|
|
|
|
}
|
|
|
|
|
2022-10-12 11:26:02 +00:00
|
|
|
QueryPlan && InterpreterSelectQueryAnalyzer::extractQueryPlan() &&
|
|
|
|
{
|
|
|
|
planner.buildQueryPlanIfNeeded();
|
|
|
|
return std::move(planner).extractQueryPlan();
|
|
|
|
}
|
2022-07-14 11:20:16 +00:00
|
|
|
|
2023-01-20 11:19:16 +00:00
|
|
|
QueryPipelineBuilder InterpreterSelectQueryAnalyzer::buildQueryPipeline()
|
2022-12-15 12:03:09 +00:00
|
|
|
{
|
|
|
|
planner.buildQueryPlanIfNeeded();
|
|
|
|
auto & query_plan = planner.getQueryPlan();
|
|
|
|
|
2023-02-02 18:25:32 +00:00
|
|
|
auto optimization_settings = QueryPlanOptimizationSettings::fromContext(context);
|
|
|
|
auto build_pipeline_settings = BuildQueryPipelineSettings::fromContext(context);
|
2022-12-15 12:03:09 +00:00
|
|
|
|
2023-01-20 11:19:16 +00:00
|
|
|
return std::move(*query_plan.buildQueryPipeline(optimization_settings, build_pipeline_settings));
|
2022-12-15 12:03:09 +00:00
|
|
|
}
|
|
|
|
|
2022-12-14 17:43:49 +00:00
|
|
|
void InterpreterSelectQueryAnalyzer::addStorageLimits(const StorageLimitsList & storage_limits)
|
|
|
|
{
|
|
|
|
planner.addStorageLimits(storage_limits);
|
|
|
|
}
|
|
|
|
|
2023-11-09 15:13:57 +00:00
|
|
|
void InterpreterSelectQueryAnalyzer::extendQueryLogElemImpl(QueryLogElement & elem, const ASTPtr & /*ast*/, ContextPtr /*context*/) const
|
|
|
|
{
|
|
|
|
for (const auto & used_row_policy : planner.getUsedRowPolicies())
|
|
|
|
elem.used_row_policies.emplace(used_row_policy);
|
|
|
|
}
|
|
|
|
|
2024-01-09 06:33:48 +00:00
|
|
|
void registerInterpreterSelectQueryAnalyzer(InterpreterFactory & factory)
|
|
|
|
{
|
|
|
|
auto create_fn = [] (const InterpreterFactory::Arguments & args)
|
|
|
|
{
|
|
|
|
return std::make_unique<InterpreterSelectQueryAnalyzer>(args.query, args.context, args.options);
|
|
|
|
};
|
|
|
|
factory.registerInterpreter("InterpreterSelectQueryAnalyzer", create_fn);
|
|
|
|
}
|
|
|
|
|
2022-07-14 11:20:16 +00:00
|
|
|
}
|