thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 23:52:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make it work

Browse Source
This commit is contained in:
Amos Bird 2024-02-22 02:39:54 +08:00
parent 12914d2268
commit 207d36d7c1
No known key found for this signature in database
GPG Key ID: 80D430DCBECFEDB4
16 changed files with 71 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/Analyzer/Utils.cpp
View File

@ -388,7 +388,7 @@ QueryTreeNodes extractAllTableReferences(const QueryTreeNodePtr & tree)
return result;
}
QueryTreeNodes extractTableExpressions(const QueryTreeNodePtr & join_tree_node, bool add_array_join)
QueryTreeNodes extractTableExpressions(const QueryTreeNodePtr & join_tree_node, bool add_array_join, bool recursive)
{
QueryTreeNodes result;
@ -406,15 +406,28 @@ QueryTreeNodes extractTableExpressions(const QueryTreeNodePtr & join_tree_node,
{
case QueryTreeNodeType::TABLE:
[[fallthrough]];
case QueryTreeNodeType::QUERY:
[[fallthrough]];
case QueryTreeNodeType::UNION:
[[fallthrough]];
case QueryTreeNodeType::TABLE_FUNCTION:
{
result.push_back(std::move(node_to_process));
break;
}
case QueryTreeNodeType::QUERY:
{
if (recursive)
nodes_to_process.push_back(node_to_process->as<QueryNode>()->getJoinTree());
result.push_back(std::move(node_to_process));
break;
}
case QueryTreeNodeType::UNION:
{
if (recursive)
{
for (const auto & union_node : node_to_process->as<UnionNode>()->getQueries().getNodes())
nodes_to_process.push_back(union_node);
}
result.push_back(std::move(node_to_process));
break;
}
case QueryTreeNodeType::ARRAY_JOIN:
{
auto & array_join_node = node_to_process->as<ArrayJoinNode &>();

2
src/Analyzer/Utils.h
View File

@ -54,7 +54,7 @@ void addTableExpressionOrJoinIntoTablesInSelectQuery(ASTPtr & tables_in_select_q
QueryTreeNodes extractAllTableReferences(const QueryTreeNodePtr & tree);
/// Extract table, table function, query, union from join tree.
QueryTreeNodes extractTableExpressions(const QueryTreeNodePtr & join_tree_node, bool add_array_join = false);
QueryTreeNodes extractTableExpressions(const QueryTreeNodePtr & join_tree_node, bool add_array_join = false, bool recursive = false);
/// Extract left table expression from join tree.
QueryTreeNodePtr extractLeftTableExpression(const QueryTreeNodePtr & join_tree_node);

2
src/Interpreters/InterpreterSelectQuery.cpp
View File

@ -792,8 +792,6 @@ InterpreterSelectQuery::InterpreterSelectQuery(
/// Reuse already built sets for multiple passes of analysis
prepared_sets = query_analyzer->getPreparedSets();
/// Do not try move conditions to PREWHERE for the second time.
/// Otherwise, we won't be able to fallback from inefficient PREWHERE to WHERE later.
analyze();
}

3
src/Planner/CollectTableExpressionData.cpp
View File

@ -252,7 +252,8 @@ void checkStorageSupportPrewhere(const QueryTreeNodePtr & table_expression)
void collectTableExpressionData(QueryTreeNodePtr & query_node, PlannerContextPtr & planner_context)
{
auto & query_node_typed = query_node->as<QueryNode &>();
auto table_expressions_nodes = extractTableExpressions(query_node_typed.getJoinTree());
auto table_expressions_nodes
= extractTableExpressions(query_node_typed.getJoinTree(), false /* add_array_join */, true /* recursive */);
for (auto & table_expression_node : table_expressions_nodes)
{

5
src/Planner/Planner.cpp
View File

@ -176,7 +176,7 @@ void collectFiltersForAnalysis(const QueryTreeNodePtr & query_tree, const Planne
return;
ResultReplacementMap replacement_map;
auto updated_query_tree = replaceTableExpressionsWithDummyTables(query_tree, planner_context->getQueryContext(), &replacement_map);
auto updated_query_tree = replaceTableExpressionsWithDummyTables(query_tree, *planner_context, &replacement_map);
std::unordered_map<const IStorage *, TableExpressionData *> dummy_storage_to_table_expression_data;
@ -1359,10 +1359,11 @@ void Planner::buildPlanForQueryNode()
}
}
bool top_level = planner_context->getTableExpressionNodeToData().empty();
collectTableExpressionData(query_tree, planner_context);
checkStoragesSupportTransactions(planner_context);
if (!select_query_options.only_analyze)
if (!select_query_options.only_analyze && top_level)
collectFiltersForAnalysis(query_tree, planner_context);
if (query_context->canUseTaskBasedParallelReplicas())

3
src/Planner/PlannerContext.cpp
View File

@ -44,6 +44,9 @@ bool GlobalPlannerContext::hasColumnIdentifier(const ColumnIdentifier & column_i
PlannerContext::PlannerContext(ContextMutablePtr query_context_, GlobalPlannerContextPtr global_planner_context_)
: query_context(std::move(query_context_))
, global_planner_context(std::move(global_planner_context_))
, column_node_to_column_identifier(global_planner_context->column_node_to_column_identifier)
, table_expression_node_to_data(global_planner_context->table_expression_node_to_data)
, prepared_sets(global_planner_context->prepared_sets)
{}
TableExpressionData & PlannerContext::getOrCreateTableExpressionData(const QueryTreeNodePtr & table_expression_node)

17
src/Planner/PlannerContext.h
View File

@ -55,6 +55,17 @@ public:
private:
std::unordered_set<ColumnIdentifier> column_identifiers;
friend class PlannerContext;
/// Column node to column identifier
std::unordered_map<QueryTreeNodePtr, ColumnIdentifier> column_node_to_column_identifier;
/// Table expression node to data
std::unordered_map<QueryTreeNodePtr, TableExpressionData> table_expression_node_to_data;
/// Set key to set
PreparedSets prepared_sets;
};
using GlobalPlannerContextPtr = std::shared_ptr<GlobalPlannerContext>;
@ -158,13 +169,13 @@ private:
GlobalPlannerContextPtr global_planner_context;
/// Column node to column identifier
std::unordered_map<QueryTreeNodePtr, ColumnIdentifier> column_node_to_column_identifier;
std::unordered_map<QueryTreeNodePtr, ColumnIdentifier> & column_node_to_column_identifier;
/// Table expression node to data
std::unordered_map<QueryTreeNodePtr, TableExpressionData> table_expression_node_to_data;
std::unordered_map<QueryTreeNodePtr, TableExpressionData> & table_expression_node_to_data;
/// Set key to set
PreparedSets prepared_sets;
PreparedSets & prepared_sets;
};
using PlannerContextPtr = std::shared_ptr<PlannerContext>;

60
src/Planner/Utils.cpp
View File

@ -386,70 +386,34 @@ QueryTreeNodePtr mergeConditionNodes(const QueryTreeNodes & condition_nodes, con
return function_node;
}
QueryTreeNodePtr replaceTableExpressionsWithDummyTables(const QueryTreeNodePtr & query_node,
const ContextPtr & context,
//PlannerContext & planner_context,
ResultReplacementMap * result_replacement_map)
QueryTreeNodePtr replaceTableExpressionsWithDummyTables(
const QueryTreeNodePtr & query_node, const PlannerContext & planner_context, ResultReplacementMap * result_replacement_map)
{
auto & query_node_typed = query_node->as<QueryNode &>();
auto table_expressions = extractTableExpressions(query_node_typed.getJoinTree());
std::unordered_map<const IQueryTreeNode *, QueryTreeNodePtr> replacement_map;
size_t subquery_index = 0;
for (auto & table_expression : table_expressions)
for (const auto & [table_expression, _dummy] : planner_context.getTableExpressionNodeToData())
{
auto * table_node = table_expression->as<TableNode>();
auto * table_function_node = table_expression->as<TableFunctionNode>();
auto * subquery_node = table_expression->as<QueryNode>();
auto * union_node = table_expression->as<UnionNode>();
StoragePtr storage_dummy;
if (table_node || table_function_node)
{
const auto & storage_snapshot = table_node ? table_node->getStorageSnapshot() : table_function_node->getStorageSnapshot();
auto get_column_options = GetColumnsOptions(GetColumnsOptions::All).withExtendedObjects().withVirtuals();
storage_dummy = std::make_shared<StorageDummy>(
StoragePtr storage_dummy = std::make_shared<StorageDummy>(
storage_snapshot->storage.getStorageID(),
ColumnsDescription(storage_snapshot->getColumns(get_column_options)),
storage_snapshot);
auto dummy_table_node = std::make_shared<TableNode>(std::move(storage_dummy), planner_context.getQueryContext());
if (result_replacement_map)
result_replacement_map->emplace(table_expression, dummy_table_node);
dummy_table_node->setAlias(table_expression->getAlias());
replacement_map.emplace(table_expression.get(), std::move(dummy_table_node));
}
else if (subquery_node || union_node)
{
const auto & subquery_projection_columns
= subquery_node ? subquery_node->getProjectionColumns() : union_node->computeProjectionColumns();
NameSet unique_column_names;
NamesAndTypes storage_dummy_columns;
storage_dummy_columns.reserve(subquery_projection_columns.size());
for (const auto & projection_column : subquery_projection_columns)
{
auto [_, inserted] = unique_column_names.insert(projection_column.name);
if (inserted)
storage_dummy_columns.emplace_back(projection_column);
}
storage_dummy = std::make_shared<StorageDummy>(
StorageID{"dummy", "subquery_" + std::to_string(subquery_index)},
ColumnsDescription::fromNamesAndTypes(storage_dummy_columns));
++subquery_index;
}
auto dummy_table_node = std::make_shared<TableNode>(std::move(storage_dummy), context);
if (result_replacement_map)
result_replacement_map->emplace(table_expression, dummy_table_node);
dummy_table_node->setAlias(table_expression->getAlias());
// auto & src_table_expression_data = planner_context.getOrCreateTableExpressionData(table_expression);
// auto & dst_table_expression_data = planner_context.getOrCreateTableExpressionData(dummy_table_node);
// dst_table_expression_data = src_table_expression_data;
replacement_map.emplace(table_expression.get(), std::move(dummy_table_node));
}
return query_node->cloneAndReplace(replacement_map);

5
src/Planner/Utils.h
View File

@ -70,9 +70,8 @@ QueryTreeNodePtr mergeConditionNodes(const QueryTreeNodes & condition_nodes, con
/// Replace table expressions from query JOIN TREE with dummy tables
using ResultReplacementMap = std::unordered_map<QueryTreeNodePtr, QueryTreeNodePtr>;
QueryTreeNodePtr replaceTableExpressionsWithDummyTables(const QueryTreeNodePtr & query_node,
const ContextPtr & context,
ResultReplacementMap * result_replacement_map = nullptr);
QueryTreeNodePtr replaceTableExpressionsWithDummyTables(
const QueryTreeNodePtr & query_node, const PlannerContext & planner_context, ResultReplacementMap * result_replacement_map = nullptr);
/// Build subquery to read specified columns from table expression
QueryTreeNodePtr buildSubqueryToReadColumnsFromTableExpression(const NamesAndTypes & columns,

2
src/Processors/QueryPlan/IQueryPlanStep.h
View File

@ -2,7 +2,6 @@
#include <Core/Block.h>
#include <Core/SortDescription.h>
#include <Processors/QueryPlan/BuildQueryPipelineSettings.h>
#include <Processors/QueryPlan/QueryPlan.h>
namespace DB
{
@ -65,6 +64,7 @@ public:
using DataStreams = std::vector<DataStream>;
class QueryPlan;
using QueryPlanRawPtrs = std::list<QueryPlan *>;
/// Single step of query plan.

2
src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyCondition.cpp
View File

@ -36,6 +36,8 @@ void optimizePrimaryKeyCondition(const Stack & stack)
else
break;
}
source_step_with_filter->applyFilters();
}
}

11
src/Processors/QueryPlan/Optimizations/optimizeTree.cpp
View File

@ -114,10 +114,12 @@ void optimizeTreeSecondPass(const QueryPlanOptimizationSettings & optimization_s
while (!stack.empty())
{
/// NOTE: optimizePrewhere can modify the stack.
optimizePrewhere(stack, nodes);
optimizePrimaryKeyCondition(stack);
/// NOTE: optimizePrewhere can modify the stack.
/// Prewhere optimization relies on PK optimization (getConditionEstimatorByPredicate)
optimizePrewhere(stack, nodes);
auto & frame = stack.back();
if (frame.next_child == 0)
@ -223,11 +225,6 @@ void optimizeTreeThirdPass(QueryPlan & plan, QueryPlan::Node & root, QueryPlan::
continue;
}
if (auto * source_step_with_filter = dynamic_cast<SourceStepWithFilter *>(frame.node->step.get()))
{
source_step_with_filter->applyFilters();
}
addPlansForSets(plan, *frame.node, nodes);
stack.pop_back();

1
src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp
View File

@ -589,7 +589,6 @@ bool optimizeUseAggregateProjections(QueryPlan::Node & node, QueryPlan::Nodes &
}
else if (!candidates.real.empty())
{
reading->applyFilters();
auto ordinary_reading_select_result = reading->selectRangesToRead(parts, alter_conversions);
size_t ordinary_reading_marks = ordinary_reading_select_result->selected_marks;

1
src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp
View File

@ -141,7 +141,6 @@ bool optimizeUseNormalProjections(Stack & stack, QueryPlan::Nodes & nodes)
const auto & query_info = reading->getQueryInfo();
MergeTreeDataSelectExecutor reader(reading->getMergeTreeData());
reading->applyFilters();
auto ordinary_reading_select_result = reading->selectRangesToRead(parts, alter_conversions);
size_t ordinary_reading_marks = ordinary_reading_select_result->selected_marks;

2
src/Processors/QueryPlan/SourceStepWithFilter.cpp
View File

@ -53,7 +53,7 @@ Block SourceStepWithFilter::applyPrewhereActions(Block block, const PrewhereInfo
}
else if (prewhere_info->need_filter)
{
if (const auto * type = typeid_cast<const DataTypeNullable *>(prewhere_column.type.get()); type->onlyNull())
if (const auto * type = typeid_cast<const DataTypeNullable *>(prewhere_column.type.get()); type && type->onlyNull())
{
prewhere_column.column = prewhere_column.type->createColumnConst(block.rows(), Null());
}

16
tests/queries/0_stateless/02882_primary_key_index_in_function_different_types.reference
View File

@ -7,7 +7,7 @@ CreatingSets
Keys:
id
value
Condition: and((id in (-Inf, 10]), (value in 1-element set))
Condition: and((value in 1-element set), (id in (-Inf, 10]))
Parts: 1/1
Granules: 1/1
CreatingSets
@ -19,7 +19,7 @@ CreatingSets
Keys:
id
value
Condition: and((id in (-Inf, 10]), (value in 1-element set))
Condition: and((value in 1-element set), (id in (-Inf, 10]))
Parts: 1/1
Granules: 1/1
CreatingSets
@ -31,7 +31,7 @@ CreatingSets
Keys:
id
value
Condition: and((id in (-Inf, 10]), (value in 5-element set))
Condition: and((value in 5-element set), (id in (-Inf, 10]))
Parts: 1/1
Granules: 1/1
CreatingSets
@ -43,7 +43,7 @@ CreatingSets
Keys:
id
value
Condition: and((id in (-Inf, 10]), (value in 5-element set))
Condition: and((value in 5-element set), (id in (-Inf, 10]))
Parts: 1/1
Granules: 1/1
CreatingSets
@ -55,7 +55,7 @@ CreatingSets
Keys:
id
value
Condition: and((id in (-Inf, 10]), (value in 1-element set))
Condition: and((value in 1-element set), (id in (-Inf, 10]))
Parts: 1/1
Granules: 1/1
CreatingSets
@ -67,7 +67,7 @@ CreatingSets
Keys:
id
value
Condition: and((id in (-Inf, 10]), (value in 1-element set))
Condition: and((value in 1-element set), (id in (-Inf, 10]))
Parts: 1/1
Granules: 1/1
CreatingSets
@ -79,7 +79,7 @@ CreatingSets
Keys:
id
value
Condition: and((id in (-Inf, 10]), (value in 5-element set))
Condition: and((value in 5-element set), (id in (-Inf, 10]))
Parts: 1/1
Granules: 1/1
CreatingSets
@ -91,6 +91,6 @@ CreatingSets
Keys:
id
value
Condition: and((id in (-Inf, 10]), (value in 5-element set))
Condition: and((value in 5-element set), (id in (-Inf, 10]))
Parts: 1/1
Granules: 1/1