diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 5faf8dd97c0..9b83d5660f7 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -7919,6 +7919,9 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier if (query_node_typed.hasWindow()) visitor.visit(query_node_typed.getWindowNode()); + if (query_node_typed.hasQualify()) + visitor.visit(query_node_typed.getQualify()); + if (query_node_typed.hasOrderBy()) visitor.visit(query_node_typed.getOrderByNode()); @@ -8067,6 +8070,9 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier if (query_node_typed.hasWindow()) resolveWindowNodeList(query_node_typed.getWindowNode(), scope); + if (query_node_typed.hasQualify()) + resolveExpressionNode(query_node_typed.getQualify(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + if (query_node_typed.hasOrderBy()) { replaceNodesWithPositionalArguments(query_node_typed.getOrderByNode(), query_node_typed.getProjection().getNodes(), scope); diff --git a/src/Analyzer/QueryNode.cpp b/src/Analyzer/QueryNode.cpp index f1361c328db..c0659e1998b 100644 --- a/src/Analyzer/QueryNode.cpp +++ b/src/Analyzer/QueryNode.cpp @@ -197,6 +197,12 @@ void QueryNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, s getWindow().dumpTreeImpl(buffer, format_state, indent + 4); } + if (hasQualify()) + { + buffer << '\n' << std::string(indent + 2, ' ') << "QUALIFY\n"; + getQualify()->dumpTreeImpl(buffer, format_state, indent + 4); + } + if (hasOrderBy()) { buffer << '\n' << std::string(indent + 2, ' ') << "ORDER BY\n"; @@ -381,6 +387,9 @@ ASTPtr QueryNode::toASTImpl(const ConvertToASTOptions & options) const if (hasWindow()) select_query->setExpression(ASTSelectQuery::Expression::WINDOW, getWindow().toAST(options)); + if (hasQualify()) + select_query->setExpression(ASTSelectQuery::Expression::QUALIFY, getQualify()->toAST(options)); + if (hasOrderBy()) select_query->setExpression(ASTSelectQuery::Expression::ORDER_BY, getOrderBy().toAST(options)); diff --git a/src/Analyzer/QueryNode.h b/src/Analyzer/QueryNode.h index af187df72a8..efabf604730 100644 --- a/src/Analyzer/QueryNode.h +++ b/src/Analyzer/QueryNode.h @@ -416,6 +416,24 @@ public: return children[window_child_index]; } + /// Returns true if query node QUALIFY section is not empty, false otherwise + bool hasQualify() const + { + return getQualify() != nullptr; + } + + /// Get QUALIFY section node + const QueryTreeNodePtr & getQualify() const + { + return children[qualify_child_index]; + } + + /// Get QUALIFY section node + QueryTreeNodePtr & getQualify() + { + return children[qualify_child_index]; + } + /// Returns true if query node ORDER BY section is not empty, false otherwise bool hasOrderBy() const { @@ -622,13 +640,14 @@ private: static constexpr size_t group_by_child_index = 5; static constexpr size_t having_child_index = 6; static constexpr size_t window_child_index = 7; - static constexpr size_t order_by_child_index = 8; - static constexpr size_t interpolate_child_index = 9; - static constexpr size_t limit_by_limit_child_index = 10; - static constexpr size_t limit_by_offset_child_index = 11; - static constexpr size_t limit_by_child_index = 12; - static constexpr size_t limit_child_index = 13; - static constexpr size_t offset_child_index = 14; + static constexpr size_t qualify_child_index = 8; + static constexpr size_t order_by_child_index = 9; + static constexpr size_t interpolate_child_index = 10; + static constexpr size_t limit_by_limit_child_index = 11; + static constexpr size_t limit_by_offset_child_index = 12; + static constexpr size_t limit_by_child_index = 13; + static constexpr size_t limit_child_index = 14; + static constexpr size_t offset_child_index = 15; static constexpr size_t children_size = offset_child_index + 1; }; diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index a4f20472432..876e583d393 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -330,6 +330,10 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_q if (window_list) current_query_tree->getWindowNode() = buildWindowList(window_list, current_context); + auto qualify_expression = select_query_typed.qualify(); + if (qualify_expression) + current_query_tree->getQualify() = buildExpression(qualify_expression, current_context); + auto select_order_by_list = select_query_typed.orderBy(); if (select_order_by_list) current_query_tree->getOrderByNode() = buildSortList(select_order_by_list, current_context); diff --git a/src/Analyzer/ValidationUtils.cpp b/src/Analyzer/ValidationUtils.cpp index af35632ab81..60cc1dd521f 100644 --- a/src/Analyzer/ValidationUtils.cpp +++ b/src/Analyzer/ValidationUtils.cpp @@ -56,6 +56,9 @@ void validateFilters(const QueryTreeNodePtr & query_node) if (query_node_typed.hasHaving()) validateFilter(query_node_typed.getHaving(), "HAVING", query_node); + + if (query_node_typed.hasQualify()) + validateFilter(query_node_typed.getQualify(), "QUALIFY", query_node); } namespace diff --git a/src/Client/Suggest.cpp b/src/Client/Suggest.cpp index f63dbc64d27..ebfa2e89ea1 100644 --- a/src/Client/Suggest.cpp +++ b/src/Client/Suggest.cpp @@ -47,7 +47,7 @@ Suggest::Suggest() "GRANT", "REVOKE", "OPTION", "ADMIN", "EXCEPT", "REPLACE", "IDENTIFIED", "HOST", "NAME", "READONLY", "WRITABLE", "PERMISSIVE", "FOR", "RESTRICTIVE", "RANDOMIZED", "INTERVAL", "LIMITS", "ONLY", "TRACKING", "IP", "REGEXP", "ILIKE", "CLEANUP", "APPEND", - "IGNORE NULLS", "RESPECT NULLS", "OVER", "PASTE"}); + "IGNORE NULLS", "RESPECT NULLS", "OVER", "PASTE", "WINDOW", "QUALIFY"}); } static String getLoadSuggestionQuery(Int32 suggestion_limit, bool basic_suggestion) diff --git a/src/Parsers/ASTSelectQuery.cpp b/src/Parsers/ASTSelectQuery.cpp index 586477e1cfd..f99933b7969 100644 --- a/src/Parsers/ASTSelectQuery.cpp +++ b/src/Parsers/ASTSelectQuery.cpp @@ -144,6 +144,12 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F window()->as().formatImplMultiline(s, state, frame); } + if (qualify()) + { + s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "QUALIFY " << (s.hilite ? hilite_none : ""); + qualify()->formatImpl(s, state, frame); + } + if (!order_by_all && orderBy()) { s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "ORDER BY" << (s.hilite ? hilite_none : ""); diff --git a/src/Parsers/ASTSelectQuery.h b/src/Parsers/ASTSelectQuery.h index eb171dc00ee..9fd6becbaaa 100644 --- a/src/Parsers/ASTSelectQuery.h +++ b/src/Parsers/ASTSelectQuery.h @@ -25,6 +25,7 @@ public: GROUP_BY, HAVING, WINDOW, + QUALIFY, ORDER_BY, LIMIT_BY_OFFSET, LIMIT_BY_LENGTH, @@ -55,6 +56,8 @@ public: return "HAVING"; case Expression::WINDOW: return "WINDOW"; + case Expression::QUALIFY: + return "QUALIFY"; case Expression::ORDER_BY: return "ORDER BY"; case Expression::LIMIT_BY_OFFSET: @@ -95,6 +98,7 @@ public: ASTPtr & refPrewhere() { return getExpression(Expression::PREWHERE); } ASTPtr & refWhere() { return getExpression(Expression::WHERE); } ASTPtr & refHaving() { return getExpression(Expression::HAVING); } + ASTPtr & refQualify() { return getExpression(Expression::QUALIFY); } ASTPtr with() const { return getExpression(Expression::WITH); } ASTPtr select() const { return getExpression(Expression::SELECT); } @@ -104,6 +108,7 @@ public: ASTPtr groupBy() const { return getExpression(Expression::GROUP_BY); } ASTPtr having() const { return getExpression(Expression::HAVING); } ASTPtr window() const { return getExpression(Expression::WINDOW); } + ASTPtr qualify() const { return getExpression(Expression::QUALIFY); } ASTPtr orderBy() const { return getExpression(Expression::ORDER_BY); } ASTPtr limitByOffset() const { return getExpression(Expression::LIMIT_BY_OFFSET); } ASTPtr limitByLength() const { return getExpression(Expression::LIMIT_BY_LENGTH); } @@ -113,7 +118,7 @@ public: ASTPtr settings() const { return getExpression(Expression::SETTINGS); } ASTPtr interpolate() const { return getExpression(Expression::INTERPOLATE); } - bool hasFiltration() const { return where() || prewhere() || having(); } + bool hasFiltration() const { return where() || prewhere() || having() || qualify(); } /// Set/Reset/Remove expression. void setExpression(Expression expr, ASTPtr && ast); diff --git a/src/Parsers/CommonParsers.h b/src/Parsers/CommonParsers.h index 2277e348b0f..12c452d38c4 100644 --- a/src/Parsers/CommonParsers.h +++ b/src/Parsers/CommonParsers.h @@ -507,6 +507,7 @@ namespace DB MR_MACROS(WHEN, "WHEN") \ MR_MACROS(WHERE, "WHERE") \ MR_MACROS(WINDOW, "WINDOW") \ + MR_MACROS(QUALIFY, "QUALIFY") \ MR_MACROS(WITH_ADMIN_OPTION, "WITH ADMIN OPTION") \ MR_MACROS(WITH_CHECK, "WITH CHECK") \ MR_MACROS(WITH_FILL, "WITH FILL") \ diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index dce0bc62d5b..a483ac92f39 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -1481,6 +1481,7 @@ const char * ParserAlias::restricted_keywords[] = "USING", "WHERE", "WINDOW", + "QUALIFY", "WITH", "INTERSECT", "EXCEPT", diff --git a/src/Parsers/ParserSelectQuery.cpp b/src/Parsers/ParserSelectQuery.cpp index ce15c7b03fd..e1ded0ab902 100644 --- a/src/Parsers/ParserSelectQuery.cpp +++ b/src/Parsers/ParserSelectQuery.cpp @@ -49,6 +49,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword s_totals(Keyword::TOTALS); ParserKeyword s_having(Keyword::HAVING); ParserKeyword s_window(Keyword::WINDOW); + ParserKeyword s_qualify(Keyword::QUALIFY); ParserKeyword s_order_by(Keyword::ORDER_BY); ParserKeyword s_limit(Keyword::LIMIT); ParserKeyword s_settings(Keyword::SETTINGS); @@ -86,6 +87,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ASTPtr group_expression_list; ASTPtr having_expression; ASTPtr window_list; + ASTPtr qualify_expression; ASTPtr order_expression_list; ASTPtr interpolate_expression_list; ASTPtr limit_by_length; @@ -266,6 +268,13 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } } + /// QUALIFY expr + if (s_qualify.ignore(pos, expected)) + { + if (!exp_elem.parse(pos, qualify_expression, expected)) + return false; + } + /// ORDER BY expr ASC|DESC COLLATE 'locale' list if (s_order_by.ignore(pos, expected)) { @@ -489,6 +498,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) select_query->setExpression(ASTSelectQuery::Expression::GROUP_BY, std::move(group_expression_list)); select_query->setExpression(ASTSelectQuery::Expression::HAVING, std::move(having_expression)); select_query->setExpression(ASTSelectQuery::Expression::WINDOW, std::move(window_list)); + select_query->setExpression(ASTSelectQuery::Expression::QUALIFY, std::move(qualify_expression)); select_query->setExpression(ASTSelectQuery::Expression::ORDER_BY, std::move(order_expression_list)); select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY_OFFSET, std::move(limit_by_offset)); select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY_LENGTH, std::move(limit_by_length)); diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index d75573c8d99..b21dd5203e8 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -1367,6 +1367,16 @@ void Planner::buildPlanForQueryNode() select_query_info.has_aggregates = hasAggregateFunctionNodes(query_tree); select_query_info.need_aggregate = query_node.hasGroupBy() || select_query_info.has_aggregates; + if (!select_query_info.has_window && query_node.hasQualify()) + { + if (query_node.hasHaving()) + query_node.getHaving() = mergeConditionNodes({query_node.getHaving(), query_node.getQualify()}, query_context); + else + query_node.getHaving() = query_node.getQualify(); + + query_node.getQualify() = {}; + } + if (!select_query_info.need_aggregate && query_node.hasHaving()) { if (query_node.hasWhere()) @@ -1636,6 +1646,9 @@ void Planner::buildPlanForQueryNode() addWindowSteps(query_plan, planner_context, window_analysis_result); } + if (expression_analysis_result.hasQualify()) + addFilterStep(query_plan, expression_analysis_result.getQualify(), "QUALIFY", result_actions_to_execute); + const auto & projection_analysis_result = expression_analysis_result.getProjection(); addExpressionStep(query_plan, projection_analysis_result.projection_actions, "Projection", result_actions_to_execute); diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp index dd3769ee10b..ad8db83d66c 100644 --- a/src/Planner/PlannerExpressionAnalysis.cpp +++ b/src/Planner/PlannerExpressionAnalysis.cpp @@ -513,6 +513,16 @@ PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNo if (window_analysis_result_optional) current_output_columns = actions_chain.getLastStepAvailableOutputColumns(); + std::optional qualify_analysis_result_optional; + std::optional qualify_action_step_index_optional; + + if (query_node.hasQualify()) + { + qualify_analysis_result_optional = analyzeFilter(query_node.getQualify(), current_output_columns, planner_context, actions_chain); + qualify_action_step_index_optional = actions_chain.getLastStepIndex(); + current_output_columns = actions_chain.getLastStepAvailableOutputColumns(); + } + auto projection_analysis_result = analyzeProjection(query_node, current_output_columns, planner_context, actions_chain); current_output_columns = actions_chain.getLastStepAvailableOutputColumns(); @@ -604,7 +614,7 @@ PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNo PlannerExpressionsAnalysisResult expressions_analysis_result(std::move(projection_analysis_result)); - if (where_action_step_index_optional && where_analysis_result_optional) + if (where_analysis_result_optional && where_action_step_index_optional) { auto & where_analysis_result = *where_analysis_result_optional; auto & where_actions_chain_node = actions_chain.at(*where_action_step_index_optional); @@ -615,7 +625,7 @@ PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNo if (aggregation_analysis_result_optional) expressions_analysis_result.addAggregation(std::move(*aggregation_analysis_result_optional)); - if (having_action_step_index_optional && having_analysis_result_optional) + if (having_analysis_result_optional && having_action_step_index_optional) { auto & having_analysis_result = *having_analysis_result_optional; auto & having_actions_chain_node = actions_chain.at(*having_action_step_index_optional); @@ -626,6 +636,14 @@ PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNo if (window_analysis_result_optional) expressions_analysis_result.addWindow(std::move(*window_analysis_result_optional)); + if (qualify_analysis_result_optional && qualify_action_step_index_optional) + { + auto & qualify_analysis_result = *qualify_analysis_result_optional; + auto & qualify_actions_chain_node = actions_chain.at(*qualify_action_step_index_optional); + qualify_analysis_result.remove_filter_column = !qualify_actions_chain_node->getChildRequiredOutputColumnsNames().contains(qualify_analysis_result.filter_column_name); + expressions_analysis_result.addQualify(std::move(qualify_analysis_result)); + } + if (sort_analysis_result_optional) expressions_analysis_result.addSort(std::move(*sort_analysis_result_optional)); diff --git a/src/Planner/PlannerExpressionAnalysis.h b/src/Planner/PlannerExpressionAnalysis.h index 792cfdec2ff..0773272e49a 100644 --- a/src/Planner/PlannerExpressionAnalysis.h +++ b/src/Planner/PlannerExpressionAnalysis.h @@ -129,6 +129,21 @@ public: window_analysis_result = std::move(window_analysis_result_); } + bool hasQualify() const + { + return qualify_analysis_result.filter_actions != nullptr; + } + + const FilterAnalysisResult & getQualify() const + { + return qualify_analysis_result; + } + + void addQualify(FilterAnalysisResult qualify_analysis_result_) + { + qualify_analysis_result = std::move(qualify_analysis_result_); + } + bool hasSort() const { return sort_analysis_result.before_order_by_actions != nullptr; @@ -165,6 +180,7 @@ private: AggregationAnalysisResult aggregation_analysis_result; FilterAnalysisResult having_analysis_result; WindowAnalysisResult window_analysis_result; + FilterAnalysisResult qualify_analysis_result; SortAnalysisResult sort_analysis_result; LimitByAnalysisResult limit_by_analysis_result; };