diff --git a/src/Analyzer/ArrayJoinNode.cpp b/src/Analyzer/ArrayJoinNode.cpp index 490e227d617..1a7969ce64a 100644 --- a/src/Analyzer/ArrayJoinNode.cpp +++ b/src/Analyzer/ArrayJoinNode.cpp @@ -49,7 +49,7 @@ QueryTreeNodePtr ArrayJoinNode::cloneImpl() const return std::make_shared(getTableExpression(), getJoinExpressionsNode(), is_left); } -ASTPtr ArrayJoinNode::toASTImpl() const +ASTPtr ArrayJoinNode::toASTImpl(ConvertToASTOptions options) const { auto array_join_ast = std::make_shared(); array_join_ast->kind = is_left ? ASTArrayJoin::Kind::Left : ASTArrayJoin::Kind::Inner; @@ -63,9 +63,9 @@ ASTPtr ArrayJoinNode::toASTImpl() const auto * column_node = array_join_expression->as(); if (column_node && column_node->getExpression()) - array_join_expression_ast = column_node->getExpression()->toAST(); + array_join_expression_ast = column_node->getExpression()->toAST(options); else - array_join_expression_ast = array_join_expression->toAST(); + array_join_expression_ast = array_join_expression->toAST(options); array_join_expression_ast->setAlias(array_join_expression->getAlias()); array_join_expressions_ast->children.push_back(std::move(array_join_expression_ast)); diff --git a/src/Analyzer/ArrayJoinNode.h b/src/Analyzer/ArrayJoinNode.h index 50d53df465a..f19f1b67971 100644 --- a/src/Analyzer/ArrayJoinNode.h +++ b/src/Analyzer/ArrayJoinNode.h @@ -99,7 +99,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; private: bool is_left = false; diff --git a/src/Analyzer/ColumnNode.cpp b/src/Analyzer/ColumnNode.cpp index c07d7bab717..568daa321ea 100644 --- a/src/Analyzer/ColumnNode.cpp +++ b/src/Analyzer/ColumnNode.cpp @@ -91,12 +91,12 @@ QueryTreeNodePtr ColumnNode::cloneImpl() const return std::make_shared(column, getSourceWeakPointer()); } -ASTPtr ColumnNode::toASTImpl() const +ASTPtr ColumnNode::toASTImpl(ConvertToASTOptions options) const { std::vector column_identifier_parts; auto column_source = getColumnSourceOrNull(); - if (column_source) + if (column_source && options.fully_qualified_identifiers) { auto node_type = column_source->getNodeType(); if (node_type == QueryTreeNodeType::TABLE || diff --git a/src/Analyzer/ColumnNode.h b/src/Analyzer/ColumnNode.h index 79c0e23c86f..1597cc465af 100644 --- a/src/Analyzer/ColumnNode.h +++ b/src/Analyzer/ColumnNode.h @@ -103,6 +103,11 @@ public: */ QueryTreeNodePtr getColumnSource() const; + void dropColumnSource() + { + getSourceWeakPointer().reset(); + } + /** Get column source. * If column source is not valid null is returned. */ @@ -132,7 +137,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; private: const QueryTreeNodeWeakPtr & getSourceWeakPointer() const diff --git a/src/Analyzer/ColumnTransformers.cpp b/src/Analyzer/ColumnTransformers.cpp index ee336a0e7f3..772916bbed6 100644 --- a/src/Analyzer/ColumnTransformers.cpp +++ b/src/Analyzer/ColumnTransformers.cpp @@ -91,7 +91,7 @@ QueryTreeNodePtr ApplyColumnTransformerNode::cloneImpl() const return std::make_shared(getExpressionNode()); } -ASTPtr ApplyColumnTransformerNode::toASTImpl() const +ASTPtr ApplyColumnTransformerNode::toASTImpl(ConvertToASTOptions options) const { auto ast_apply_transformer = std::make_shared(); const auto & expression_node = getExpressionNode(); @@ -100,14 +100,14 @@ ASTPtr ApplyColumnTransformerNode::toASTImpl() const { auto & function_expression = expression_node->as(); ast_apply_transformer->func_name = function_expression.getFunctionName(); - ast_apply_transformer->parameters = function_expression.getParametersNode()->toAST(); + ast_apply_transformer->parameters = function_expression.getParametersNode()->toAST(options); } else { auto & lambda_expression = expression_node->as(); if (!lambda_expression.getArgumentNames().empty()) ast_apply_transformer->lambda_arg = lambda_expression.getArgumentNames()[0]; - ast_apply_transformer->lambda = lambda_expression.toAST(); + ast_apply_transformer->lambda = lambda_expression.toAST(options); } return ast_apply_transformer; @@ -227,7 +227,7 @@ QueryTreeNodePtr ExceptColumnTransformerNode::cloneImpl() const return std::make_shared(except_column_names, is_strict); } -ASTPtr ExceptColumnTransformerNode::toASTImpl() const +ASTPtr ExceptColumnTransformerNode::toASTImpl(ConvertToASTOptions /* options */) const { auto ast_except_transformer = std::make_shared(); @@ -334,7 +334,7 @@ QueryTreeNodePtr ReplaceColumnTransformerNode::cloneImpl() const return result_replace_transformer; } -ASTPtr ReplaceColumnTransformerNode::toASTImpl() const +ASTPtr ReplaceColumnTransformerNode::toASTImpl(ConvertToASTOptions options) const { auto ast_replace_transformer = std::make_shared(); @@ -347,7 +347,7 @@ ASTPtr ReplaceColumnTransformerNode::toASTImpl() const { auto replacement_ast = std::make_shared(); replacement_ast->name = replacements_names[i]; - replacement_ast->children.push_back(replacement_expressions_nodes[i]->toAST()); + replacement_ast->children.push_back(replacement_expressions_nodes[i]->toAST(options)); ast_replace_transformer->children.push_back(std::move(replacement_ast)); } diff --git a/src/Analyzer/ColumnTransformers.h b/src/Analyzer/ColumnTransformers.h index e96e606d923..2b06e66d07e 100644 --- a/src/Analyzer/ColumnTransformers.h +++ b/src/Analyzer/ColumnTransformers.h @@ -141,7 +141,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; private: ApplyColumnTransformerType apply_transformer_type = ApplyColumnTransformerType::LAMBDA; @@ -220,7 +220,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; private: ExceptColumnTransformerType except_transformer_type; @@ -298,7 +298,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; private: ListNode & getReplacements() diff --git a/src/Analyzer/ConstantNode.cpp b/src/Analyzer/ConstantNode.cpp index 79fc38cd617..8d286cc99a1 100644 --- a/src/Analyzer/ConstantNode.cpp +++ b/src/Analyzer/ConstantNode.cpp @@ -75,11 +75,14 @@ QueryTreeNodePtr ConstantNode::cloneImpl() const return std::make_shared(constant_value, source_expression); } -ASTPtr ConstantNode::toASTImpl() const +ASTPtr ConstantNode::toASTImpl(ConvertToASTOptions options) const { const auto & constant_value_literal = constant_value->getValue(); auto constant_value_ast = std::make_shared(constant_value_literal); + if (!options.add_cast_for_constants) + return constant_value_ast; + bool need_to_add_cast_function = false; auto constant_value_literal_type = constant_value_literal.getType(); WhichDataType constant_value_type(constant_value->getType()); diff --git a/src/Analyzer/ConstantNode.h b/src/Analyzer/ConstantNode.h index 6b58533a701..e7d38c5bbfa 100644 --- a/src/Analyzer/ConstantNode.h +++ b/src/Analyzer/ConstantNode.h @@ -83,7 +83,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; private: ConstantValuePtr constant_value; diff --git a/src/Analyzer/FunctionNode.cpp b/src/Analyzer/FunctionNode.cpp index fe170c8482e..d0344c5efdf 100644 --- a/src/Analyzer/FunctionNode.cpp +++ b/src/Analyzer/FunctionNode.cpp @@ -197,7 +197,7 @@ QueryTreeNodePtr FunctionNode::cloneImpl() const return result_function; } -ASTPtr FunctionNode::toASTImpl() const +ASTPtr FunctionNode::toASTImpl(ConvertToASTOptions options) const { auto function_ast = std::make_shared(); @@ -212,12 +212,12 @@ ASTPtr FunctionNode::toASTImpl() const const auto & parameters = getParameters(); if (!parameters.getNodes().empty()) { - function_ast->children.push_back(parameters.toAST()); + function_ast->children.push_back(parameters.toAST(options)); function_ast->parameters = function_ast->children.back(); } const auto & arguments = getArguments(); - function_ast->children.push_back(arguments.toAST()); + function_ast->children.push_back(arguments.toAST(options)); function_ast->arguments = function_ast->children.back(); auto window_node = getWindowNode(); @@ -226,7 +226,7 @@ ASTPtr FunctionNode::toASTImpl() const if (auto * identifier_node = window_node->as()) function_ast->window_name = identifier_node->getIdentifier().getFullName(); else - function_ast->window_definition = window_node->toAST(); + function_ast->window_definition = window_node->toAST(options); } return function_ast; diff --git a/src/Analyzer/FunctionNode.h b/src/Analyzer/FunctionNode.h index 89a684c1d0f..2e899fe2801 100644 --- a/src/Analyzer/FunctionNode.h +++ b/src/Analyzer/FunctionNode.h @@ -209,7 +209,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; private: String function_name; diff --git a/src/Analyzer/IQueryTreeNode.cpp b/src/Analyzer/IQueryTreeNode.cpp index ba361af0007..7603a8a1593 100644 --- a/src/Analyzer/IQueryTreeNode.cpp +++ b/src/Analyzer/IQueryTreeNode.cpp @@ -331,9 +331,9 @@ QueryTreeNodePtr IQueryTreeNode::cloneAndReplace(const QueryTreeNodePtr & node_t return cloneAndReplace(replacement_map); } -ASTPtr IQueryTreeNode::toAST() const +ASTPtr IQueryTreeNode::toAST(ConvertToASTOptions options) const { - auto converted_node = toASTImpl(); + auto converted_node = toASTImpl(options); if (auto * ast_with_alias = dynamic_cast(converted_node.get())) converted_node->setAlias(alias); diff --git a/src/Analyzer/IQueryTreeNode.h b/src/Analyzer/IQueryTreeNode.h index e344dd66fbc..18562214200 100644 --- a/src/Analyzer/IQueryTreeNode.h +++ b/src/Analyzer/IQueryTreeNode.h @@ -181,8 +181,17 @@ public: */ String formatOriginalASTForErrorMessage() const; + struct ConvertToASTOptions + { + /// Add _CAST if constant litral type is different from column type + bool add_cast_for_constants = true; + + /// Identifiers are fully qualified (`database.table.column`), otherwise names are just column names (`column`) + bool fully_qualified_identifiers = true; + }; + /// Convert query tree to AST - ASTPtr toAST() const; + ASTPtr toAST(ConvertToASTOptions options = { .add_cast_for_constants = true, .fully_qualified_identifiers = true }) const; /// Convert query tree to AST and then format it for error message. String formatConvertedASTForErrorMessage() const; @@ -258,7 +267,7 @@ protected: virtual QueryTreeNodePtr cloneImpl() const = 0; /// Subclass must convert its internal state and its children to AST - virtual ASTPtr toASTImpl() const = 0; + virtual ASTPtr toASTImpl(ConvertToASTOptions options) const = 0; QueryTreeNodes children; QueryTreeWeakNodes weak_pointers; diff --git a/src/Analyzer/IdentifierNode.cpp b/src/Analyzer/IdentifierNode.cpp index cb5d9609962..3c135a3a2bc 100644 --- a/src/Analyzer/IdentifierNode.cpp +++ b/src/Analyzer/IdentifierNode.cpp @@ -58,7 +58,7 @@ QueryTreeNodePtr IdentifierNode::cloneImpl() const return std::make_shared(identifier); } -ASTPtr IdentifierNode::toASTImpl() const +ASTPtr IdentifierNode::toASTImpl(ConvertToASTOptions /* options */) const { auto identifier_parts = identifier.getParts(); return std::make_shared(std::move(identifier_parts)); diff --git a/src/Analyzer/IdentifierNode.h b/src/Analyzer/IdentifierNode.h index 358511d1f90..ced599218b1 100644 --- a/src/Analyzer/IdentifierNode.h +++ b/src/Analyzer/IdentifierNode.h @@ -59,7 +59,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; private: Identifier identifier; diff --git a/src/Analyzer/InterpolateNode.cpp b/src/Analyzer/InterpolateNode.cpp index c8c61b05853..9df2ac08018 100644 --- a/src/Analyzer/InterpolateNode.cpp +++ b/src/Analyzer/InterpolateNode.cpp @@ -44,11 +44,11 @@ QueryTreeNodePtr InterpolateNode::cloneImpl() const return std::make_shared(nullptr /*expression*/, nullptr /*interpolate_expression*/); } -ASTPtr InterpolateNode::toASTImpl() const +ASTPtr InterpolateNode::toASTImpl(ConvertToASTOptions options) const { auto result = std::make_shared(); - result->column = getExpression()->toAST()->getColumnName(); - result->children.push_back(getInterpolateExpression()->toAST()); + result->column = getExpression()->toAST(options)->getColumnName(); + result->children.push_back(getInterpolateExpression()->toAST(options)); result->expr = result->children.back(); return result; diff --git a/src/Analyzer/InterpolateNode.h b/src/Analyzer/InterpolateNode.h index 5764ea561c0..8ac31f0fb8f 100644 --- a/src/Analyzer/InterpolateNode.h +++ b/src/Analyzer/InterpolateNode.h @@ -59,7 +59,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; private: static constexpr size_t expression_child_index = 0; diff --git a/src/Analyzer/JoinNode.cpp b/src/Analyzer/JoinNode.cpp index fe4dd2c5016..829db614774 100644 --- a/src/Analyzer/JoinNode.cpp +++ b/src/Analyzer/JoinNode.cpp @@ -99,7 +99,7 @@ QueryTreeNodePtr JoinNode::cloneImpl() const return std::make_shared(getLeftTableExpression(), getRightTableExpression(), getJoinExpression(), locality, strictness, kind); } -ASTPtr JoinNode::toASTImpl() const +ASTPtr JoinNode::toASTImpl(ConvertToASTOptions /* options */) const { ASTPtr tables_in_select_query_ast = std::make_shared(); diff --git a/src/Analyzer/JoinNode.h b/src/Analyzer/JoinNode.h index f58fe3f1af5..4ea08c620ef 100644 --- a/src/Analyzer/JoinNode.h +++ b/src/Analyzer/JoinNode.h @@ -148,7 +148,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; private: JoinLocality locality = JoinLocality::Unspecified; diff --git a/src/Analyzer/LambdaNode.cpp b/src/Analyzer/LambdaNode.cpp index b60b40878ec..2df389c4029 100644 --- a/src/Analyzer/LambdaNode.cpp +++ b/src/Analyzer/LambdaNode.cpp @@ -65,17 +65,17 @@ QueryTreeNodePtr LambdaNode::cloneImpl() const return std::make_shared(argument_names, getExpression()); } -ASTPtr LambdaNode::toASTImpl() const +ASTPtr LambdaNode::toASTImpl(ConvertToASTOptions options) const { auto lambda_function_arguments_ast = std::make_shared(); auto tuple_function = std::make_shared(); tuple_function->name = "tuple"; - tuple_function->children.push_back(children[arguments_child_index]->toAST()); + tuple_function->children.push_back(children[arguments_child_index]->toAST(options)); tuple_function->arguments = tuple_function->children.back(); lambda_function_arguments_ast->children.push_back(std::move(tuple_function)); - lambda_function_arguments_ast->children.push_back(children[expression_child_index]->toAST()); + lambda_function_arguments_ast->children.push_back(children[expression_child_index]->toAST(options)); auto lambda_function_ast = std::make_shared(); lambda_function_ast->name = "lambda"; diff --git a/src/Analyzer/LambdaNode.h b/src/Analyzer/LambdaNode.h index 65b0d3de84e..f64d49fff08 100644 --- a/src/Analyzer/LambdaNode.h +++ b/src/Analyzer/LambdaNode.h @@ -98,7 +98,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; private: Names argument_names; diff --git a/src/Analyzer/ListNode.cpp b/src/Analyzer/ListNode.cpp index 7bbb884fa7f..238ba0c8133 100644 --- a/src/Analyzer/ListNode.cpp +++ b/src/Analyzer/ListNode.cpp @@ -54,7 +54,7 @@ QueryTreeNodePtr ListNode::cloneImpl() const return std::make_shared(); } -ASTPtr ListNode::toASTImpl() const +ASTPtr ListNode::toASTImpl(ConvertToASTOptions options) const { auto expression_list_ast = std::make_shared(); @@ -62,7 +62,7 @@ ASTPtr ListNode::toASTImpl() const expression_list_ast->children.resize(children_size); for (size_t i = 0; i < children_size; ++i) - expression_list_ast->children[i] = children[i]->toAST(); + expression_list_ast->children[i] = children[i]->toAST(options); return expression_list_ast; } diff --git a/src/Analyzer/ListNode.h b/src/Analyzer/ListNode.h index 75013f7ee6a..9cf2011eb39 100644 --- a/src/Analyzer/ListNode.h +++ b/src/Analyzer/ListNode.h @@ -57,7 +57,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; }; } diff --git a/src/Analyzer/MatcherNode.cpp b/src/Analyzer/MatcherNode.cpp index 5c8738e0504..d6bdd50562d 100644 --- a/src/Analyzer/MatcherNode.cpp +++ b/src/Analyzer/MatcherNode.cpp @@ -204,7 +204,7 @@ QueryTreeNodePtr MatcherNode::cloneImpl() const return matcher_node; } -ASTPtr MatcherNode::toASTImpl() const +ASTPtr MatcherNode::toASTImpl(ConvertToASTOptions options) const { ASTPtr result; ASTPtr transformers; @@ -216,7 +216,7 @@ ASTPtr MatcherNode::toASTImpl() const transformers = std::make_shared(); for (const auto & column_transformer : column_transformers) - transformers->children.push_back(column_transformer->toAST()); + transformers->children.push_back(column_transformer->toAST(options)); } if (matcher_type == MatcherNodeType::ASTERISK) diff --git a/src/Analyzer/MatcherNode.h b/src/Analyzer/MatcherNode.h index e79c1cb4bf2..985000f7f65 100644 --- a/src/Analyzer/MatcherNode.h +++ b/src/Analyzer/MatcherNode.h @@ -148,7 +148,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; private: explicit MatcherNode(MatcherNodeType matcher_type_, diff --git a/src/Analyzer/QueryNode.cpp b/src/Analyzer/QueryNode.cpp index 774f3376f48..c4c36c9f192 100644 --- a/src/Analyzer/QueryNode.cpp +++ b/src/Analyzer/QueryNode.cpp @@ -259,7 +259,7 @@ QueryTreeNodePtr QueryNode::cloneImpl() const return result_query_node; } -ASTPtr QueryNode::toASTImpl() const +ASTPtr QueryNode::toASTImpl(ConvertToASTOptions options) const { auto select_query = std::make_shared(); select_query->distinct = is_distinct; @@ -271,9 +271,9 @@ ASTPtr QueryNode::toASTImpl() const select_query->group_by_all = is_group_by_all; if (hasWith()) - select_query->setExpression(ASTSelectQuery::Expression::WITH, getWith().toAST()); + select_query->setExpression(ASTSelectQuery::Expression::WITH, getWith().toAST(options)); - auto projection_ast = getProjection().toAST(); + auto projection_ast = getProjection().toAST(options); auto & projection_expression_list_ast = projection_ast->as(); size_t projection_expression_list_ast_children_size = projection_expression_list_ast.children.size(); if (projection_expression_list_ast_children_size != getProjection().getNodes().size()) @@ -297,40 +297,40 @@ ASTPtr QueryNode::toASTImpl() const select_query->setExpression(ASTSelectQuery::Expression::TABLES, std::move(tables_in_select_query_ast)); if (getPrewhere()) - select_query->setExpression(ASTSelectQuery::Expression::PREWHERE, getPrewhere()->toAST()); + select_query->setExpression(ASTSelectQuery::Expression::PREWHERE, getPrewhere()->toAST(options)); if (getWhere()) - select_query->setExpression(ASTSelectQuery::Expression::WHERE, getWhere()->toAST()); + select_query->setExpression(ASTSelectQuery::Expression::WHERE, getWhere()->toAST(options)); if (!is_group_by_all && hasGroupBy()) - select_query->setExpression(ASTSelectQuery::Expression::GROUP_BY, getGroupBy().toAST()); + select_query->setExpression(ASTSelectQuery::Expression::GROUP_BY, getGroupBy().toAST(options)); if (hasHaving()) - select_query->setExpression(ASTSelectQuery::Expression::HAVING, getHaving()->toAST()); + select_query->setExpression(ASTSelectQuery::Expression::HAVING, getHaving()->toAST(options)); if (hasWindow()) - select_query->setExpression(ASTSelectQuery::Expression::WINDOW, getWindow().toAST()); + select_query->setExpression(ASTSelectQuery::Expression::WINDOW, getWindow().toAST(options)); if (hasOrderBy()) - select_query->setExpression(ASTSelectQuery::Expression::ORDER_BY, getOrderBy().toAST()); + select_query->setExpression(ASTSelectQuery::Expression::ORDER_BY, getOrderBy().toAST(options)); if (hasInterpolate()) - select_query->setExpression(ASTSelectQuery::Expression::INTERPOLATE, getInterpolate()->toAST()); + select_query->setExpression(ASTSelectQuery::Expression::INTERPOLATE, getInterpolate()->toAST(options)); if (hasLimitByLimit()) - select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY_LENGTH, getLimitByLimit()->toAST()); + select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY_LENGTH, getLimitByLimit()->toAST(options)); if (hasLimitByOffset()) - select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY_OFFSET, getLimitByOffset()->toAST()); + select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY_OFFSET, getLimitByOffset()->toAST(options)); if (hasLimitBy()) - select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY, getLimitBy().toAST()); + select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY, getLimitBy().toAST(options)); if (hasLimit()) - select_query->setExpression(ASTSelectQuery::Expression::LIMIT_LENGTH, getLimit()->toAST()); + select_query->setExpression(ASTSelectQuery::Expression::LIMIT_LENGTH, getLimit()->toAST(options)); if (hasOffset()) - select_query->setExpression(ASTSelectQuery::Expression::LIMIT_OFFSET, getOffset()->toAST()); + select_query->setExpression(ASTSelectQuery::Expression::LIMIT_OFFSET, getOffset()->toAST(options)); if (hasSettingsChanges()) { diff --git a/src/Analyzer/QueryNode.h b/src/Analyzer/QueryNode.h index 54154e1e353..add9e14a0b5 100644 --- a/src/Analyzer/QueryNode.h +++ b/src/Analyzer/QueryNode.h @@ -575,7 +575,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; private: bool is_subquery = false; diff --git a/src/Analyzer/SortNode.cpp b/src/Analyzer/SortNode.cpp index da1c52ff0ef..6dff649186d 100644 --- a/src/Analyzer/SortNode.cpp +++ b/src/Analyzer/SortNode.cpp @@ -109,7 +109,7 @@ QueryTreeNodePtr SortNode::cloneImpl() const return std::make_shared(nullptr /*expression*/, sort_direction, nulls_sort_direction, collator, with_fill); } -ASTPtr SortNode::toASTImpl() const +ASTPtr SortNode::toASTImpl(ConvertToASTOptions options) const { auto result = std::make_shared(); result->direction = sort_direction == SortDirection::ASCENDING ? 1 : -1; @@ -120,10 +120,10 @@ ASTPtr SortNode::toASTImpl() const result->nulls_direction_was_explicitly_specified = nulls_sort_direction.has_value(); result->with_fill = with_fill; - result->fill_from = hasFillFrom() ? getFillFrom()->toAST() : nullptr; - result->fill_to = hasFillTo() ? getFillTo()->toAST() : nullptr; - result->fill_step = hasFillStep() ? getFillStep()->toAST() : nullptr; - result->children.push_back(getExpression()->toAST()); + result->fill_from = hasFillFrom() ? getFillFrom()->toAST(options) : nullptr; + result->fill_to = hasFillTo() ? getFillTo()->toAST(options) : nullptr; + result->fill_step = hasFillStep() ? getFillStep()->toAST(options) : nullptr; + result->children.push_back(getExpression()->toAST(options)); if (collator) { diff --git a/src/Analyzer/SortNode.h b/src/Analyzer/SortNode.h index 04f9fe798e1..da0996f734e 100644 --- a/src/Analyzer/SortNode.h +++ b/src/Analyzer/SortNode.h @@ -137,7 +137,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; private: static constexpr size_t sort_expression_child_index = 0; diff --git a/src/Analyzer/TableFunctionNode.cpp b/src/Analyzer/TableFunctionNode.cpp index fb3a3af31e1..89e2cdc3ec8 100644 --- a/src/Analyzer/TableFunctionNode.cpp +++ b/src/Analyzer/TableFunctionNode.cpp @@ -113,14 +113,14 @@ QueryTreeNodePtr TableFunctionNode::cloneImpl() const return result; } -ASTPtr TableFunctionNode::toASTImpl() const +ASTPtr TableFunctionNode::toASTImpl(ConvertToASTOptions options) const { auto table_function_ast = std::make_shared(); table_function_ast->name = table_function_name; const auto & arguments = getArguments(); - table_function_ast->children.push_back(arguments.toAST()); + table_function_ast->children.push_back(arguments.toAST(options)); table_function_ast->arguments = table_function_ast->children.back(); return table_function_ast; diff --git a/src/Analyzer/TableFunctionNode.h b/src/Analyzer/TableFunctionNode.h index a88630ffd00..7e5f180bdcb 100644 --- a/src/Analyzer/TableFunctionNode.h +++ b/src/Analyzer/TableFunctionNode.h @@ -142,7 +142,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; private: String table_function_name; diff --git a/src/Analyzer/TableNode.cpp b/src/Analyzer/TableNode.cpp index f315d372bc9..89ea98462db 100644 --- a/src/Analyzer/TableNode.cpp +++ b/src/Analyzer/TableNode.cpp @@ -86,7 +86,7 @@ QueryTreeNodePtr TableNode::cloneImpl() const return result_table_node; } -ASTPtr TableNode::toASTImpl() const +ASTPtr TableNode::toASTImpl(ConvertToASTOptions /* options */) const { if (!temporary_table_name.empty()) return std::make_shared(temporary_table_name); diff --git a/src/Analyzer/TableNode.h b/src/Analyzer/TableNode.h index 1d5ec112ee0..0b0b3a13775 100644 --- a/src/Analyzer/TableNode.h +++ b/src/Analyzer/TableNode.h @@ -106,7 +106,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; private: StoragePtr storage; diff --git a/src/Analyzer/UnionNode.cpp b/src/Analyzer/UnionNode.cpp index 998b869cb04..fdf8880b2e1 100644 --- a/src/Analyzer/UnionNode.cpp +++ b/src/Analyzer/UnionNode.cpp @@ -140,12 +140,12 @@ QueryTreeNodePtr UnionNode::cloneImpl() const return result_union_node; } -ASTPtr UnionNode::toASTImpl() const +ASTPtr UnionNode::toASTImpl(ConvertToASTOptions options) const { auto select_with_union_query = std::make_shared(); select_with_union_query->union_mode = union_mode; select_with_union_query->is_normalized = true; - select_with_union_query->children.push_back(getQueriesNode()->toAST()); + select_with_union_query->children.push_back(getQueriesNode()->toAST(options)); select_with_union_query->list_of_selects = select_with_union_query->children.back(); if (is_subquery) diff --git a/src/Analyzer/UnionNode.h b/src/Analyzer/UnionNode.h index 5e3861da814..bd85cb802e9 100644 --- a/src/Analyzer/UnionNode.h +++ b/src/Analyzer/UnionNode.h @@ -143,7 +143,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; private: bool is_subquery = false; diff --git a/src/Analyzer/WindowNode.cpp b/src/Analyzer/WindowNode.cpp index d516f7a58b8..8aea4f3b87c 100644 --- a/src/Analyzer/WindowNode.cpp +++ b/src/Analyzer/WindowNode.cpp @@ -107,7 +107,7 @@ QueryTreeNodePtr WindowNode::cloneImpl() const return window_node; } -ASTPtr WindowNode::toASTImpl() const +ASTPtr WindowNode::toASTImpl(ConvertToASTOptions options) const { auto window_definition = std::make_shared(); @@ -115,13 +115,13 @@ ASTPtr WindowNode::toASTImpl() const if (hasPartitionBy()) { - window_definition->children.push_back(getPartitionByNode()->toAST()); + window_definition->children.push_back(getPartitionByNode()->toAST(options)); window_definition->partition_by = window_definition->children.back(); } if (hasOrderBy()) { - window_definition->children.push_back(getOrderByNode()->toAST()); + window_definition->children.push_back(getOrderByNode()->toAST(options)); window_definition->order_by = window_definition->children.back(); } @@ -132,7 +132,7 @@ ASTPtr WindowNode::toASTImpl() const if (hasFrameBeginOffset()) { - window_definition->children.push_back(getFrameBeginOffsetNode()->toAST()); + window_definition->children.push_back(getFrameBeginOffsetNode()->toAST(options)); window_definition->frame_begin_offset = window_definition->children.back(); } @@ -140,7 +140,7 @@ ASTPtr WindowNode::toASTImpl() const window_definition->frame_end_preceding = window_frame.end_preceding; if (hasFrameEndOffset()) { - window_definition->children.push_back(getFrameEndOffsetNode()->toAST()); + window_definition->children.push_back(getFrameEndOffsetNode()->toAST(options)); window_definition->frame_end_offset = window_definition->children.back(); } diff --git a/src/Analyzer/WindowNode.h b/src/Analyzer/WindowNode.h index 9dfb3e6ef2a..3b00b8860bf 100644 --- a/src/Analyzer/WindowNode.h +++ b/src/Analyzer/WindowNode.h @@ -175,7 +175,7 @@ protected: QueryTreeNodePtr cloneImpl() const override; - ASTPtr toASTImpl() const override; + ASTPtr toASTImpl(ConvertToASTOptions options) const override; private: static constexpr size_t order_by_child_index = 0; diff --git a/src/Analyzer/tests/gtest_query_tree_node.cpp b/src/Analyzer/tests/gtest_query_tree_node.cpp index 079869b2a53..00a698071a3 100644 --- a/src/Analyzer/tests/gtest_query_tree_node.cpp +++ b/src/Analyzer/tests/gtest_query_tree_node.cpp @@ -36,7 +36,7 @@ public: return std::make_shared(); } - ASTPtr toASTImpl() const override + ASTPtr toASTImpl(ConvertToASTOptions /* options */) const override { return nullptr; } diff --git a/src/Common/mysqlxx/Exception.cpp b/src/Common/mysqlxx/Exception.cpp index 0f5320da754..ed1e2278a35 100644 --- a/src/Common/mysqlxx/Exception.cpp +++ b/src/Common/mysqlxx/Exception.cpp @@ -10,9 +10,11 @@ namespace mysqlxx { -std::string errorMessage(MYSQL * driver) +std::string errorMessage(MYSQL * driver, const std::string & query) { - return fmt::format("{} ({}:{})", mysql_error(driver), driver->host ? driver->host : "(nullptr)", driver->port); + return fmt::format("{}{} ({}:{})", mysql_error(driver), + query.empty() ? "" : " while executing query: '" + query + "'", + driver->host ? driver->host : "(nullptr)", driver->port); } void checkError(MYSQL * driver) diff --git a/src/Common/mysqlxx/Query.cpp b/src/Common/mysqlxx/Query.cpp index 97b29fa21df..42c35d26ecf 100644 --- a/src/Common/mysqlxx/Query.cpp +++ b/src/Common/mysqlxx/Query.cpp @@ -64,7 +64,7 @@ void Query::executeImpl() case CR_SERVER_LOST: throw ConnectionLost(errorMessage(mysql_driver), err_no); default: - throw BadQuery(errorMessage(mysql_driver), err_no); + throw BadQuery(errorMessage(mysql_driver, query), err_no); } } } diff --git a/src/Common/mysqlxx/Value.cpp b/src/Common/mysqlxx/Value.cpp index 6954080f864..41d717669b9 100644 --- a/src/Common/mysqlxx/Value.cpp +++ b/src/Common/mysqlxx/Value.cpp @@ -160,14 +160,16 @@ void Value::throwException(const char * text) const if (!isNull()) { - info.append(": "); + info.append(": '"); info.append(m_data, m_length); + info.append("'"); } if (res && res->getQuery()) { - info.append(", query: "); + info.append(", query: '"); info.append(res->getQuery()->str().substr(0, preview_length)); + info.append("'"); } throw CannotParseValue(info); diff --git a/src/Common/mysqlxx/mysqlxx/Exception.h b/src/Common/mysqlxx/mysqlxx/Exception.h index 7886368e747..54153a7f150 100644 --- a/src/Common/mysqlxx/mysqlxx/Exception.h +++ b/src/Common/mysqlxx/mysqlxx/Exception.h @@ -53,7 +53,7 @@ struct CannotParseValue : public Exception }; -std::string errorMessage(MYSQL * driver); +std::string errorMessage(MYSQL * driver, const std::string & query = ""); /// For internal need of library. void checkError(MYSQL * driver); diff --git a/src/Common/tests/gtest_global_register.h b/src/Common/tests/gtest_global_register.h index c4bde825109..3a7fa77c893 100644 --- a/src/Common/tests/gtest_global_register.h +++ b/src/Common/tests/gtest_global_register.h @@ -1,8 +1,13 @@ #pragma once #include +#include #include +inline void tryRegisterAggregateFunctions() +{ + static struct Register { Register() { DB::registerAggregateFunctions(); } } registered; +} inline void tryRegisterFunctions() { diff --git a/src/Interpreters/InterpreterSelectQueryAnalyzer.h b/src/Interpreters/InterpreterSelectQueryAnalyzer.h index 2c8af49cf0e..f39e55072ef 100644 --- a/src/Interpreters/InterpreterSelectQueryAnalyzer.h +++ b/src/Interpreters/InterpreterSelectQueryAnalyzer.h @@ -71,6 +71,13 @@ public: /// Set number_of_current_replica and count_participating_replicas in client_info void setProperClientInfo(size_t replica_number, size_t count_participating_replicas); + const QueryTreeNodePtr & getQueryTree() const { return query_tree; } + + SelectQueryInfo getSelectQueryInfo() + { + planner.buildQueryPlanIfNeeded(); return planner.buildSelectQueryInfo(); + } + private: ASTPtr query; ContextMutablePtr context; diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 2242bf92e6b..9e7e0e62ba0 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -1159,11 +1159,7 @@ void Planner::buildPlanForQueryNode() query_node.getWhere() = {}; } - SelectQueryInfo select_query_info; - select_query_info.original_query = queryNodeToSelectQuery(query_tree); - select_query_info.query = select_query_info.original_query; - select_query_info.query_tree = query_tree; - select_query_info.planner_context = planner_context; + SelectQueryInfo select_query_info = buildSelectQueryInfo(); StorageLimitsList current_storage_limits = storage_limits; select_query_info.local_storage_limits = buildStorageLimits(*query_context, select_query_options); @@ -1454,6 +1450,16 @@ void Planner::buildPlanForQueryNode() addBuildSubqueriesForSetsStepIfNeeded(query_plan, select_query_options, planner_context, result_actions_to_execute); } +SelectQueryInfo Planner::buildSelectQueryInfo() const +{ + SelectQueryInfo select_query_info; + select_query_info.original_query = queryNodeToSelectQuery(query_tree); + select_query_info.query = select_query_info.original_query; + select_query_info.query_tree = query_tree; + select_query_info.planner_context = planner_context; + return select_query_info; +} + void Planner::addStorageLimits(const StorageLimitsList & limits) { for (const auto & limit : limits) diff --git a/src/Planner/Planner.h b/src/Planner/Planner.h index 443dfa114ee..6fdce80b73a 100644 --- a/src/Planner/Planner.h +++ b/src/Planner/Planner.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB { @@ -50,6 +51,8 @@ public: return std::move(query_plan); } + SelectQueryInfo buildSelectQueryInfo() const; + void addStorageLimits(const StorageLimitsList & limits); private: diff --git a/src/Processors/Merges/Algorithms/tests/gtest_graphite.cpp b/src/Processors/Merges/Algorithms/tests/gtest_graphite.cpp index 1bb6acbcb5c..46980ceb56b 100644 --- a/src/Processors/Merges/Algorithms/tests/gtest_graphite.cpp +++ b/src/Processors/Merges/Algorithms/tests/gtest_graphite.cpp @@ -18,17 +18,6 @@ using namespace DB; -static int regAggregateFunctions = 0; - -void tryRegisterAggregateFunctions() -{ - if (!regAggregateFunctions) - { - registerAggregateFunctions(); - regAggregateFunctions = 1; - } -} - static ConfigProcessor::LoadedConfig loadConfiguration(const std::string & config_path) { ConfigProcessor config_processor(config_path, true, true); diff --git a/src/Processors/Sources/MySQLSource.cpp b/src/Processors/Sources/MySQLSource.cpp index 434d413a238..9c7e83b3869 100644 --- a/src/Processors/Sources/MySQLSource.cpp +++ b/src/Processors/Sources/MySQLSource.cpp @@ -107,6 +107,11 @@ void MySQLWithFailoverSource::onStart() throw; } } + catch (const mysqlxx::BadQuery & e) + { + LOG_ERROR(log, "Error processing query '{}': {}", query_str, e.displayText()); + throw; + } } initPositionMappingFromQueryResultStructure(); diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index db43b2fc3f8..fb895d04b8f 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -190,6 +190,7 @@ struct SelectQueryInfo PlannerContextPtr planner_context; /// Storage table expression + /// It's guaranteed to be present in JOIN TREE of `query_tree` QueryTreeNodePtr table_expression; /// Table expression modifiers for storage diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp index 7e133538e41..c389f7d9f7f 100644 --- a/src/Storages/StorageMySQL.cpp +++ b/src/Storages/StorageMySQL.cpp @@ -84,6 +84,7 @@ Pipe StorageMySQL::read( storage_snapshot->check(column_names_); String query = transformQueryForExternalDatabase( query_info_, + column_names_, storage_snapshot->metadata->getColumns().getOrdinary(), IdentifierQuotingStyle::BackticksMySQL, remote_database_name, diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index cf87d23bf94..8e1a799fa07 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -91,7 +91,9 @@ Pipe StoragePostgreSQL::read( /// Connection is already made to the needed database, so it should not be present in the query; /// remote_table_schema is empty if it is not specified, will access only table_name. String query = transformQueryForExternalDatabase( - query_info_, storage_snapshot->metadata->getColumns().getOrdinary(), + query_info_, + column_names_, + storage_snapshot->metadata->getColumns().getOrdinary(), IdentifierQuotingStyle::DoubleQuotes, remote_table_schema, remote_table_name, context_); LOG_TRACE(log, "Query: {}", query); diff --git a/src/Storages/StorageSQLite.cpp b/src/Storages/StorageSQLite.cpp index 706bc31122c..10eba370d26 100644 --- a/src/Storages/StorageSQLite.cpp +++ b/src/Storages/StorageSQLite.cpp @@ -66,6 +66,7 @@ Pipe StorageSQLite::read( String query = transformQueryForExternalDatabase( query_info, + column_names, storage_snapshot->metadata->getColumns().getOrdinary(), IdentifierQuotingStyle::DoubleQuotes, "", diff --git a/src/Storages/StorageXDBC.cpp b/src/Storages/StorageXDBC.cpp index cb5532e91ac..9b3e203e337 100644 --- a/src/Storages/StorageXDBC.cpp +++ b/src/Storages/StorageXDBC.cpp @@ -74,7 +74,9 @@ std::function StorageXDBC::getReadPOSTDataCallback( QueryProcessingStage::Enum & /*processed_stage*/, size_t /*max_block_size*/) const { - String query = transformQueryForExternalDatabase(query_info, + String query = transformQueryForExternalDatabase( + query_info, + column_names, columns_description.getOrdinary(), bridge_helper->getIdentifierQuotingStyle(), remote_database_name, diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp index 131bc2b85e3..270af37b7de 100644 --- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -9,16 +9,20 @@ #include #include #include +#include #include #include #include #include +#include +#include +#include using namespace DB; -/// NOTE How to do better? +/// TODO: use gtest fixture struct State { State(const State&) = delete; @@ -31,9 +35,9 @@ struct State return state; } - const NamesAndTypesList & getColumns() const + const NamesAndTypesList & getColumns(size_t idx = 0) const { - return tables[0].columns; + return tables[idx].columns; } std::vector getTables(size_t num = 0) const @@ -46,10 +50,10 @@ struct State private: - static DatabaseAndTableWithAlias createDBAndTable(String table_name) + static DatabaseAndTableWithAlias createDBAndTable(String table_name, String database_name = "test") { DatabaseAndTableWithAlias res; - res.database = "test"; + res.database = database_name; res.table = table_name; return res; } @@ -75,12 +79,18 @@ private: {"num", std::make_shared()}, {"attr", std::make_shared()}, }), + TableWithColumnNamesAndTypes( + createDBAndTable("external_table"), + { + {"ttt", std::make_shared()}, + }), }; explicit State() : context(Context::createCopy(getContext().context)) { tryRegisterFunctions(); + tryRegisterAggregateFunctions(); DatabasePtr database = std::make_shared("test", context); for (const auto & tab : tables) @@ -91,14 +101,28 @@ private: context, table_name, std::make_shared( - StorageID(db_name, table_name), ColumnsDescription{getColumns()}, ConstraintsDescription{}, String{})); + StorageID(db_name, table_name), ColumnsDescription{tab.columns}, ConstraintsDescription{}, String{})); } DatabaseCatalog::instance().attachDatabase(database->getDatabaseName(), database); + // DatabaseCatalog::instance().attachDatabase("system", mockSystemDatabase()); + context->setCurrentDatabase("test"); } + + DatabasePtr mockSystemDatabase() + { + DatabasePtr database = std::make_shared("system", context); + auto tab = TableWithColumnNamesAndTypes(createDBAndTable("one", "system"), { {"dummy", std::make_shared()} }); + database->attachTable(context, tab.table.table, + std::make_shared( + StorageID(tab.table.database, tab.table.table), + ColumnsDescription{tab.columns}, ConstraintsDescription{}, String{})); + + return database; + } }; -static void check( +static void checkOld( const State & state, size_t table_num, const std::string & query, @@ -109,38 +133,106 @@ static void check( SelectQueryInfo query_info; SelectQueryOptions select_options; query_info.syntax_analyzer_result - = TreeRewriter(state.context).analyzeSelect(ast, DB::TreeRewriterResult(state.getColumns()), select_options, state.getTables(table_num)); + = TreeRewriter(state.context).analyzeSelect(ast, DB::TreeRewriterResult(state.getColumns(0)), select_options, state.getTables(table_num)); query_info.query = ast; std::string transformed_query = transformQueryForExternalDatabase( - query_info, state.getColumns(), IdentifierQuotingStyle::DoubleQuotes, "test", "table", state.context); + query_info, + query_info.syntax_analyzer_result->requiredSourceColumns(), + state.getColumns(0), IdentifierQuotingStyle::DoubleQuotes, "test", "table", state.context); EXPECT_EQ(transformed_query, expected) << query; } +/// Required for transformQueryForExternalDatabase. In real life table expression is calculated via planner. +/// But in tests we can just find it in JOIN TREE. +static QueryTreeNodePtr findTableExpression(const QueryTreeNodePtr & node, const String & table_name) +{ + if (node->getNodeType() == QueryTreeNodeType::TABLE) + { + if (node->as()->getStorageID().table_name == table_name) + return node; + } + + if (node->getNodeType() == QueryTreeNodeType::JOIN) + { + if (auto res = findTableExpression(node->as()->getLeftTableExpression(), table_name)) + return res; + if (auto res = findTableExpression(node->as()->getRightTableExpression(), table_name)) + return res; + } + return nullptr; +} + +/// `column_names` - Normally it's passed to query plan step. But in test we do it manually. +static void checkNewAnalyzer( + const State & state, + const Names & column_names, + const std::string & query, + const std::string & expected) +{ + ParserSelectQuery parser; + ASTPtr ast = parseQuery(parser, query, 1000, 1000); + + SelectQueryOptions select_options; + InterpreterSelectQueryAnalyzer interpreter(ast, state.context, select_options); + SelectQueryInfo query_info = interpreter.getSelectQueryInfo(); + const auto * query_node = query_info.query_tree->as(); + if (!query_node) + throw Exception(ErrorCodes::LOGICAL_ERROR, "QueryNode expected"); + + query_info.table_expression = findTableExpression(query_node->getJoinTree(), "table"); + + std::string transformed_query = transformQueryForExternalDatabase( + query_info, column_names, state.getColumns(0), IdentifierQuotingStyle::DoubleQuotes, "test", "table", state.context); + + EXPECT_EQ(transformed_query, expected) << query; +} + +static void check( + const State & state, + size_t table_num, + const Names & column_names, + const std::string & query, + const std::string & expected, + const std::string & expected_new = "") +{ + { + SCOPED_TRACE("Old analyzer"); + checkOld(state, table_num, query, expected); + } + { + SCOPED_TRACE("New analyzer"); + checkNewAnalyzer(state, column_names, query, expected_new.empty() ? expected : expected_new); + } +} TEST(TransformQueryForExternalDatabase, InWithSingleElement) { const State & state = State::instance(); - check(state, 1, + check(state, 1, {"column"}, "SELECT column FROM test.table WHERE 1 IN (1)", - R"(SELECT "column" FROM "test"."table" WHERE 1 = 1)"); - check(state, 1, + R"(SELECT "column" FROM "test"."table" WHERE 1 = 1)", + R"(SELECT "column" FROM "test"."table")"); + + check(state, 1, {"column"}, "SELECT column FROM test.table WHERE column IN (1, 2)", R"(SELECT "column" FROM "test"."table" WHERE "column" IN (1, 2))"); - check(state, 1, - "SELECT column FROM test.table WHERE column NOT IN ('hello', 'world')", - R"(SELECT "column" FROM "test"."table" WHERE "column" NOT IN ('hello', 'world'))"); + + check(state, 1, {"field"}, + "SELECT field FROM test.table WHERE field NOT IN ('hello', 'world')", + R"(SELECT "field" FROM "test"."table" WHERE "field" NOT IN ('hello', 'world'))"); } TEST(TransformQueryForExternalDatabase, InWithMultipleColumns) { const State & state = State::instance(); - check(state, 1, + check(state, 1, {"column"}, "SELECT column FROM test.table WHERE (1,1) IN ((1,1))", - R"(SELECT "column" FROM "test"."table" WHERE 1 = 1)"); - check(state, 1, + R"(SELECT "column" FROM "test"."table" WHERE 1 = 1)", + R"(SELECT "column" FROM "test"."table")"); + check(state, 1, {"field", "value"}, "SELECT field, value FROM test.table WHERE (field, value) IN (('foo', 'bar'))", R"(SELECT "field", "value" FROM "test"."table" WHERE ("field", "value") IN (('foo', 'bar')))"); } @@ -149,17 +241,17 @@ TEST(TransformQueryForExternalDatabase, InWithTable) { const State & state = State::instance(); - check(state, 1, + check(state, 1, {"column"}, "SELECT column FROM test.table WHERE 1 IN external_table", R"(SELECT "column" FROM "test"."table")"); - check(state, 1, - "SELECT column FROM test.table WHERE 1 IN (x)", + check(state, 1, {"column"}, + "WITH x as (SELECT * FROM external_table) SELECT column FROM test.table WHERE 1 IN (x)", R"(SELECT "column" FROM "test"."table")"); - check(state, 1, - "SELECT column, field, value FROM test.table WHERE column IN (field, value)", - R"(SELECT "column", "field", "value" FROM "test"."table" WHERE "column" IN ("field", "value"))"); - check(state, 1, - "SELECT column FROM test.table WHERE column NOT IN hello AND column = 123", + check(state, 1, {"column", "field", "value"}, + "SELECT column, field, value FROM test.table WHERE column IN (1, 2)", + R"(SELECT "column", "field", "value" FROM "test"."table" WHERE "column" IN (1, 2))"); + check(state, 1, {"column"}, + "SELECT column FROM test.table WHERE column NOT IN external_table AND column = 123", R"(SELECT "column" FROM "test"."table" WHERE "column" = 123)"); } @@ -167,32 +259,32 @@ TEST(TransformQueryForExternalDatabase, Like) { const State & state = State::instance(); - check(state, 1, - "SELECT column FROM test.table WHERE column LIKE '%hello%'", - R"(SELECT "column" FROM "test"."table" WHERE "column" LIKE '%hello%')"); - check(state, 1, - "SELECT column FROM test.table WHERE column NOT LIKE 'w%rld'", - R"(SELECT "column" FROM "test"."table" WHERE "column" NOT LIKE 'w%rld')"); + check(state, 1, {"field"}, + "SELECT field FROM test.table WHERE field LIKE '%hello%'", + R"(SELECT "field" FROM "test"."table" WHERE "field" LIKE '%hello%')"); + check(state, 1, {"field"}, + "SELECT field FROM test.table WHERE field NOT LIKE 'w%rld'", + R"(SELECT "field" FROM "test"."table" WHERE "field" NOT LIKE 'w%rld')"); } TEST(TransformQueryForExternalDatabase, Substring) { const State & state = State::instance(); - check(state, 1, - "SELECT column FROM test.table WHERE left(column, 10) = RIGHT(column, 10) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello'", - R"(SELECT "column" FROM "test"."table")"); + check(state, 1, {"field"}, + "SELECT field FROM test.table WHERE left(field, 10) = RIGHT(field, 10) AND SUBSTRING(field FROM 1 FOR 2) = 'Hello'", + R"(SELECT "field" FROM "test"."table")"); } TEST(TransformQueryForExternalDatabase, MultipleAndSubqueries) { const State & state = State::instance(); - check(state, 1, - "SELECT column FROM test.table WHERE 1 = 1 AND toString(column) = '42' AND column = 42 AND left(column, 10) = RIGHT(column, 10) AND column IN (1, 42) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello' AND column != 4", + check(state, 1, {"column"}, + "SELECT column FROM test.table WHERE 1 = 1 AND toString(column) = '42' AND column = 42 AND left(toString(column), 10) = RIGHT(toString(column), 10) AND column IN (1, 42) AND SUBSTRING(toString(column) FROM 1 FOR 2) = 'Hello' AND column != 4", R"(SELECT "column" FROM "test"."table" WHERE 1 AND ("column" = 42) AND ("column" IN (1, 42)) AND ("column" != 4))"); - check(state, 1, - "SELECT column FROM test.table WHERE toString(column) = '42' AND left(column, 10) = RIGHT(column, 10) AND column = 42", + check(state, 1, {"column"}, + "SELECT column FROM test.table WHERE toString(column) = '42' AND left(toString(column), 10) = RIGHT(toString(column), 10) AND column = 42", R"(SELECT "column" FROM "test"."table" WHERE "column" = 42)"); } @@ -200,7 +292,7 @@ TEST(TransformQueryForExternalDatabase, Issue7245) { const State & state = State::instance(); - check(state, 1, + check(state, 1, {"apply_id", "apply_type", "apply_status", "create_time"}, "SELECT apply_id FROM test.table WHERE apply_type = 2 AND create_time > addDays(toDateTime('2019-01-01 01:02:03'),-7) AND apply_status IN (3,4)", R"(SELECT "apply_id", "apply_type", "apply_status", "create_time" FROM "test"."table" WHERE ("apply_type" = 2) AND ("create_time" > '2018-12-25 01:02:03') AND ("apply_status" IN (3, 4)))"); } @@ -209,8 +301,8 @@ TEST(TransformQueryForExternalDatabase, Aliases) { const State & state = State::instance(); - check(state, 1, - "SELECT field AS value, field AS display WHERE field NOT IN ('') AND display LIKE '%test%'", + check(state, 1, {"field"}, + "SELECT field AS value, field AS display FROM table WHERE field NOT IN ('') AND display LIKE '%test%'", R"(SELECT "field" FROM "test"."table" WHERE ("field" NOT IN ('')) AND ("field" LIKE '%test%'))"); } @@ -218,10 +310,10 @@ TEST(TransformQueryForExternalDatabase, ForeignColumnInWhere) { const State & state = State::instance(); - check(state, 2, + check(state, 2, {"column", "apply_id"}, "SELECT column FROM test.table " "JOIN test.table2 AS table2 ON (test.table.apply_id = table2.num) " - "WHERE column > 2 AND (apply_id = 1 OR table2.num = 1) AND table2.attr != ''", + "WHERE column > 2 AND apply_id = 1 AND table2.num = 1 AND table2.attr != ''", R"(SELECT "column", "apply_id" FROM "test"."table" WHERE ("column" > 2) AND ("apply_id" = 1))"); } @@ -229,7 +321,7 @@ TEST(TransformQueryForExternalDatabase, NoStrict) { const State & state = State::instance(); - check(state, 1, + check(state, 1, {"field"}, "SELECT field FROM table WHERE field IN (SELECT attr FROM table2)", R"(SELECT "field" FROM "test"."table")"); } @@ -239,37 +331,37 @@ TEST(TransformQueryForExternalDatabase, Strict) const State & state = State::instance(); state.context->setSetting("external_table_strict_query", true); - check(state, 1, + check(state, 1, {"field"}, "SELECT field FROM table WHERE field = '1'", R"(SELECT "field" FROM "test"."table" WHERE "field" = '1')"); - check(state, 1, + check(state, 1, {"field"}, "SELECT field FROM table WHERE field IN ('1', '2')", R"(SELECT "field" FROM "test"."table" WHERE "field" IN ('1', '2'))"); - check(state, 1, + check(state, 1, {"field"}, "SELECT field FROM table WHERE field LIKE '%test%'", R"(SELECT "field" FROM "test"."table" WHERE "field" LIKE '%test%')"); /// removeUnknownSubexpressionsFromWhere() takes place - EXPECT_THROW(check(state, 1, "SELECT field FROM table WHERE field IN (SELECT attr FROM table2)", ""), Exception); + EXPECT_THROW(check(state, 1, {"field"}, "SELECT field FROM table WHERE field IN (SELECT attr FROM table2)", ""), Exception); /// !isCompatible() takes place - EXPECT_THROW(check(state, 1, "SELECT column FROM test.table WHERE left(column, 10) = RIGHT(column, 10) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello'", ""), Exception); + EXPECT_THROW(check(state, 1, {"column"}, "SELECT column FROM test.table WHERE left(column, 10) = RIGHT(column, 10) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello'", ""), Exception); } TEST(TransformQueryForExternalDatabase, Null) { const State & state = State::instance(); - check(state, 1, + check(state, 1, {"field"}, "SELECT field FROM table WHERE field IS NULL", R"(SELECT "field" FROM "test"."table" WHERE "field" IS NULL)"); - check(state, 1, + check(state, 1, {"field"}, "SELECT field FROM table WHERE field IS NOT NULL", R"(SELECT "field" FROM "test"."table" WHERE "field" IS NOT NULL)"); - check(state, 1, + check(state, 1, {"field"}, "SELECT field FROM table WHERE isNull(field)", R"(SELECT "field" FROM "test"."table" WHERE "field" IS NULL)"); - check(state, 1, + check(state, 1, {"field"}, "SELECT field FROM table WHERE isNotNull(field)", R"(SELECT "field" FROM "test"."table" WHERE "field" IS NOT NULL)"); } @@ -278,7 +370,28 @@ TEST(TransformQueryForExternalDatabase, ToDate) { const State & state = State::instance(); - check(state, 1, + check(state, 1, {"a", "b", "foo"}, "SELECT foo FROM table WHERE a=10 AND b=toDate('2019-10-05')", R"(SELECT "a", "b", "foo" FROM "test"."table" WHERE ("a" = 10) AND ("b" = '2019-10-05'))"); } + +TEST(TransformQueryForExternalDatabase, Analyzer) +{ + const State & state = State::instance(); + + check(state, 1, {"field"}, + "SELECT count() FROM table WHERE field LIKE '%name_%'", + R"(SELECT "field" FROM "test"."table" WHERE "field" LIKE '%name_%')"); + + check(state, 1, {"column"}, + "SELECT 1 FROM table", + R"(SELECT "column" FROM "test"."table")"); + + check(state, 1, {"column"}, + "SELECT sleepEachRow(1) FROM table", + R"(SELECT "column" FROM "test"."table")"); + + check(state, 1, {"column", "apply_id", "apply_type", "apply_status", "create_time", "field", "value", "a", "b", "foo"}, + "SELECT * FROM table WHERE (column) IN (1)", + R"(SELECT "column", "apply_id", "apply_type", "apply_status", "create_time", "field", "value", "a", "b", "foo" FROM "test"."table" WHERE "column" IN (1))"); +} diff --git a/src/Storages/transformQueryForExternalDatabase.cpp b/src/Storages/transformQueryForExternalDatabase.cpp index 1ff310c3fac..da40673d979 100644 --- a/src/Storages/transformQueryForExternalDatabase.cpp +++ b/src/Storages/transformQueryForExternalDatabase.cpp @@ -14,6 +14,8 @@ #include #include +#include + namespace DB { @@ -108,9 +110,9 @@ void dropAliases(ASTPtr & node) } -bool isCompatible(IAST & node) +bool isCompatible(ASTPtr & node) { - if (auto * function = node.as()) + if (auto * function = node->as()) { if (function->parameters) /// Parametric aggregate functions return false; @@ -154,20 +156,30 @@ bool isCompatible(IAST & node) && (function->arguments->children.size() != 2 || function->arguments->children[1]->as())) return false; - for (const auto & expr : function->arguments->children) - if (!isCompatible(*expr)) + for (auto & expr : function->arguments->children) + if (!isCompatible(expr)) return false; return true; } - if (const auto * literal = node.as()) + if (const auto * literal = node->as()) { + if (literal->value.getType() == Field::Types::Tuple) + { + /// Represent a tuple with zero or one elements as (x) instead of tuple(x). + auto tuple_value = literal->value.safeGet(); + if (tuple_value.size() == 1) + { + node = makeASTFunction("", std::make_shared(tuple_value[0])); + return true; + } + } /// Foreign databases often have no support for Array. But Tuple literals are passed to support IN clause. return literal->value.getType() != Field::Types::Array; } - return node.as(); + return node->as(); } bool removeUnknownSubexpressions(ASTPtr & node, const NameSet & known_names); @@ -241,23 +253,15 @@ bool removeUnknownSubexpressionsFromWhere(ASTPtr & node, const NamesAndTypesList return removeUnknownSubexpressions(node, known_names); } -} - -String transformQueryForExternalDatabase( - const SelectQueryInfo & query_info, +String transformQueryForExternalDatabaseImpl( + ASTPtr clone_query, + Names used_columns, const NamesAndTypesList & available_columns, IdentifierQuotingStyle identifier_quoting_style, const String & database, const String & table, ContextPtr context) { - auto clone_query = query_info.query->clone(); - - /// TODO: Analyzer syntax analyzer result - if (!query_info.syntax_analyzer_result) - throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "transform query for external database is unsupported"); - - const Names used_columns = query_info.syntax_analyzer_result->requiredSourceColumns(); bool strict = context->getSettingsRef().external_table_strict_query; auto select = std::make_shared(); @@ -278,11 +282,12 @@ String transformQueryForExternalDatabase( ASTPtr original_where = clone_query->as().where(); bool where_has_known_columns = removeUnknownSubexpressionsFromWhere(original_where, available_columns); + if (original_where && where_has_known_columns) { replaceConstantExpressions(original_where, context, available_columns); - if (isCompatible(*original_where)) + if (isCompatible(original_where)) { select->setExpression(ASTSelectQuery::Expression::WHERE, std::move(original_where)); } @@ -290,14 +295,14 @@ String transformQueryForExternalDatabase( { throw Exception(ErrorCodes::INCORRECT_QUERY, "Query contains non-compatible expressions (and external_table_strict_query=true)"); } - else if (const auto * function = original_where->as()) + else if (auto * function = original_where->as()) { if (function->name == "and") { auto new_function_and = makeASTFunction("and"); - for (const auto & elem : function->arguments->children) + for (auto & elem : function->arguments->children) { - if (isCompatible(*elem)) + if (isCompatible(elem)) new_function_and->arguments->children.push_back(elem); } if (new_function_and->arguments->children.size() == 1) @@ -309,7 +314,8 @@ String transformQueryForExternalDatabase( } else if (strict && original_where) { - throw Exception(ErrorCodes::INCORRECT_QUERY, "Query contains non-compatible expressions (and external_table_strict_query=true)"); + throw Exception(ErrorCodes::INCORRECT_QUERY, "Query contains non-compatible expressions '{}' (and external_table_strict_query=true)", + original_where->formatForErrorMessage()); } auto * literal_expr = typeid_cast(original_where.get()); @@ -338,3 +344,50 @@ String transformQueryForExternalDatabase( } } + +String transformQueryForExternalDatabase( + const SelectQueryInfo & query_info, + const Names & column_names, + const NamesAndTypesList & available_columns, + IdentifierQuotingStyle identifier_quoting_style, + const String & database, + const String & table, + ContextPtr context) +{ + if (!query_info.syntax_analyzer_result) + { + if (!query_info.query_tree) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Query is not analyzed: no query tree"); + if (!query_info.planner_context) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Query is not analyzed: no planner context"); + if (!query_info.table_expression) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Query is not analyzed: no table expression"); + + if (column_names.empty()) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "No column names for query '{}' to external table '{}.{}'", + database, table, query_info.query_tree->formatASTForErrorMessage()); + + auto clone_query = getASTForExternalDatabaseFromQueryTree(query_info.query_tree); + + return transformQueryForExternalDatabaseImpl( + clone_query, + column_names, + available_columns, + identifier_quoting_style, + database, + table, + context); + } + + auto clone_query = query_info.query->clone(); + return transformQueryForExternalDatabaseImpl( + clone_query, + query_info.syntax_analyzer_result->requiredSourceColumns(), + available_columns, + identifier_quoting_style, + database, + table, + context); +} + +} diff --git a/src/Storages/transformQueryForExternalDatabase.h b/src/Storages/transformQueryForExternalDatabase.h index 3756dd97feb..0f2b0a5822f 100644 --- a/src/Storages/transformQueryForExternalDatabase.h +++ b/src/Storages/transformQueryForExternalDatabase.h @@ -28,6 +28,7 @@ class IAST; */ String transformQueryForExternalDatabase( const SelectQueryInfo & query_info, + const Names & column_names, const NamesAndTypesList & available_columns, IdentifierQuotingStyle identifier_quoting_style, const String & database, diff --git a/src/Storages/transformQueryForExternalDatabaseAnalyzer.cpp b/src/Storages/transformQueryForExternalDatabaseAnalyzer.cpp new file mode 100644 index 00000000000..782515d795e --- /dev/null +++ b/src/Storages/transformQueryForExternalDatabaseAnalyzer.cpp @@ -0,0 +1,81 @@ +#include + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + + +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int UNSUPPORTED_METHOD; +} + +namespace +{ + +class PrepareForExternalDatabaseVisitor : public InDepthQueryTreeVisitor +{ +public: + using Base = InDepthQueryTreeVisitor; + using Base::Base; + + void visitImpl(QueryTreeNodePtr & node) + { + auto * constant_node = node->as(); + if (constant_node) + { + auto result_type = constant_node->getResultType(); + if (isDate(result_type) || isDateTime(result_type) || isDateTime64(result_type)) + { + /// Use string representation of constant date and time values + /// The code is ugly - how to convert artbitrary Field to proper string representation? + /// (maybe we can just consider numbers as unix timestamps?) + auto result_column = result_type->createColumnConst(1, constant_node->getValue()); + const IColumn & inner_column = assert_cast(*result_column).getDataColumn(); + + WriteBufferFromOwnString out; + result_type->getDefaultSerialization()->serializeText(inner_column, 0, out, FormatSettings()); + node = std::make_shared(std::make_shared(out.str(), result_type)); + } + } + } +}; + +} + +ASTPtr getASTForExternalDatabaseFromQueryTree(const QueryTreeNodePtr & query_tree) +{ + auto new_tree = query_tree->clone(); + + PrepareForExternalDatabaseVisitor visitor; + visitor.visit(new_tree); + const auto * query_node = new_tree->as(); + + const auto & query_node_ast = query_node->toAST({ .add_cast_for_constants = false, .fully_qualified_identifiers = false }); + + const auto * union_ast = query_node_ast->as(); + if (!union_ast) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "QueryNode AST is not a ASTSelectWithUnionQuery"); + + if (union_ast->list_of_selects->children.size() != 1) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "QueryNode AST is not a single ASTSelectQuery, got {}", union_ast->list_of_selects->children.size()); + + return union_ast->list_of_selects->children.at(0); +} + +} diff --git a/src/Storages/transformQueryForExternalDatabaseAnalyzer.h b/src/Storages/transformQueryForExternalDatabaseAnalyzer.h new file mode 100644 index 00000000000..f8983619d1f --- /dev/null +++ b/src/Storages/transformQueryForExternalDatabaseAnalyzer.h @@ -0,0 +1,11 @@ +#pragma once + +#include + + +namespace DB +{ + +ASTPtr getASTForExternalDatabaseFromQueryTree(const QueryTreeNodePtr & query_tree); + +} diff --git a/tests/queries/0_stateless/02479_mysql_connect_to_self.reference b/tests/queries/0_stateless/02479_mysql_connect_to_self.reference index 573541ac970..07cd8dea949 100644 --- a/tests/queries/0_stateless/02479_mysql_connect_to_self.reference +++ b/tests/queries/0_stateless/02479_mysql_connect_to_self.reference @@ -1 +1,39 @@ -0 +--- +1 one -1 een +2 two -2 twee +3 three -3 drie +4 four -4 vier +5 five -5 vijf +--- +5 +--- +1 +1 +1 +1 +1 +--- +1 +2 +3 +4 +5 +--- +-5 five +-4 four +-1 one +-3 three +-2 two +--- +-3 three +-1 one +-2 two +-4 four +-5 five +--- +-1 +-3 +-4 +-5 +--- +4 diff --git a/tests/queries/0_stateless/02479_mysql_connect_to_self.sql b/tests/queries/0_stateless/02479_mysql_connect_to_self.sql index 7ff5b3e3382..832d4a6347d 100644 --- a/tests/queries/0_stateless/02479_mysql_connect_to_self.sql +++ b/tests/queries/0_stateless/02479_mysql_connect_to_self.sql @@ -1,4 +1,32 @@ -- Tags: no-fasttest -SELECT * -FROM mysql('127.0.0.1:9004', system, one, 'default', '') -SETTINGS send_logs_level = 'fatal'; -- failed connection tries are ok, if it succeeded after retry. + +SET send_logs_level = 'fatal'; -- failed connection tries are ok, if it succeeded after retry. + +CREATE TABLE foo (key UInt32, a String, b Int64, c String) ENGINE = TinyLog; +INSERT INTO foo VALUES (1, 'one', -1, 'een'), (2, 'two', -2, 'twee'), (3, 'three', -3, 'drie'), (4, 'four', -4, 'vier'), (5, 'five', -5, 'vijf'); + +SET allow_experimental_analyzer = 1; + +SELECT '---'; +SELECT * FROM mysql('127.0.0.1:9004', currentDatabase(), foo, 'default', '') ORDER BY key; + +SELECT '---'; +SELECT count() FROM mysql('127.0.0.1:9004', currentDatabase(), foo, 'default', ''); + +SELECT '---'; +SELECT 1 FROM mysql('127.0.0.1:9004', currentDatabase(), foo, 'default', ''); + +SELECT '---'; +SELECT key FROM mysql('127.0.0.1:9004', currentDatabase(), foo, 'default', '') ORDER BY key; + +SELECT '---'; +SELECT b, a FROM mysql('127.0.0.1:9004', currentDatabase(), foo, 'default', '') ORDER BY a; + +SELECT '---'; +SELECT b, a FROM mysql('127.0.0.1:9004', currentDatabase(), foo, 'default', '') ORDER BY c; + +SELECT '---'; +SELECT b FROM mysql('127.0.0.1:9004', currentDatabase(), foo, 'default', '') WHERE c != 'twee' ORDER BY b; + +SELECT '---'; +SELECT count() FROM mysql('127.0.0.1:9004', currentDatabase(), foo, 'default', '') WHERE c != 'twee';