thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-23 16:12:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Support transformQueryForExternalDatabase for analyzer

Browse Source
This commit is contained in:
vdimir 2023-03-07 20:39:26 +00:00 committed by Vladimir C
parent 069cac8d6e
commit 023d14a894
59 changed files with 555 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/Analyzer/ArrayJoinNode.cpp
View File

@ -49,7 +49,7 @@ QueryTreeNodePtr ArrayJoinNode::cloneImpl() const
return std::make_shared<ArrayJoinNode>(getTableExpression(), getJoinExpressionsNode(), is_left);
}
ASTPtr ArrayJoinNode::toASTImpl() const
ASTPtr ArrayJoinNode::toASTImpl(ConvertToASTOptions options) const
{
auto array_join_ast = std::make_shared<ASTArrayJoin>();
array_join_ast->kind = is_left ? ASTArrayJoin::Kind::Left : ASTArrayJoin::Kind::Inner;
@ -63,9 +63,9 @@ ASTPtr ArrayJoinNode::toASTImpl() const
auto * column_node = array_join_expression->as<ColumnNode>();
if (column_node && column_node->getExpression())
array_join_expression_ast = column_node->getExpression()->toAST();
array_join_expression_ast = column_node->getExpression()->toAST(options);
else
array_join_expression_ast = array_join_expression->toAST();
array_join_expression_ast = array_join_expression->toAST(options);
array_join_expression_ast->setAlias(array_join_expression->getAlias());
array_join_expressions_ast->children.push_back(std::move(array_join_expression_ast));

2
src/Analyzer/ArrayJoinNode.h
View File

@ -99,7 +99,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
private:
bool is_left = false;

4
src/Analyzer/ColumnNode.cpp
View File

@ -91,12 +91,12 @@ QueryTreeNodePtr ColumnNode::cloneImpl() const
return std::make_shared<ColumnNode>(column, getSourceWeakPointer());
}
ASTPtr ColumnNode::toASTImpl() const
ASTPtr ColumnNode::toASTImpl(ConvertToASTOptions options) const
{
std::vector<std::string> column_identifier_parts;
auto column_source = getColumnSourceOrNull();
if (column_source)
if (column_source && options.fully_qualified_identifiers)
{
auto node_type = column_source->getNodeType();
if (node_type == QueryTreeNodeType::TABLE ||

7
src/Analyzer/ColumnNode.h
View File

@ -103,6 +103,11 @@ public:
*/
QueryTreeNodePtr getColumnSource() const;
void dropColumnSource()
{
getSourceWeakPointer().reset();
}
/** Get column source.
* If column source is not valid null is returned.
*/
@ -132,7 +137,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
private:
const QueryTreeNodeWeakPtr & getSourceWeakPointer() const

12
src/Analyzer/ColumnTransformers.cpp
View File

@ -91,7 +91,7 @@ QueryTreeNodePtr ApplyColumnTransformerNode::cloneImpl() const
return std::make_shared<ApplyColumnTransformerNode>(getExpressionNode());
}
ASTPtr ApplyColumnTransformerNode::toASTImpl() const
ASTPtr ApplyColumnTransformerNode::toASTImpl(ConvertToASTOptions options) const
{
auto ast_apply_transformer = std::make_shared<ASTColumnsApplyTransformer>();
const auto & expression_node = getExpressionNode();
@ -100,14 +100,14 @@ ASTPtr ApplyColumnTransformerNode::toASTImpl() const
{
auto & function_expression = expression_node->as<FunctionNode &>();
ast_apply_transformer->func_name = function_expression.getFunctionName();
ast_apply_transformer->parameters = function_expression.getParametersNode()->toAST();
ast_apply_transformer->parameters = function_expression.getParametersNode()->toAST(options);
}
else
{
auto & lambda_expression = expression_node->as<LambdaNode &>();
if (!lambda_expression.getArgumentNames().empty())
ast_apply_transformer->lambda_arg = lambda_expression.getArgumentNames()[0];
ast_apply_transformer->lambda = lambda_expression.toAST();
ast_apply_transformer->lambda = lambda_expression.toAST(options);
}
return ast_apply_transformer;
@ -227,7 +227,7 @@ QueryTreeNodePtr ExceptColumnTransformerNode::cloneImpl() const
return std::make_shared<ExceptColumnTransformerNode>(except_column_names, is_strict);
}
ASTPtr ExceptColumnTransformerNode::toASTImpl() const
ASTPtr ExceptColumnTransformerNode::toASTImpl(ConvertToASTOptions /* options */) const
{
auto ast_except_transformer = std::make_shared<ASTColumnsExceptTransformer>();
@ -334,7 +334,7 @@ QueryTreeNodePtr ReplaceColumnTransformerNode::cloneImpl() const
return result_replace_transformer;
}
ASTPtr ReplaceColumnTransformerNode::toASTImpl() const
ASTPtr ReplaceColumnTransformerNode::toASTImpl(ConvertToASTOptions options) const
{
auto ast_replace_transformer = std::make_shared<ASTColumnsReplaceTransformer>();
@ -347,7 +347,7 @@ ASTPtr ReplaceColumnTransformerNode::toASTImpl() const
{
auto replacement_ast = std::make_shared<ASTColumnsReplaceTransformer::Replacement>();
replacement_ast->name = replacements_names[i];
replacement_ast->children.push_back(replacement_expressions_nodes[i]->toAST());
replacement_ast->children.push_back(replacement_expressions_nodes[i]->toAST(options));
ast_replace_transformer->children.push_back(std::move(replacement_ast));
}

6
src/Analyzer/ColumnTransformers.h
View File

@ -141,7 +141,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
private:
ApplyColumnTransformerType apply_transformer_type = ApplyColumnTransformerType::LAMBDA;
@ -220,7 +220,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
private:
ExceptColumnTransformerType except_transformer_type;
@ -298,7 +298,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
private:
ListNode & getReplacements()

5
src/Analyzer/ConstantNode.cpp
View File

@ -75,11 +75,14 @@ QueryTreeNodePtr ConstantNode::cloneImpl() const
return std::make_shared<ConstantNode>(constant_value, source_expression);
}
ASTPtr ConstantNode::toASTImpl() const
ASTPtr ConstantNode::toASTImpl(ConvertToASTOptions options) const
{
const auto & constant_value_literal = constant_value->getValue();
auto constant_value_ast = std::make_shared<ASTLiteral>(constant_value_literal);
if (!options.add_cast_for_constants)
return constant_value_ast;
bool need_to_add_cast_function = false;
auto constant_value_literal_type = constant_value_literal.getType();
WhichDataType constant_value_type(constant_value->getType());

2
src/Analyzer/ConstantNode.h
View File

@ -83,7 +83,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
private:
ConstantValuePtr constant_value;

8
src/Analyzer/FunctionNode.cpp
View File

@ -197,7 +197,7 @@ QueryTreeNodePtr FunctionNode::cloneImpl() const
return result_function;
}
ASTPtr FunctionNode::toASTImpl() const
ASTPtr FunctionNode::toASTImpl(ConvertToASTOptions options) const
{
auto function_ast = std::make_shared<ASTFunction>();
@ -212,12 +212,12 @@ ASTPtr FunctionNode::toASTImpl() const
const auto & parameters = getParameters();
if (!parameters.getNodes().empty())
{
function_ast->children.push_back(parameters.toAST());
function_ast->children.push_back(parameters.toAST(options));
function_ast->parameters = function_ast->children.back();
}
const auto & arguments = getArguments();
function_ast->children.push_back(arguments.toAST());
function_ast->children.push_back(arguments.toAST(options));
function_ast->arguments = function_ast->children.back();
auto window_node = getWindowNode();
@ -226,7 +226,7 @@ ASTPtr FunctionNode::toASTImpl() const
if (auto * identifier_node = window_node->as<IdentifierNode>())
function_ast->window_name = identifier_node->getIdentifier().getFullName();
else
function_ast->window_definition = window_node->toAST();
function_ast->window_definition = window_node->toAST(options);
}
return function_ast;

2
src/Analyzer/FunctionNode.h
View File

@ -209,7 +209,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
private:
String function_name;

4
src/Analyzer/IQueryTreeNode.cpp
View File

@ -331,9 +331,9 @@ QueryTreeNodePtr IQueryTreeNode::cloneAndReplace(const QueryTreeNodePtr & node_t
return cloneAndReplace(replacement_map);
}
ASTPtr IQueryTreeNode::toAST() const
ASTPtr IQueryTreeNode::toAST(ConvertToASTOptions options) const
{
auto converted_node = toASTImpl();
auto converted_node = toASTImpl(options);
if (auto * ast_with_alias = dynamic_cast<ASTWithAlias *>(converted_node.get()))
converted_node->setAlias(alias);

13
src/Analyzer/IQueryTreeNode.h
View File

@ -181,8 +181,17 @@ public:
*/
String formatOriginalASTForErrorMessage() const;
struct ConvertToASTOptions
{
/// Add _CAST if constant litral type is different from column type
bool add_cast_for_constants = true;
/// Identifiers are fully qualified (`database.table.column`), otherwise names are just column names (`column`)
bool fully_qualified_identifiers = true;
};
/// Convert query tree to AST
ASTPtr toAST() const;
ASTPtr toAST(ConvertToASTOptions options = { .add_cast_for_constants = true, .fully_qualified_identifiers = true }) const;
/// Convert query tree to AST and then format it for error message.
String formatConvertedASTForErrorMessage() const;
@ -258,7 +267,7 @@ protected:
virtual QueryTreeNodePtr cloneImpl() const = 0;
/// Subclass must convert its internal state and its children to AST
virtual ASTPtr toASTImpl() const = 0;
virtual ASTPtr toASTImpl(ConvertToASTOptions options) const = 0;
QueryTreeNodes children;
QueryTreeWeakNodes weak_pointers;

2
src/Analyzer/IdentifierNode.cpp
View File

@ -58,7 +58,7 @@ QueryTreeNodePtr IdentifierNode::cloneImpl() const
return std::make_shared<IdentifierNode>(identifier);
}
ASTPtr IdentifierNode::toASTImpl() const
ASTPtr IdentifierNode::toASTImpl(ConvertToASTOptions /* options */) const
{
auto identifier_parts = identifier.getParts();
return std::make_shared<ASTIdentifier>(std::move(identifier_parts));

2
src/Analyzer/IdentifierNode.h
View File

@ -59,7 +59,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
private:
Identifier identifier;

6
src/Analyzer/InterpolateNode.cpp
View File

@ -44,11 +44,11 @@ QueryTreeNodePtr InterpolateNode::cloneImpl() const
return std::make_shared<InterpolateNode>(nullptr /*expression*/, nullptr /*interpolate_expression*/);
}
ASTPtr InterpolateNode::toASTImpl() const
ASTPtr InterpolateNode::toASTImpl(ConvertToASTOptions options) const
{
auto result = std::make_shared<ASTInterpolateElement>();
result->column = getExpression()->toAST()->getColumnName();
result->children.push_back(getInterpolateExpression()->toAST());
result->column = getExpression()->toAST(options)->getColumnName();
result->children.push_back(getInterpolateExpression()->toAST(options));
result->expr = result->children.back();
return result;

2
src/Analyzer/InterpolateNode.h
View File

@ -59,7 +59,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
private:
static constexpr size_t expression_child_index = 0;

2
src/Analyzer/JoinNode.cpp
View File

@ -99,7 +99,7 @@ QueryTreeNodePtr JoinNode::cloneImpl() const
return std::make_shared<JoinNode>(getLeftTableExpression(), getRightTableExpression(), getJoinExpression(), locality, strictness, kind);
}
ASTPtr JoinNode::toASTImpl() const
ASTPtr JoinNode::toASTImpl(ConvertToASTOptions /* options */) const
{
ASTPtr tables_in_select_query_ast = std::make_shared<ASTTablesInSelectQuery>();

2
src/Analyzer/JoinNode.h
View File

@ -148,7 +148,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
private:
JoinLocality locality = JoinLocality::Unspecified;

6
src/Analyzer/LambdaNode.cpp
View File

@ -65,17 +65,17 @@ QueryTreeNodePtr LambdaNode::cloneImpl() const
return std::make_shared<LambdaNode>(argument_names, getExpression());
}
ASTPtr LambdaNode::toASTImpl() const
ASTPtr LambdaNode::toASTImpl(ConvertToASTOptions options) const
{
auto lambda_function_arguments_ast = std::make_shared<ASTExpressionList>();
auto tuple_function = std::make_shared<ASTFunction>();
tuple_function->name = "tuple";
tuple_function->children.push_back(children[arguments_child_index]->toAST());
tuple_function->children.push_back(children[arguments_child_index]->toAST(options));
tuple_function->arguments = tuple_function->children.back();
lambda_function_arguments_ast->children.push_back(std::move(tuple_function));
lambda_function_arguments_ast->children.push_back(children[expression_child_index]->toAST());
lambda_function_arguments_ast->children.push_back(children[expression_child_index]->toAST(options));
auto lambda_function_ast = std::make_shared<ASTFunction>();
lambda_function_ast->name = "lambda";

2
src/Analyzer/LambdaNode.h
View File

@ -98,7 +98,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
private:
Names argument_names;

4
src/Analyzer/ListNode.cpp
View File

@ -54,7 +54,7 @@ QueryTreeNodePtr ListNode::cloneImpl() const
return std::make_shared<ListNode>();
}
ASTPtr ListNode::toASTImpl() const
ASTPtr ListNode::toASTImpl(ConvertToASTOptions options) const
{
auto expression_list_ast = std::make_shared<ASTExpressionList>();
@ -62,7 +62,7 @@ ASTPtr ListNode::toASTImpl() const
expression_list_ast->children.resize(children_size);
for (size_t i = 0; i < children_size; ++i)
expression_list_ast->children[i] = children[i]->toAST();
expression_list_ast->children[i] = children[i]->toAST(options);
return expression_list_ast;
}

2
src/Analyzer/ListNode.h
View File

@ -57,7 +57,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
};
}

4
src/Analyzer/MatcherNode.cpp
View File

@ -204,7 +204,7 @@ QueryTreeNodePtr MatcherNode::cloneImpl() const
return matcher_node;
}
ASTPtr MatcherNode::toASTImpl() const
ASTPtr MatcherNode::toASTImpl(ConvertToASTOptions options) const
{
ASTPtr result;
ASTPtr transformers;
@ -216,7 +216,7 @@ ASTPtr MatcherNode::toASTImpl() const
transformers = std::make_shared<ASTColumnsTransformerList>();
for (const auto & column_transformer : column_transformers)
transformers->children.push_back(column_transformer->toAST());
transformers->children.push_back(column_transformer->toAST(options));
}
if (matcher_type == MatcherNodeType::ASTERISK)

2
src/Analyzer/MatcherNode.h
View File

@ -148,7 +148,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
private:
explicit MatcherNode(MatcherNodeType matcher_type_,

30
src/Analyzer/QueryNode.cpp
View File

@ -259,7 +259,7 @@ QueryTreeNodePtr QueryNode::cloneImpl() const
return result_query_node;
}
ASTPtr QueryNode::toASTImpl() const
ASTPtr QueryNode::toASTImpl(ConvertToASTOptions options) const
{
auto select_query = std::make_shared<ASTSelectQuery>();
select_query->distinct = is_distinct;
@ -271,9 +271,9 @@ ASTPtr QueryNode::toASTImpl() const
select_query->group_by_all = is_group_by_all;
if (hasWith())
select_query->setExpression(ASTSelectQuery::Expression::WITH, getWith().toAST());
select_query->setExpression(ASTSelectQuery::Expression::WITH, getWith().toAST(options));
auto projection_ast = getProjection().toAST();
auto projection_ast = getProjection().toAST(options);
auto & projection_expression_list_ast = projection_ast->as<ASTExpressionList &>();
size_t projection_expression_list_ast_children_size = projection_expression_list_ast.children.size();
if (projection_expression_list_ast_children_size != getProjection().getNodes().size())
@ -297,40 +297,40 @@ ASTPtr QueryNode::toASTImpl() const
select_query->setExpression(ASTSelectQuery::Expression::TABLES, std::move(tables_in_select_query_ast));
if (getPrewhere())
select_query->setExpression(ASTSelectQuery::Expression::PREWHERE, getPrewhere()->toAST());
select_query->setExpression(ASTSelectQuery::Expression::PREWHERE, getPrewhere()->toAST(options));
if (getWhere())
select_query->setExpression(ASTSelectQuery::Expression::WHERE, getWhere()->toAST());
select_query->setExpression(ASTSelectQuery::Expression::WHERE, getWhere()->toAST(options));
if (!is_group_by_all && hasGroupBy())
select_query->setExpression(ASTSelectQuery::Expression::GROUP_BY, getGroupBy().toAST());
select_query->setExpression(ASTSelectQuery::Expression::GROUP_BY, getGroupBy().toAST(options));
if (hasHaving())
select_query->setExpression(ASTSelectQuery::Expression::HAVING, getHaving()->toAST());
select_query->setExpression(ASTSelectQuery::Expression::HAVING, getHaving()->toAST(options));
if (hasWindow())
select_query->setExpression(ASTSelectQuery::Expression::WINDOW, getWindow().toAST());
select_query->setExpression(ASTSelectQuery::Expression::WINDOW, getWindow().toAST(options));
if (hasOrderBy())
select_query->setExpression(ASTSelectQuery::Expression::ORDER_BY, getOrderBy().toAST());
select_query->setExpression(ASTSelectQuery::Expression::ORDER_BY, getOrderBy().toAST(options));
if (hasInterpolate())
select_query->setExpression(ASTSelectQuery::Expression::INTERPOLATE, getInterpolate()->toAST());
select_query->setExpression(ASTSelectQuery::Expression::INTERPOLATE, getInterpolate()->toAST(options));
if (hasLimitByLimit())
select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY_LENGTH, getLimitByLimit()->toAST());
select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY_LENGTH, getLimitByLimit()->toAST(options));
if (hasLimitByOffset())
select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY_OFFSET, getLimitByOffset()->toAST());
select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY_OFFSET, getLimitByOffset()->toAST(options));
if (hasLimitBy())
select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY, getLimitBy().toAST());
select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY, getLimitBy().toAST(options));
if (hasLimit())
select_query->setExpression(ASTSelectQuery::Expression::LIMIT_LENGTH, getLimit()->toAST());
select_query->setExpression(ASTSelectQuery::Expression::LIMIT_LENGTH, getLimit()->toAST(options));
if (hasOffset())
select_query->setExpression(ASTSelectQuery::Expression::LIMIT_OFFSET, getOffset()->toAST());
select_query->setExpression(ASTSelectQuery::Expression::LIMIT_OFFSET, getOffset()->toAST(options));
if (hasSettingsChanges())
{

2
src/Analyzer/QueryNode.h
View File

@ -575,7 +575,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
private:
bool is_subquery = false;

10
src/Analyzer/SortNode.cpp
View File

@ -109,7 +109,7 @@ QueryTreeNodePtr SortNode::cloneImpl() const
return std::make_shared<SortNode>(nullptr /*expression*/, sort_direction, nulls_sort_direction, collator, with_fill);
}
ASTPtr SortNode::toASTImpl() const
ASTPtr SortNode::toASTImpl(ConvertToASTOptions options) const
{
auto result = std::make_shared<ASTOrderByElement>();
result->direction = sort_direction == SortDirection::ASCENDING ? 1 : -1;
@ -120,10 +120,10 @@ ASTPtr SortNode::toASTImpl() const
result->nulls_direction_was_explicitly_specified = nulls_sort_direction.has_value();
result->with_fill = with_fill;
result->fill_from = hasFillFrom() ? getFillFrom()->toAST() : nullptr;
result->fill_to = hasFillTo() ? getFillTo()->toAST() : nullptr;
result->fill_step = hasFillStep() ? getFillStep()->toAST() : nullptr;
result->children.push_back(getExpression()->toAST());
result->fill_from = hasFillFrom() ? getFillFrom()->toAST(options) : nullptr;
result->fill_to = hasFillTo() ? getFillTo()->toAST(options) : nullptr;
result->fill_step = hasFillStep() ? getFillStep()->toAST(options) : nullptr;
result->children.push_back(getExpression()->toAST(options));
if (collator)
{

2
src/Analyzer/SortNode.h
View File

@ -137,7 +137,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
private:
static constexpr size_t sort_expression_child_index = 0;

4
src/Analyzer/TableFunctionNode.cpp
View File

@ -113,14 +113,14 @@ QueryTreeNodePtr TableFunctionNode::cloneImpl() const
return result;
}
ASTPtr TableFunctionNode::toASTImpl() const
ASTPtr TableFunctionNode::toASTImpl(ConvertToASTOptions options) const
{
auto table_function_ast = std::make_shared<ASTFunction>();
table_function_ast->name = table_function_name;
const auto & arguments = getArguments();
table_function_ast->children.push_back(arguments.toAST());
table_function_ast->children.push_back(arguments.toAST(options));
table_function_ast->arguments = table_function_ast->children.back();
return table_function_ast;

2
src/Analyzer/TableFunctionNode.h
View File

@ -142,7 +142,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
private:
String table_function_name;

2
src/Analyzer/TableNode.cpp
View File

@ -86,7 +86,7 @@ QueryTreeNodePtr TableNode::cloneImpl() const
return result_table_node;
}
ASTPtr TableNode::toASTImpl() const
ASTPtr TableNode::toASTImpl(ConvertToASTOptions /* options */) const
{
if (!temporary_table_name.empty())
return std::make_shared<ASTTableIdentifier>(temporary_table_name);

2
src/Analyzer/TableNode.h
View File

@ -106,7 +106,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
private:
StoragePtr storage;

4
src/Analyzer/UnionNode.cpp
View File

@ -140,12 +140,12 @@ QueryTreeNodePtr UnionNode::cloneImpl() const
return result_union_node;
}
ASTPtr UnionNode::toASTImpl() const
ASTPtr UnionNode::toASTImpl(ConvertToASTOptions options) const
{
auto select_with_union_query = std::make_shared<ASTSelectWithUnionQuery>();
select_with_union_query->union_mode = union_mode;
select_with_union_query->is_normalized = true;
select_with_union_query->children.push_back(getQueriesNode()->toAST());
select_with_union_query->children.push_back(getQueriesNode()->toAST(options));
select_with_union_query->list_of_selects = select_with_union_query->children.back();
if (is_subquery)

2
src/Analyzer/UnionNode.h
View File

@ -143,7 +143,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
private:
bool is_subquery = false;

10
src/Analyzer/WindowNode.cpp
View File

@ -107,7 +107,7 @@ QueryTreeNodePtr WindowNode::cloneImpl() const
return window_node;
}
ASTPtr WindowNode::toASTImpl() const
ASTPtr WindowNode::toASTImpl(ConvertToASTOptions options) const
{
auto window_definition = std::make_shared<ASTWindowDefinition>();
@ -115,13 +115,13 @@ ASTPtr WindowNode::toASTImpl() const
if (hasPartitionBy())
{
window_definition->children.push_back(getPartitionByNode()->toAST());
window_definition->children.push_back(getPartitionByNode()->toAST(options));
window_definition->partition_by = window_definition->children.back();
}
if (hasOrderBy())
{
window_definition->children.push_back(getOrderByNode()->toAST());
window_definition->children.push_back(getOrderByNode()->toAST(options));
window_definition->order_by = window_definition->children.back();
}
@ -132,7 +132,7 @@ ASTPtr WindowNode::toASTImpl() const
if (hasFrameBeginOffset())
{
window_definition->children.push_back(getFrameBeginOffsetNode()->toAST());
window_definition->children.push_back(getFrameBeginOffsetNode()->toAST(options));
window_definition->frame_begin_offset = window_definition->children.back();
}
@ -140,7 +140,7 @@ ASTPtr WindowNode::toASTImpl() const
window_definition->frame_end_preceding = window_frame.end_preceding;
if (hasFrameEndOffset())
{
window_definition->children.push_back(getFrameEndOffsetNode()->toAST());
window_definition->children.push_back(getFrameEndOffsetNode()->toAST(options));
window_definition->frame_end_offset = window_definition->children.back();
}

2
src/Analyzer/WindowNode.h
View File

@ -175,7 +175,7 @@ protected:
QueryTreeNodePtr cloneImpl() const override;
ASTPtr toASTImpl() const override;
ASTPtr toASTImpl(ConvertToASTOptions options) const override;
private:
static constexpr size_t order_by_child_index = 0;

2
src/Analyzer/tests/gtest_query_tree_node.cpp
View File

@ -36,7 +36,7 @@ public:
return std::make_shared<SourceNode>();
}
ASTPtr toASTImpl() const override
ASTPtr toASTImpl(ConvertToASTOptions /* options */) const override
{
return nullptr;
}

6
src/Common/mysqlxx/Exception.cpp
View File

@ -10,9 +10,11 @@
namespace mysqlxx
{
std::string errorMessage(MYSQL * driver)
std::string errorMessage(MYSQL * driver, const std::string & query)
{
return fmt::format("{} ({}:{})", mysql_error(driver), driver->host ? driver->host : "(nullptr)", driver->port);
return fmt::format("{}{} ({}:{})", mysql_error(driver),
query.empty() ? "" : " while executing query: '" + query + "'",
driver->host ? driver->host : "(nullptr)", driver->port);
}
void checkError(MYSQL * driver)

2
src/Common/mysqlxx/Query.cpp
View File

@ -64,7 +64,7 @@ void Query::executeImpl()
case CR_SERVER_LOST:
throw ConnectionLost(errorMessage(mysql_driver), err_no);
default:
throw BadQuery(errorMessage(mysql_driver), err_no);
throw BadQuery(errorMessage(mysql_driver, query), err_no);
}
}
}

6
src/Common/mysqlxx/Value.cpp
View File

@ -160,14 +160,16 @@ void Value::throwException(const char * text) const
if (!isNull())
{
info.append(": ");
info.append(": '");
info.append(m_data, m_length);
info.append("'");
}
if (res && res->getQuery())
{
info.append(", query: ");
info.append(", query: '");
info.append(res->getQuery()->str().substr(0, preview_length));
info.append("'");
}
throw CannotParseValue(info);

2
src/Common/mysqlxx/mysqlxx/Exception.h
View File

@ -53,7 +53,7 @@ struct CannotParseValue : public Exception
};
std::string errorMessage(MYSQL * driver);
std::string errorMessage(MYSQL * driver, const std::string & query = "");
/// For internal need of library.
void checkError(MYSQL * driver);

5
src/Common/tests/gtest_global_register.h
View File

@ -1,8 +1,13 @@
#pragma once
#include <Functions/registerFunctions.h>
#include <AggregateFunctions/registerAggregateFunctions.h>
#include <Formats/registerFormats.h>
inline void tryRegisterAggregateFunctions()
{
static struct Register { Register() { DB::registerAggregateFunctions(); } } registered;
}
inline void tryRegisterFunctions()
{

7
src/Interpreters/InterpreterSelectQueryAnalyzer.h
View File

@ -71,6 +71,13 @@ public:
/// Set number_of_current_replica and count_participating_replicas in client_info
void setProperClientInfo(size_t replica_number, size_t count_participating_replicas);
const QueryTreeNodePtr & getQueryTree() const { return query_tree; }
SelectQueryInfo getSelectQueryInfo()
{
planner.buildQueryPlanIfNeeded(); return planner.buildSelectQueryInfo();
}
private:
ASTPtr query;
ContextMutablePtr context;

16
src/Planner/Planner.cpp
View File

@ -1159,11 +1159,7 @@ void Planner::buildPlanForQueryNode()
query_node.getWhere() = {};
}
SelectQueryInfo select_query_info;
select_query_info.original_query = queryNodeToSelectQuery(query_tree);
select_query_info.query = select_query_info.original_query;
select_query_info.query_tree = query_tree;
select_query_info.planner_context = planner_context;
SelectQueryInfo select_query_info = buildSelectQueryInfo();
StorageLimitsList current_storage_limits = storage_limits;
select_query_info.local_storage_limits = buildStorageLimits(*query_context, select_query_options);
@ -1454,6 +1450,16 @@ void Planner::buildPlanForQueryNode()
addBuildSubqueriesForSetsStepIfNeeded(query_plan, select_query_options, planner_context, result_actions_to_execute);
}
SelectQueryInfo Planner::buildSelectQueryInfo() const
{
SelectQueryInfo select_query_info;
select_query_info.original_query = queryNodeToSelectQuery(query_tree);
select_query_info.query = select_query_info.original_query;
select_query_info.query_tree = query_tree;
select_query_info.planner_context = planner_context;
return select_query_info;
}
void Planner::addStorageLimits(const StorageLimitsList & limits)
{
for (const auto & limit : limits)

3
src/Planner/Planner.h
View File

@ -6,6 +6,7 @@
#include <Analyzer/QueryTreePassManager.h>
#include <Processors/QueryPlan/QueryPlan.h>
#include <Interpreters/Context_fwd.h>
#include <Storages/SelectQueryInfo.h>
namespace DB
{
@ -50,6 +51,8 @@ public:
return std::move(query_plan);
}
SelectQueryInfo buildSelectQueryInfo() const;
void addStorageLimits(const StorageLimitsList & limits);
private:

11
src/Processors/Merges/Algorithms/tests/gtest_graphite.cpp
View File

@ -18,17 +18,6 @@
using namespace DB;
static int regAggregateFunctions = 0;
void tryRegisterAggregateFunctions()
{
if (!regAggregateFunctions)
{
registerAggregateFunctions();
regAggregateFunctions = 1;
}
}
static ConfigProcessor::LoadedConfig loadConfiguration(const std::string & config_path)
{
ConfigProcessor config_processor(config_path, true, true);

5
src/Processors/Sources/MySQLSource.cpp
View File

@ -107,6 +107,11 @@ void MySQLWithFailoverSource::onStart()
throw;
}
}
catch (const mysqlxx::BadQuery & e)
{
LOG_ERROR(log, "Error processing query '{}': {}", query_str, e.displayText());
throw;
}
}
initPositionMappingFromQueryResultStructure();

1
src/Storages/SelectQueryInfo.h
View File

@ -190,6 +190,7 @@ struct SelectQueryInfo
PlannerContextPtr planner_context;
/// Storage table expression
/// It's guaranteed to be present in JOIN TREE of `query_tree`
QueryTreeNodePtr table_expression;
/// Table expression modifiers for storage

1
src/Storages/StorageMySQL.cpp
View File

@ -84,6 +84,7 @@ Pipe StorageMySQL::read(
storage_snapshot->check(column_names_);
String query = transformQueryForExternalDatabase(
query_info_,
column_names_,
storage_snapshot->metadata->getColumns().getOrdinary(),
IdentifierQuotingStyle::BackticksMySQL,
remote_database_name,

4
src/Storages/StoragePostgreSQL.cpp
View File

@ -91,7 +91,9 @@ Pipe StoragePostgreSQL::read(
/// Connection is already made to the needed database, so it should not be present in the query;
/// remote_table_schema is empty if it is not specified, will access only table_name.
String query = transformQueryForExternalDatabase(
query_info_, storage_snapshot->metadata->getColumns().getOrdinary(),
query_info_,
column_names_,
storage_snapshot->metadata->getColumns().getOrdinary(),
IdentifierQuotingStyle::DoubleQuotes, remote_table_schema, remote_table_name, context_);
LOG_TRACE(log, "Query: {}", query);

1
src/Storages/StorageSQLite.cpp
View File

@ -66,6 +66,7 @@ Pipe StorageSQLite::read(
String query = transformQueryForExternalDatabase(
query_info,
column_names,
storage_snapshot->metadata->getColumns().getOrdinary(),
IdentifierQuotingStyle::DoubleQuotes,
"",

4
src/Storages/StorageXDBC.cpp
View File

@ -74,7 +74,9 @@ std::function<void(std::ostream &)> StorageXDBC::getReadPOSTDataCallback(
QueryProcessingStage::Enum & /*processed_stage*/,
size_t /*max_block_size*/) const
{
String query = transformQueryForExternalDatabase(query_info,
String query = transformQueryForExternalDatabase(
query_info,
column_names,
columns_description.getOrdinary(),
bridge_helper->getIdentifierQuotingStyle(),
remote_database_name,

223
src/Storages/tests/gtest_transform_query_for_external_database.cpp
View File

@ -9,16 +9,20 @@
#include <DataTypes/DataTypeString.h>
#include <Interpreters/Context.h>
#include <Interpreters/TreeRewriter.h>
#include <Interpreters/InterpreterSelectQueryAnalyzer.h>
#include <Databases/DatabaseMemory.h>
#include <Storages/StorageMemory.h>
#include <Common/tests/gtest_global_context.h>
#include <Common/tests/gtest_global_register.h>
#include <Analyzer/QueryNode.h>
#include <Analyzer/TableNode.h>
#include <Analyzer/JoinNode.h>
using namespace DB;
/// NOTE How to do better?
/// TODO: use gtest fixture
struct State
{
State(const State&) = delete;
@ -31,9 +35,9 @@ struct State
return state;
}
const NamesAndTypesList & getColumns() const
const NamesAndTypesList & getColumns(size_t idx = 0) const
{
return tables[0].columns;
return tables[idx].columns;
}
std::vector<TableWithColumnNamesAndTypes> getTables(size_t num = 0) const
@ -46,10 +50,10 @@ struct State
private:
static DatabaseAndTableWithAlias createDBAndTable(String table_name)
static DatabaseAndTableWithAlias createDBAndTable(String table_name, String database_name = "test")
{
DatabaseAndTableWithAlias res;
res.database = "test";
res.database = database_name;
res.table = table_name;
return res;
}
@ -75,12 +79,18 @@ private:
{"num", std::make_shared<DataTypeUInt8>()},
{"attr", std::make_shared<DataTypeString>()},
}),
TableWithColumnNamesAndTypes(
createDBAndTable("external_table"),
{
{"ttt", std::make_shared<DataTypeUInt8>()},
}),
};
explicit State()
: context(Context::createCopy(getContext().context))
{
tryRegisterFunctions();
tryRegisterAggregateFunctions();
DatabasePtr database = std::make_shared<DatabaseMemory>("test", context);
for (const auto & tab : tables)
@ -91,14 +101,28 @@ private:
context,
table_name,
std::make_shared<StorageMemory>(
StorageID(db_name, table_name), ColumnsDescription{getColumns()}, ConstraintsDescription{}, String{}));
StorageID(db_name, table_name), ColumnsDescription{tab.columns}, ConstraintsDescription{}, String{}));
}
DatabaseCatalog::instance().attachDatabase(database->getDatabaseName(), database);
// DatabaseCatalog::instance().attachDatabase("system", mockSystemDatabase());
context->setCurrentDatabase("test");
}
DatabasePtr mockSystemDatabase()
{
DatabasePtr database = std::make_shared<DatabaseMemory>("system", context);
auto tab = TableWithColumnNamesAndTypes(createDBAndTable("one", "system"), { {"dummy", std::make_shared<DataTypeUInt8>()} });
database->attachTable(context, tab.table.table,
std::make_shared<StorageMemory>(
StorageID(tab.table.database, tab.table.table),
ColumnsDescription{tab.columns}, ConstraintsDescription{}, String{}));
return database;
}
};
static void check(
static void checkOld(
const State & state,
size_t table_num,
const std::string & query,
@ -109,38 +133,106 @@ static void check(
SelectQueryInfo query_info;
SelectQueryOptions select_options;
query_info.syntax_analyzer_result
= TreeRewriter(state.context).analyzeSelect(ast, DB::TreeRewriterResult(state.getColumns()), select_options, state.getTables(table_num));
= TreeRewriter(state.context).analyzeSelect(ast, DB::TreeRewriterResult(state.getColumns(0)), select_options, state.getTables(table_num));
query_info.query = ast;
std::string transformed_query = transformQueryForExternalDatabase(
query_info, state.getColumns(), IdentifierQuotingStyle::DoubleQuotes, "test", "table", state.context);
query_info,
query_info.syntax_analyzer_result->requiredSourceColumns(),
state.getColumns(0), IdentifierQuotingStyle::DoubleQuotes, "test", "table", state.context);
EXPECT_EQ(transformed_query, expected) << query;
}
/// Required for transformQueryForExternalDatabase. In real life table expression is calculated via planner.
/// But in tests we can just find it in JOIN TREE.
static QueryTreeNodePtr findTableExpression(const QueryTreeNodePtr & node, const String & table_name)
{
if (node->getNodeType() == QueryTreeNodeType::TABLE)
{
if (node->as<TableNode>()->getStorageID().table_name == table_name)
return node;
}
if (node->getNodeType() == QueryTreeNodeType::JOIN)
{
if (auto res = findTableExpression(node->as<JoinNode>()->getLeftTableExpression(), table_name))
return res;
if (auto res = findTableExpression(node->as<JoinNode>()->getRightTableExpression(), table_name))
return res;
}
return nullptr;
}
/// `column_names` - Normally it's passed to query plan step. But in test we do it manually.
static void checkNewAnalyzer(
const State & state,
const Names & column_names,
const std::string & query,
const std::string & expected)
{
ParserSelectQuery parser;
ASTPtr ast = parseQuery(parser, query, 1000, 1000);
SelectQueryOptions select_options;
InterpreterSelectQueryAnalyzer interpreter(ast, state.context, select_options);
SelectQueryInfo query_info = interpreter.getSelectQueryInfo();
const auto * query_node = query_info.query_tree->as<QueryNode>();
if (!query_node)
throw Exception(ErrorCodes::LOGICAL_ERROR, "QueryNode expected");
query_info.table_expression = findTableExpression(query_node->getJoinTree(), "table");
std::string transformed_query = transformQueryForExternalDatabase(
query_info, column_names, state.getColumns(0), IdentifierQuotingStyle::DoubleQuotes, "test", "table", state.context);
EXPECT_EQ(transformed_query, expected) << query;
}
static void check(
const State & state,
size_t table_num,
const Names & column_names,
const std::string & query,
const std::string & expected,
const std::string & expected_new = "")
{
{
SCOPED_TRACE("Old analyzer");
checkOld(state, table_num, query, expected);
}
{
SCOPED_TRACE("New analyzer");
checkNewAnalyzer(state, column_names, query, expected_new.empty() ? expected : expected_new);
}
}
TEST(TransformQueryForExternalDatabase, InWithSingleElement)
{
const State & state = State::instance();
check(state, 1,
check(state, 1, {"column"},
"SELECT column FROM test.table WHERE 1 IN (1)",
R"(SELECT "column" FROM "test"."table" WHERE 1 = 1)");
check(state, 1,
R"(SELECT "column" FROM "test"."table" WHERE 1 = 1)",
R"(SELECT "column" FROM "test"."table")");
check(state, 1, {"column"},
"SELECT column FROM test.table WHERE column IN (1, 2)",
R"(SELECT "column" FROM "test"."table" WHERE "column" IN (1, 2))");
check(state, 1,
"SELECT column FROM test.table WHERE column NOT IN ('hello', 'world')",
R"(SELECT "column" FROM "test"."table" WHERE "column" NOT IN ('hello', 'world'))");
check(state, 1, {"field"},
"SELECT field FROM test.table WHERE field NOT IN ('hello', 'world')",
R"(SELECT "field" FROM "test"."table" WHERE "field" NOT IN ('hello', 'world'))");
}
TEST(TransformQueryForExternalDatabase, InWithMultipleColumns)
{
const State & state = State::instance();
check(state, 1,
check(state, 1, {"column"},
"SELECT column FROM test.table WHERE (1,1) IN ((1,1))",
R"(SELECT "column" FROM "test"."table" WHERE 1 = 1)");
check(state, 1,
R"(SELECT "column" FROM "test"."table" WHERE 1 = 1)",
R"(SELECT "column" FROM "test"."table")");
check(state, 1, {"field", "value"},
"SELECT field, value FROM test.table WHERE (field, value) IN (('foo', 'bar'))",
R"(SELECT "field", "value" FROM "test"."table" WHERE ("field", "value") IN (('foo', 'bar')))");
}
@ -149,17 +241,17 @@ TEST(TransformQueryForExternalDatabase, InWithTable)
{
const State & state = State::instance();
check(state, 1,
check(state, 1, {"column"},
"SELECT column FROM test.table WHERE 1 IN external_table",
R"(SELECT "column" FROM "test"."table")");
check(state, 1,
"SELECT column FROM test.table WHERE 1 IN (x)",
check(state, 1, {"column"},
"WITH x as (SELECT * FROM external_table) SELECT column FROM test.table WHERE 1 IN (x)",
R"(SELECT "column" FROM "test"."table")");
check(state, 1,
"SELECT column, field, value FROM test.table WHERE column IN (field, value)",
R"(SELECT "column", "field", "value" FROM "test"."table" WHERE "column" IN ("field", "value"))");
check(state, 1,
"SELECT column FROM test.table WHERE column NOT IN hello AND column = 123",
check(state, 1, {"column", "field", "value"},
"SELECT column, field, value FROM test.table WHERE column IN (1, 2)",
R"(SELECT "column", "field", "value" FROM "test"."table" WHERE "column" IN (1, 2))");
check(state, 1, {"column"},
"SELECT column FROM test.table WHERE column NOT IN external_table AND column = 123",
R"(SELECT "column" FROM "test"."table" WHERE "column" = 123)");
}
@ -167,32 +259,32 @@ TEST(TransformQueryForExternalDatabase, Like)
{
const State & state = State::instance();
check(state, 1,
"SELECT column FROM test.table WHERE column LIKE '%hello%'",
R"(SELECT "column" FROM "test"."table" WHERE "column" LIKE '%hello%')");
check(state, 1,
"SELECT column FROM test.table WHERE column NOT LIKE 'w%rld'",
R"(SELECT "column" FROM "test"."table" WHERE "column" NOT LIKE 'w%rld')");
check(state, 1, {"field"},
"SELECT field FROM test.table WHERE field LIKE '%hello%'",
R"(SELECT "field" FROM "test"."table" WHERE "field" LIKE '%hello%')");
check(state, 1, {"field"},
"SELECT field FROM test.table WHERE field NOT LIKE 'w%rld'",
R"(SELECT "field" FROM "test"."table" WHERE "field" NOT LIKE 'w%rld')");
}
TEST(TransformQueryForExternalDatabase, Substring)
{
const State & state = State::instance();
check(state, 1,
"SELECT column FROM test.table WHERE left(column, 10) = RIGHT(column, 10) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello'",
R"(SELECT "column" FROM "test"."table")");
check(state, 1, {"field"},
"SELECT field FROM test.table WHERE left(field, 10) = RIGHT(field, 10) AND SUBSTRING(field FROM 1 FOR 2) = 'Hello'",
R"(SELECT "field" FROM "test"."table")");
}
TEST(TransformQueryForExternalDatabase, MultipleAndSubqueries)
{
const State & state = State::instance();
check(state, 1,
"SELECT column FROM test.table WHERE 1 = 1 AND toString(column) = '42' AND column = 42 AND left(column, 10) = RIGHT(column, 10) AND column IN (1, 42) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello' AND column != 4",
check(state, 1, {"column"},
"SELECT column FROM test.table WHERE 1 = 1 AND toString(column) = '42' AND column = 42 AND left(toString(column), 10) = RIGHT(toString(column), 10) AND column IN (1, 42) AND SUBSTRING(toString(column) FROM 1 FOR 2) = 'Hello' AND column != 4",
R"(SELECT "column" FROM "test"."table" WHERE 1 AND ("column" = 42) AND ("column" IN (1, 42)) AND ("column" != 4))");
check(state, 1,
"SELECT column FROM test.table WHERE toString(column) = '42' AND left(column, 10) = RIGHT(column, 10) AND column = 42",
check(state, 1, {"column"},
"SELECT column FROM test.table WHERE toString(column) = '42' AND left(toString(column), 10) = RIGHT(toString(column), 10) AND column = 42",
R"(SELECT "column" FROM "test"."table" WHERE "column" = 42)");
}
@ -200,7 +292,7 @@ TEST(TransformQueryForExternalDatabase, Issue7245)
{
const State & state = State::instance();
check(state, 1,
check(state, 1, {"apply_id", "apply_type", "apply_status", "create_time"},
"SELECT apply_id FROM test.table WHERE apply_type = 2 AND create_time > addDays(toDateTime('2019-01-01 01:02:03'),-7) AND apply_status IN (3,4)",
R"(SELECT "apply_id", "apply_type", "apply_status", "create_time" FROM "test"."table" WHERE ("apply_type" = 2) AND ("create_time" > '2018-12-25 01:02:03') AND ("apply_status" IN (3, 4)))");
}
@ -209,8 +301,8 @@ TEST(TransformQueryForExternalDatabase, Aliases)
{
const State & state = State::instance();
check(state, 1,
"SELECT field AS value, field AS display WHERE field NOT IN ('') AND display LIKE '%test%'",
check(state, 1, {"field"},
"SELECT field AS value, field AS display FROM table WHERE field NOT IN ('') AND display LIKE '%test%'",
R"(SELECT "field" FROM "test"."table" WHERE ("field" NOT IN ('')) AND ("field" LIKE '%test%'))");
}
@ -218,10 +310,10 @@ TEST(TransformQueryForExternalDatabase, ForeignColumnInWhere)
{
const State & state = State::instance();
check(state, 2,
check(state, 2, {"column", "apply_id"},
"SELECT column FROM test.table "
"JOIN test.table2 AS table2 ON (test.table.apply_id = table2.num) "
"WHERE column > 2 AND (apply_id = 1 OR table2.num = 1) AND table2.attr != ''",
"WHERE column > 2 AND apply_id = 1 AND table2.num = 1 AND table2.attr != ''",
R"(SELECT "column", "apply_id" FROM "test"."table" WHERE ("column" > 2) AND ("apply_id" = 1))");
}
@ -229,7 +321,7 @@ TEST(TransformQueryForExternalDatabase, NoStrict)
{
const State & state = State::instance();
check(state, 1,
check(state, 1, {"field"},
"SELECT field FROM table WHERE field IN (SELECT attr FROM table2)",
R"(SELECT "field" FROM "test"."table")");
}
@ -239,37 +331,37 @@ TEST(TransformQueryForExternalDatabase, Strict)
const State & state = State::instance();
state.context->setSetting("external_table_strict_query", true);
check(state, 1,
check(state, 1, {"field"},
"SELECT field FROM table WHERE field = '1'",
R"(SELECT "field" FROM "test"."table" WHERE "field" = '1')");
check(state, 1,
check(state, 1, {"field"},
"SELECT field FROM table WHERE field IN ('1', '2')",
R"(SELECT "field" FROM "test"."table" WHERE "field" IN ('1', '2'))");
check(state, 1,
check(state, 1, {"field"},
"SELECT field FROM table WHERE field LIKE '%test%'",
R"(SELECT "field" FROM "test"."table" WHERE "field" LIKE '%test%')");
/// removeUnknownSubexpressionsFromWhere() takes place
EXPECT_THROW(check(state, 1, "SELECT field FROM table WHERE field IN (SELECT attr FROM table2)", ""), Exception);
EXPECT_THROW(check(state, 1, {"field"}, "SELECT field FROM table WHERE field IN (SELECT attr FROM table2)", ""), Exception);
/// !isCompatible() takes place
EXPECT_THROW(check(state, 1, "SELECT column FROM test.table WHERE left(column, 10) = RIGHT(column, 10) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello'", ""), Exception);
EXPECT_THROW(check(state, 1, {"column"}, "SELECT column FROM test.table WHERE left(column, 10) = RIGHT(column, 10) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello'", ""), Exception);
}
TEST(TransformQueryForExternalDatabase, Null)
{
const State & state = State::instance();
check(state, 1,
check(state, 1, {"field"},
"SELECT field FROM table WHERE field IS NULL",
R"(SELECT "field" FROM "test"."table" WHERE "field" IS NULL)");
check(state, 1,
check(state, 1, {"field"},
"SELECT field FROM table WHERE field IS NOT NULL",
R"(SELECT "field" FROM "test"."table" WHERE "field" IS NOT NULL)");
check(state, 1,
check(state, 1, {"field"},
"SELECT field FROM table WHERE isNull(field)",
R"(SELECT "field" FROM "test"."table" WHERE "field" IS NULL)");
check(state, 1,
check(state, 1, {"field"},
"SELECT field FROM table WHERE isNotNull(field)",
R"(SELECT "field" FROM "test"."table" WHERE "field" IS NOT NULL)");
}
@ -278,7 +370,28 @@ TEST(TransformQueryForExternalDatabase, ToDate)
{
const State & state = State::instance();
check(state, 1,
check(state, 1, {"a", "b", "foo"},
"SELECT foo FROM table WHERE a=10 AND b=toDate('2019-10-05')",
R"(SELECT "a", "b", "foo" FROM "test"."table" WHERE ("a" = 10) AND ("b" = '2019-10-05'))");
}
TEST(TransformQueryForExternalDatabase, Analyzer)
{
const State & state = State::instance();
check(state, 1, {"field"},
"SELECT count() FROM table WHERE field LIKE '%name_%'",
R"(SELECT "field" FROM "test"."table" WHERE "field" LIKE '%name_%')");
check(state, 1, {"column"},
"SELECT 1 FROM table",
R"(SELECT "column" FROM "test"."table")");
check(state, 1, {"column"},
"SELECT sleepEachRow(1) FROM table",
R"(SELECT "column" FROM "test"."table")");
check(state, 1, {"column", "apply_id", "apply_type", "apply_status", "create_time", "field", "value", "a", "b", "foo"},
"SELECT * FROM table WHERE (column) IN (1)",
R"(SELECT "column", "apply_id", "apply_type", "apply_status", "create_time", "field", "value", "a", "b", "foo" FROM "test"."table" WHERE "column" IN (1))");
}

97
src/Storages/transformQueryForExternalDatabase.cpp
View File

@ -14,6 +14,8 @@
#include <Storages/transformQueryForExternalDatabase.h>
#include <Storages/MergeTree/KeyCondition.h>
#include <Storages/transformQueryForExternalDatabaseAnalyzer.h>
namespace DB
{
@ -108,9 +110,9 @@ void dropAliases(ASTPtr & node)
}
bool isCompatible(IAST & node)
bool isCompatible(ASTPtr & node)
{
if (auto * function = node.as<ASTFunction>())
if (auto * function = node->as<ASTFunction>())
{
if (function->parameters) /// Parametric aggregate functions
return false;
@ -154,20 +156,30 @@ bool isCompatible(IAST & node)
&& (function->arguments->children.size() != 2 || function->arguments->children[1]->as<ASTTableIdentifier>()))
return false;
for (const auto & expr : function->arguments->children)
if (!isCompatible(*expr))
for (auto & expr : function->arguments->children)
if (!isCompatible(expr))
return false;
return true;
}
if (const auto * literal = node.as<ASTLiteral>())
if (const auto * literal = node->as<ASTLiteral>())
{
if (literal->value.getType() == Field::Types::Tuple)
{
/// Represent a tuple with zero or one elements as (x) instead of tuple(x).
auto tuple_value = literal->value.safeGet<Tuple>();
if (tuple_value.size() == 1)
{
node = makeASTFunction("", std::make_shared<ASTLiteral>(tuple_value[0]));
return true;
}
}
/// Foreign databases often have no support for Array. But Tuple literals are passed to support IN clause.
return literal->value.getType() != Field::Types::Array;
}
return node.as<ASTIdentifier>();
return node->as<ASTIdentifier>();
}
bool removeUnknownSubexpressions(ASTPtr & node, const NameSet & known_names);
@ -241,23 +253,15 @@ bool removeUnknownSubexpressionsFromWhere(ASTPtr & node, const NamesAndTypesList
return removeUnknownSubexpressions(node, known_names);
}
}
String transformQueryForExternalDatabase(
const SelectQueryInfo & query_info,
String transformQueryForExternalDatabaseImpl(
ASTPtr clone_query,
Names used_columns,
const NamesAndTypesList & available_columns,
IdentifierQuotingStyle identifier_quoting_style,
const String & database,
const String & table,
ContextPtr context)
{
auto clone_query = query_info.query->clone();
/// TODO: Analyzer syntax analyzer result
if (!query_info.syntax_analyzer_result)
throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "transform query for external database is unsupported");
const Names used_columns = query_info.syntax_analyzer_result->requiredSourceColumns();
bool strict = context->getSettingsRef().external_table_strict_query;
auto select = std::make_shared<ASTSelectQuery>();
@ -278,11 +282,12 @@ String transformQueryForExternalDatabase(
ASTPtr original_where = clone_query->as<ASTSelectQuery &>().where();
bool where_has_known_columns = removeUnknownSubexpressionsFromWhere(original_where, available_columns);
if (original_where && where_has_known_columns)
{
replaceConstantExpressions(original_where, context, available_columns);
if (isCompatible(*original_where))
if (isCompatible(original_where))
{
select->setExpression(ASTSelectQuery::Expression::WHERE, std::move(original_where));
}
@ -290,14 +295,14 @@ String transformQueryForExternalDatabase(
{
throw Exception(ErrorCodes::INCORRECT_QUERY, "Query contains non-compatible expressions (and external_table_strict_query=true)");
}
else if (const auto * function = original_where->as<ASTFunction>())
else if (auto * function = original_where->as<ASTFunction>())
{
if (function->name == "and")
{
auto new_function_and = makeASTFunction("and");
for (const auto & elem : function->arguments->children)
for (auto & elem : function->arguments->children)
{
if (isCompatible(*elem))
if (isCompatible(elem))
new_function_and->arguments->children.push_back(elem);
}
if (new_function_and->arguments->children.size() == 1)
@ -309,7 +314,8 @@ String transformQueryForExternalDatabase(
}
else if (strict && original_where)
{
throw Exception(ErrorCodes::INCORRECT_QUERY, "Query contains non-compatible expressions (and external_table_strict_query=true)");
throw Exception(ErrorCodes::INCORRECT_QUERY, "Query contains non-compatible expressions '{}' (and external_table_strict_query=true)",
original_where->formatForErrorMessage());
}
auto * literal_expr = typeid_cast<ASTLiteral *>(original_where.get());
@ -338,3 +344,50 @@ String transformQueryForExternalDatabase(
}
}
String transformQueryForExternalDatabase(
const SelectQueryInfo & query_info,
const Names & column_names,
const NamesAndTypesList & available_columns,
IdentifierQuotingStyle identifier_quoting_style,
const String & database,
const String & table,
ContextPtr context)
{
if (!query_info.syntax_analyzer_result)
{
if (!query_info.query_tree)
throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Query is not analyzed: no query tree");
if (!query_info.planner_context)
throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Query is not analyzed: no planner context");
if (!query_info.table_expression)
throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Query is not analyzed: no table expression");
if (column_names.empty())
throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "No column names for query '{}' to external table '{}.{}'",
database, table, query_info.query_tree->formatASTForErrorMessage());
auto clone_query = getASTForExternalDatabaseFromQueryTree(query_info.query_tree);
return transformQueryForExternalDatabaseImpl(
clone_query,
column_names,
available_columns,
identifier_quoting_style,
database,
table,
context);
}
auto clone_query = query_info.query->clone();
return transformQueryForExternalDatabaseImpl(
clone_query,
query_info.syntax_analyzer_result->requiredSourceColumns(),
available_columns,
identifier_quoting_style,
database,
table,
context);
}
}

1
src/Storages/transformQueryForExternalDatabase.h
View File

@ -28,6 +28,7 @@ class IAST;
*/
String transformQueryForExternalDatabase(
const SelectQueryInfo & query_info,
const Names & column_names,
const NamesAndTypesList & available_columns,
IdentifierQuotingStyle identifier_quoting_style,
const String & database,

81
src/Storages/transformQueryForExternalDatabaseAnalyzer.cpp Normal file
View File

@ -0,0 +1,81 @@
#include <Storages/transformQueryForExternalDatabaseAnalyzer.h>
#include <Storages/transformQueryForExternalDatabase.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Interpreters/convertFieldToType.h>
#include <Columns/ColumnConst.h>
#include <Analyzer/QueryNode.h>
#include <Analyzer/ColumnNode.h>
#include <Analyzer/ConstantNode.h>
#include <Analyzer/ConstantValue.h>
#include <Analyzer/TableNode.h>
#include <Analyzer/JoinNode.h>
#include <DataTypes/DataTypesNumber.h>
namespace DB
{
namespace ErrorCodes
{
extern const int UNSUPPORTED_METHOD;
}
namespace
{
class PrepareForExternalDatabaseVisitor : public InDepthQueryTreeVisitor<PrepareForExternalDatabaseVisitor>
{
public:
using Base = InDepthQueryTreeVisitor<PrepareForExternalDatabaseVisitor>;
using Base::Base;
void visitImpl(QueryTreeNodePtr & node)
{
auto * constant_node = node->as<ConstantNode>();
if (constant_node)
{
auto result_type = constant_node->getResultType();
if (isDate(result_type) || isDateTime(result_type) || isDateTime64(result_type))
{
/// Use string representation of constant date and time values
/// The code is ugly - how to convert artbitrary Field to proper string representation?
/// (maybe we can just consider numbers as unix timestamps?)
auto result_column = result_type->createColumnConst(1, constant_node->getValue());
const IColumn & inner_column = assert_cast<const ColumnConst &>(*result_column).getDataColumn();
WriteBufferFromOwnString out;
result_type->getDefaultSerialization()->serializeText(inner_column, 0, out, FormatSettings());
node = std::make_shared<ConstantNode>(std::make_shared<ConstantValue>(out.str(), result_type));
}
}
}
};
}
ASTPtr getASTForExternalDatabaseFromQueryTree(const QueryTreeNodePtr & query_tree)
{
auto new_tree = query_tree->clone();
PrepareForExternalDatabaseVisitor visitor;
visitor.visit(new_tree);
const auto * query_node = new_tree->as<QueryNode>();
const auto & query_node_ast = query_node->toAST({ .add_cast_for_constants = false, .fully_qualified_identifiers = false });
const auto * union_ast = query_node_ast->as<ASTSelectWithUnionQuery>();
if (!union_ast)
throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "QueryNode AST is not a ASTSelectWithUnionQuery");
if (union_ast->list_of_selects->children.size() != 1)
throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "QueryNode AST is not a single ASTSelectQuery, got {}", union_ast->list_of_selects->children.size());
return union_ast->list_of_selects->children.at(0);
}
}

11
src/Storages/transformQueryForExternalDatabaseAnalyzer.h Normal file
View File

@ -0,0 +1,11 @@
#pragma once
#include <Analyzer/IQueryTreeNode.h>
namespace DB
{
ASTPtr getASTForExternalDatabaseFromQueryTree(const QueryTreeNodePtr & query_tree);
}

40
tests/queries/0_stateless/02479_mysql_connect_to_self.reference
View File

@ -1 +1,39 @@
0
---
1 one -1 een
2 two -2 twee
3 three -3 drie
4 four -4 vier
5 five -5 vijf
---
5
---
1
1
1
1
1
---
1
2
3
4
5
---
-5 five
-4 four
-1 one
-3 three
-2 two
---
-3 three
-1 one
-2 two
-4 four
-5 five
---
-1
-3
-4
-5
---
4

34
tests/queries/0_stateless/02479_mysql_connect_to_self.sql
View File

@ -1,4 +1,32 @@
-- Tags: no-fasttest
SELECT *
FROM mysql('127.0.0.1:9004', system, one, 'default', '')
SETTINGS send_logs_level = 'fatal'; -- failed connection tries are ok, if it succeeded after retry.
SET send_logs_level = 'fatal'; -- failed connection tries are ok, if it succeeded after retry.
CREATE TABLE foo (key UInt32, a String, b Int64, c String) ENGINE = TinyLog;
INSERT INTO foo VALUES (1, 'one', -1, 'een'), (2, 'two', -2, 'twee'), (3, 'three', -3, 'drie'), (4, 'four', -4, 'vier'), (5, 'five', -5, 'vijf');
SET allow_experimental_analyzer = 1;
SELECT '---';
SELECT * FROM mysql('127.0.0.1:9004', currentDatabase(), foo, 'default', '') ORDER BY key;
SELECT '---';
SELECT count() FROM mysql('127.0.0.1:9004', currentDatabase(), foo, 'default', '');
SELECT '---';
SELECT 1 FROM mysql('127.0.0.1:9004', currentDatabase(), foo, 'default', '');
SELECT '---';
SELECT key FROM mysql('127.0.0.1:9004', currentDatabase(), foo, 'default', '') ORDER BY key;
SELECT '---';
SELECT b, a FROM mysql('127.0.0.1:9004', currentDatabase(), foo, 'default', '') ORDER BY a;
SELECT '---';
SELECT b, a FROM mysql('127.0.0.1:9004', currentDatabase(), foo, 'default', '') ORDER BY c;
SELECT '---';
SELECT b FROM mysql('127.0.0.1:9004', currentDatabase(), foo, 'default', '') WHERE c != 'twee' ORDER BY b;
SELECT '---';
SELECT count() FROM mysql('127.0.0.1:9004', currentDatabase(), foo, 'default', '') WHERE c != 'twee';