From 514d82ba643b8e0baf6f2f7ee67143978ed06a5b Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Mon, 1 Jul 2024 17:29:36 +0200 Subject: [PATCH 01/10] Fix --- .../evaluateConstantExpression.cpp | 114 +++++++++++------- 1 file changed, 73 insertions(+), 41 deletions(-) diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index 4e1a2bcf5ee..60c62665592 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -21,7 +22,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -69,53 +72,82 @@ std::optional evaluateConstantExpressionImpl(c ReplaceQueryParameterVisitor param_visitor(context->getQueryParameters()); param_visitor.visit(ast); - /// Notice: function name normalization is disabled when it's a secondary query, because queries are either - /// already normalized on initiator node, or not normalized and should remain unnormalized for - /// compatibility. - if (context->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY && context->getSettingsRef().normalize_function_names) - FunctionNameNormalizer::visit(ast.get()); - - auto syntax_result = TreeRewriter(context, no_throw).analyze(ast, source_columns); - if (!syntax_result) - return {}; - - /// AST potentially could be transformed to literal during TreeRewriter analyze. - /// For example if we have SQL user defined function that return literal AS subquery. - if (ASTLiteral * literal = ast->as()) - return getFieldAndDataTypeFromLiteral(literal); - - auto actions = ExpressionAnalyzer(ast, syntax_result, context).getConstActionsDAG(); - - ColumnPtr result_column; - DataTypePtr result_type; - String result_name = ast->getColumnName(); - for (const auto & action_node : actions->getOutputs()) + if (context->getSettingsRef().allow_experimental_analyzer) { - if ((action_node->result_name == result_name) && action_node->column) - { - result_column = action_node->column; - result_type = action_node->result_type; - break; - } + ASTPtr new_ast = std::make_shared(); + auto *ast_as_select = new_ast->as(); + + auto expr_list = std::make_shared(); + expr_list->children.push_back(ast->clone()); + + ast_as_select->setExpression(ASTSelectQuery::Expression::SELECT, expr_list); + + InterpreterSelectQueryAnalyzer interpreter(ast_as_select->clone(), context, SelectQueryOptions()); + auto io = interpreter.execute(); + + PullingPipelineExecutor executor(io.pipeline); + Block block; + + executor.pull(block); + + ColumnPtr result_column; + DataTypePtr result_type; + + auto & column = block.safeGetByPosition(0); + auto type = column.type; + + return std::make_pair((*column.column)[0], type); } + else + { + /// Notice: function name normalization is disabled when it's a secondary query, because queries are either + /// already normalized on initiator node, or not normalized and should remain unnormalized for + /// compatibility. + if (context->getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY && context->getSettingsRef().normalize_function_names) + FunctionNameNormalizer::visit(ast.get()); - if (!result_column) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Element of set in IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument " - "is not a constant expression (result column not found): {}", result_name); + auto syntax_result = TreeRewriter(context, no_throw).analyze(ast, source_columns); + if (!syntax_result) + return {}; - if (result_column->empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Empty result column after evaluation " - "of constant expression for IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument"); + /// AST potentially could be transformed to literal during TreeRewriter analyze. + /// For example if we have SQL user defined function that return literal AS subquery. + if (ASTLiteral * literal = ast->as()) + return getFieldAndDataTypeFromLiteral(literal); - /// Expressions like rand() or now() are not constant - if (!isColumnConst(*result_column)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Element of set in IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument " - "is not a constant expression (result column is not const): {}", result_name); + auto actions = ExpressionAnalyzer(ast, syntax_result, context).getConstActionsDAG(); - return std::make_pair((*result_column)[0], result_type); + ColumnPtr result_column; + DataTypePtr result_type; + String result_name = ast->getColumnName(); + for (const auto & action_node : actions->getOutputs()) + { + if ((action_node->result_name == result_name) && action_node->column) + { + result_column = action_node->column; + result_type = action_node->result_type; + break; + } + } + + if (!result_column) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Element of set in IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument " + "is not a constant expression (result column not found): {}", result_name); + + if (result_column->empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Empty result column after evaluation " + "of constant expression for IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument"); + + /// Expressions like rand() or now() are not constant + if (!isColumnConst(*result_column)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Element of set in IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument " + "is not a constant expression (result column is not const): {}", result_name); + + return std::make_pair((*result_column)[0], result_type); + } } std::optional tryEvaluateConstantExpression(const ASTPtr & node, const ContextPtr & context) From 31cf3a45b52dd188daca14caa3066ae36015c1be Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Mon, 1 Jul 2024 17:36:36 +0200 Subject: [PATCH 02/10] Lint --- src/Interpreters/evaluateConstantExpression.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index 60c62665592..cdb69b13800 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -75,24 +75,20 @@ std::optional evaluateConstantExpressionImpl(c if (context->getSettingsRef().allow_experimental_analyzer) { ASTPtr new_ast = std::make_shared(); - auto *ast_as_select = new_ast->as(); - auto expr_list = std::make_shared(); + expr_list->children.push_back(ast->clone()); + auto *ast_as_select = new_ast->as(); ast_as_select->setExpression(ASTSelectQuery::Expression::SELECT, expr_list); - InterpreterSelectQueryAnalyzer interpreter(ast_as_select->clone(), context, SelectQueryOptions()); + InterpreterSelectQueryAnalyzer interpreter(new_ast, context, SelectQueryOptions()); auto io = interpreter.execute(); - PullingPipelineExecutor executor(io.pipeline); Block block; - + PullingPipelineExecutor executor(io.pipeline); executor.pull(block); - ColumnPtr result_column; - DataTypePtr result_type; - auto & column = block.safeGetByPosition(0); auto type = column.type; From c70a92ace09361eb50e2fa3a58bfbb32a9ca79ba Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Tue, 2 Jul 2024 18:46:44 +0200 Subject: [PATCH 03/10] temp-commit --- .../evaluateConstantExpression.cpp | 67 ++++++++++++++++--- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index cdb69b13800..86366444e02 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -3,6 +3,10 @@ #include #include #include +#include "Analyzer/Passes/QueryAnalysisPass.h" +#include "Analyzer/QueryTreeBuilder.h" +#include "Interpreters/SelectQueryOptions.h" +#include "Planner/PlannerActionsVisitor.h" #include #include #include @@ -82,17 +86,64 @@ std::optional evaluateConstantExpressionImpl(c auto *ast_as_select = new_ast->as(); ast_as_select->setExpression(ASTSelectQuery::Expression::SELECT, expr_list); - InterpreterSelectQueryAnalyzer interpreter(new_ast, context, SelectQueryOptions()); - auto io = interpreter.execute(); + // InterpreterSelectQueryAnalyzer interpreter(new_ast, context, SelectQueryOptions()); + // auto & io = interpreter.getPlanner(); - Block block; - PullingPipelineExecutor executor(io.pipeline); - executor.pull(block); + // Block block; + // PullingPipelineExecutor executor(io.pipeline); + // executor.pull(block); - auto & column = block.safeGetByPosition(0); - auto type = column.type; - return std::make_pair((*column.column)[0], type); + auto test = buildQueryTree(ast, context); + SelectQueryOptions options = {}; + + options.only_analyze = true; + + Planner p = Planner(test, options); + + QueryAnalysisPass query_analysis_pass; + query_analysis_pass.run(test, context); + + auto actions_dag = std::make_shared(); + + PlannerActionsVisitor actions_visitor(p.getPlannerContext(), false); + auto expr_nodes = actions_visitor.visit(*actions_dag, test); + + // auto & column = block.safeGetByPosition(0); + // auto type = column.type; + // + // return std::make_pair((*column.column)[0], type); + + ColumnPtr result_column; + DataTypePtr result_type; + String result_name = ast->getColumnName(); + for (const auto & action_node : actions_dag->getOutputs()) + { + if ((action_node->result_name == result_name) && action_node->column) + { + result_column = action_node->column; + result_type = action_node->result_type; + break; + } + } + + if (!result_column) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Element of set in IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument " + "is not a constant expression (result column not found): {}", result_name); + + if (result_column->empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Empty result column after evaluation " + "of constant expression for IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument"); + + /// Expressions like rand() or now() are not constant + if (!isColumnConst(*result_column)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Element of set in IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument " + "is not a constant expression (result column is not const): {}", result_name); + + return std::make_pair((*result_column)[0], result_type); } else { From 1e42be052e0796aa78c27d57a6b2e11ba14ca72d Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Tue, 2 Jul 2024 19:46:02 +0200 Subject: [PATCH 04/10] temp-commit --- .../evaluateConstantExpression.cpp | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index 86366444e02..ad55acb6704 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -3,10 +3,17 @@ #include #include #include +#include "Analyzer/IQueryTreeNode.h" #include "Analyzer/Passes/QueryAnalysisPass.h" +#include "Analyzer/QueryNode.h" #include "Analyzer/QueryTreeBuilder.h" +#include "Analyzer/TableNode.h" #include "Interpreters/SelectQueryOptions.h" #include "Planner/PlannerActionsVisitor.h" +#include "Planner/PlannerContext.h" +#include "Planner/findQueryForParallelReplicas.h" +#include "Storages/ColumnsDescription.h" +#include "Storages/StorageDummy.h" #include #include #include @@ -97,16 +104,27 @@ std::optional evaluateConstantExpressionImpl(c auto test = buildQueryTree(ast, context); SelectQueryOptions options = {}; - options.only_analyze = true; + // options.only_analyze = true; + // + // Planner p = Planner(test, options); - Planner p = Planner(test, options); + ColumnsDescription fake_column_descriptions(source_columns); + auto storage = std::make_shared(StorageID{"dummy", "dummy"}, fake_column_descriptions); + QueryTreeNodePtr fake_table_expression = std::make_shared(storage, context); - QueryAnalysisPass query_analysis_pass; + const auto *a = findQueryForParallelReplicas(test, options); + const auto *b = findTableForParallelReplicas(test, options); + FiltersForTableExpressionMap c = {}; + + GlobalPlannerContextPtr global_planner_context = std::make_shared(a, b, c); + auto planner_context = std::make_shared(context->getGlobalContext(), global_planner_context, options); + + QueryAnalysisPass query_analysis_pass(fake_table_expression); query_analysis_pass.run(test, context); auto actions_dag = std::make_shared(); - PlannerActionsVisitor actions_visitor(p.getPlannerContext(), false); + PlannerActionsVisitor actions_visitor(planner_context, false); auto expr_nodes = actions_visitor.visit(*actions_dag, test); // auto & column = block.safeGetByPosition(0); From 9dc15701d5d6de022eedd192d12b80eed013705c Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Wed, 3 Jul 2024 18:07:52 +0200 Subject: [PATCH 05/10] temp-commit --- src/Interpreters/evaluateConstantExpression.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index ad55acb6704..44071d6e0e1 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -112,11 +112,11 @@ std::optional evaluateConstantExpressionImpl(c auto storage = std::make_shared(StorageID{"dummy", "dummy"}, fake_column_descriptions); QueryTreeNodePtr fake_table_expression = std::make_shared(storage, context); - const auto *a = findQueryForParallelReplicas(test, options); - const auto *b = findTableForParallelReplicas(test, options); + // const auto *a = findQueryForParallelReplicas(test, options); + // const auto *b = findTableForParallelReplicas(test, options); FiltersForTableExpressionMap c = {}; - GlobalPlannerContextPtr global_planner_context = std::make_shared(a, b, c); + GlobalPlannerContextPtr global_planner_context = std::make_shared(nullptr, nullptr, c); auto planner_context = std::make_shared(context->getGlobalContext(), global_planner_context, options); QueryAnalysisPass query_analysis_pass(fake_table_expression); From 7f59a07fb8994e09aa637f249b6cb2327d9526a6 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Wed, 3 Jul 2024 19:29:46 +0200 Subject: [PATCH 06/10] temp-commit --- .../evaluateConstantExpression.cpp | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index 44071d6e0e1..232244e6bad 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -3,6 +3,7 @@ #include #include #include +#include "Analyzer/ConstantNode.h" #include "Analyzer/IQueryTreeNode.h" #include "Analyzer/Passes/QueryAnalysisPass.h" #include "Analyzer/QueryNode.h" @@ -11,7 +12,6 @@ #include "Interpreters/SelectQueryOptions.h" #include "Planner/PlannerActionsVisitor.h" #include "Planner/PlannerContext.h" -#include "Planner/findQueryForParallelReplicas.h" #include "Storages/ColumnsDescription.h" #include "Storages/StorageDummy.h" #include @@ -122,6 +122,10 @@ std::optional evaluateConstantExpressionImpl(c QueryAnalysisPass query_analysis_pass(fake_table_expression); query_analysis_pass.run(test, context); + auto *test2 = typeid_cast(test.get()); + if (!test2) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected ConstantNode"); + auto actions_dag = std::make_shared(); PlannerActionsVisitor actions_visitor(planner_context, false); @@ -134,34 +138,34 @@ std::optional evaluateConstantExpressionImpl(c ColumnPtr result_column; DataTypePtr result_type; - String result_name = ast->getColumnName(); - for (const auto & action_node : actions_dag->getOutputs()) - { - if ((action_node->result_name == result_name) && action_node->column) - { - result_column = action_node->column; - result_type = action_node->result_type; - break; - } - } + // String result_name = ast->getColumnName(); + // for (const auto & action_node : actions_dag->getOutputs()) + // { + // if ((action_node->result_name == result_name) && action_node->column) + // { + // result_column = action_node->column; + // result_type = action_node->result_type; + // break; + // } + // } - if (!result_column) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Element of set in IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument " - "is not a constant expression (result column not found): {}", result_name); + // if (!result_column) + // throw Exception(ErrorCodes::BAD_ARGUMENTS, + // "Element of set in IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument " + // "is not a constant expression (result column not found): {}", "fixme"); + // + // if (result_column->empty()) + // throw Exception(ErrorCodes::LOGICAL_ERROR, + // "Empty result column after evaluation " + // "of constant expression for IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument"); + // + // /// Expressions like rand() or now() are not constant + // if (!isColumnConst(*result_column)) + // throw Exception(ErrorCodes::BAD_ARGUMENTS, + // "Element of set in IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument " + // "is not a constant expression (result column is not const): {}", "fixme"); - if (result_column->empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Empty result column after evaluation " - "of constant expression for IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument"); - - /// Expressions like rand() or now() are not constant - if (!isColumnConst(*result_column)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Element of set in IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument " - "is not a constant expression (result column is not const): {}", result_name); - - return std::make_pair((*result_column)[0], result_type); + return std::make_pair(test2->getValue(), test2->getResultType()); } else { From 6eed426b14dce39f6a592c33daa13f3f3054515b Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 6 Sep 2024 15:57:42 +0200 Subject: [PATCH 07/10] Rewrite implementation --- .../IInterpreterUnionOrSelectQuery.cpp | 2 +- .../evaluateConstantExpression.cpp | 135 ++++++------------ src/Storages/StorageDistributed.cpp | 2 +- 3 files changed, 44 insertions(+), 95 deletions(-) diff --git a/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp b/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp index 0034b50f02c..b7931491704 100644 --- a/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp +++ b/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp @@ -28,7 +28,7 @@ IInterpreterUnionOrSelectQuery::IInterpreterUnionOrSelectQuery(const ASTPtr & qu /// it's possible that new analyzer will be enabled in ::getQueryProcessingStage method /// of the underlying storage when all other parts of infrastructure are not ready for it /// (built with old analyzer). - context->setSetting("allow_experimental_analyzer", false); + // context->setSetting("allow_experimental_analyzer", false); if (options.shard_num) context->addSpecialScalar( diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index 5cc035375b9..efe0da108ad 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -2,6 +2,7 @@ #include #include +#include "Common/logger_useful.h" #include #include "Analyzer/ConstantNode.h" #include "Analyzer/IQueryTreeNode.h" @@ -12,6 +13,7 @@ #include "Interpreters/SelectQueryOptions.h" #include "Planner/PlannerActionsVisitor.h" #include "Planner/PlannerContext.h" +#include "Planner/Utils.h" #include "Storages/ColumnsDescription.h" #include "Storages/StorageDummy.h" #include @@ -40,6 +42,7 @@ #include #include +#include #include @@ -85,89 +88,38 @@ std::optional evaluateConstantExpressionImpl(c ReplaceQueryParameterVisitor param_visitor(context->getQueryParameters()); param_visitor.visit(ast); + String result_name = ast->getColumnName(); + + ColumnPtr result_column; + DataTypePtr result_type; if (context->getSettingsRef().allow_experimental_analyzer) { - ASTPtr new_ast = std::make_shared(); - auto expr_list = std::make_shared(); - - expr_list->children.push_back(ast->clone()); - - auto *ast_as_select = new_ast->as(); - ast_as_select->setExpression(ASTSelectQuery::Expression::SELECT, expr_list); - - // InterpreterSelectQueryAnalyzer interpreter(new_ast, context, SelectQueryOptions()); - // auto & io = interpreter.getPlanner(); - - // Block block; - // PullingPipelineExecutor executor(io.pipeline); - // executor.pull(block); - - - auto test = buildQueryTree(ast, context); - SelectQueryOptions options = {}; - - // options.only_analyze = true; - // - // Planner p = Planner(test, options); + auto execution_context = Context::createCopy(context); + auto expression = buildQueryTree(ast, execution_context); ColumnsDescription fake_column_descriptions(source_columns); auto storage = std::make_shared(StorageID{"dummy", "dummy"}, fake_column_descriptions); - QueryTreeNodePtr fake_table_expression = std::make_shared(storage, context); - - // const auto *a = findQueryForParallelReplicas(test, options); - // const auto *b = findTableForParallelReplicas(test, options); - FiltersForTableExpressionMap c = {}; - - GlobalPlannerContextPtr global_planner_context = std::make_shared(nullptr, nullptr, c); - auto planner_context = std::make_shared(context->getGlobalContext(), global_planner_context, options); + QueryTreeNodePtr fake_table_expression = std::make_shared(storage, execution_context); QueryAnalysisPass query_analysis_pass(fake_table_expression); - query_analysis_pass.run(test, context); + query_analysis_pass.run(expression, execution_context); - auto *test2 = typeid_cast(test.get()); - if (!test2) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected ConstantNode"); + GlobalPlannerContextPtr global_planner_context = std::make_shared(nullptr, nullptr, FiltersForTableExpressionMap{}); + auto planner_context = std::make_shared(execution_context, global_planner_context, SelectQueryOptions{}); - auto actions_dag = std::make_shared(); + auto actions = buildActionsDAGFromExpressionNode(expression, {}, planner_context); - PlannerActionsVisitor actions_visitor(planner_context, false); - auto expr_nodes = actions_visitor.visit(*actions_dag, test); + if (actions.getOutputs().size() != 1) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "ActionsDAG contains more than 1 output for expression: {}", ast->formatForLogging()); + } - // auto & column = block.safeGetByPosition(0); - // auto type = column.type; - // - // return std::make_pair((*column.column)[0], type); - - ColumnPtr result_column; - DataTypePtr result_type; - // String result_name = ast->getColumnName(); - // for (const auto & action_node : actions_dag->getOutputs()) - // { - // if ((action_node->result_name == result_name) && action_node->column) - // { - // result_column = action_node->column; - // result_type = action_node->result_type; - // break; - // } - // } - - // if (!result_column) - // throw Exception(ErrorCodes::BAD_ARGUMENTS, - // "Element of set in IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument " - // "is not a constant expression (result column not found): {}", "fixme"); - // - // if (result_column->empty()) - // throw Exception(ErrorCodes::LOGICAL_ERROR, - // "Empty result column after evaluation " - // "of constant expression for IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument"); - // - // /// Expressions like rand() or now() are not constant - // if (!isColumnConst(*result_column)) - // throw Exception(ErrorCodes::BAD_ARGUMENTS, - // "Element of set in IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument " - // "is not a constant expression (result column is not const): {}", "fixme"); - - return std::make_pair(test2->getValue(), test2->getResultType()); + const auto & output = actions.getOutputs()[0]; + if (output->column) + { + result_column = output->column; + result_type = output->result_type; + } } else { @@ -188,9 +140,6 @@ std::optional evaluateConstantExpressionImpl(c auto actions = ExpressionAnalyzer(ast, syntax_result, context).getConstActionsDAG(); - ColumnPtr result_column; - DataTypePtr result_type; - String result_name = ast->getColumnName(); for (const auto & action_node : actions.getOutputs()) { if ((action_node->result_name == result_name) && action_node->column) @@ -200,25 +149,25 @@ std::optional evaluateConstantExpressionImpl(c break; } } - - if (!result_column) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Element of set in IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument " - "is not a constant expression (result column not found): {}", result_name); - - if (result_column->empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, - "Empty result column after evaluation " - "of constant expression for IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument"); - - /// Expressions like rand() or now() are not constant - if (!isColumnConst(*result_column)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Element of set in IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument " - "is not a constant expression (result column is not const): {}", result_name); - - return std::make_pair((*result_column)[0], result_type); } + + if (!result_column) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Element of set in IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument " + "is not a constant expression (result column not found): {}", result_name); + + if (result_column->empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Empty result column after evaluation " + "of constant expression for IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument"); + + /// Expressions like rand() or now() are not constant + if (!isColumnConst(*result_column)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Element of set in IN, VALUES, or LIMIT, or aggregate function parameter, or a table function argument " + "is not a constant expression (result column is not const): {}", result_name); + + return std::make_pair((*result_column)[0], result_type); } std::optional tryEvaluateConstantExpression(const ASTPtr & node, const ContextPtr & context) diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 0b80858800b..f54a0015999 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -462,7 +462,7 @@ QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage( } std::optional optimized_stage; - if (query_info.query_tree) + if (settings.allow_experimental_analyzer) optimized_stage = getOptimizedQueryProcessingStageAnalyzer(query_info, settings); else optimized_stage = getOptimizedQueryProcessingStage(query_info, settings); From 17e4616924bf8c5c682ceb8fe554ec2223ab4483 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 17 Sep 2024 18:10:56 +0200 Subject: [PATCH 08/10] Disable analyzer in WindowView correctly --- src/Storages/WindowView/StorageWindowView.cpp | 76 +++++++++++-------- src/Storages/WindowView/StorageWindowView.h | 2 +- ...user_defined_directory_in_client.reference | 1 + ...36_udf_user_defined_directory_in_client.sh | 3 +- 4 files changed, 48 insertions(+), 34 deletions(-) diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index 4a20a07ae89..11259b73576 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -79,6 +79,19 @@ namespace ErrorCodes extern const int UNSUPPORTED_METHOD; } +namespace +{ + +ContextPtr getCorrectContext() +{ + auto result = Context::createCopy(Context::getGlobalContextInstance()); + result->setSetting("allow_experimental_analyzer", false); + return result; +} + +} + + namespace { /// Fetch all window info and replace tumble or hop node names with windowID @@ -428,7 +441,7 @@ ASTPtr StorageWindowView::getCleanupQuery() void StorageWindowView::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr local_context, TableExclusiveLockHolder &) { - InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Truncate, getContext(), local_context, inner_table_id, true); + InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Truncate, getCorrectContext(), local_context, inner_table_id, true); } bool StorageWindowView::optimize( @@ -473,7 +486,7 @@ void StorageWindowView::alter( output_header.clear(); InterpreterDropQuery::executeDropQuery( - ASTDropQuery::Kind::Drop, getContext(), local_context, inner_table_id, true); + ASTDropQuery::Kind::Drop, getCorrectContext(), local_context, inner_table_id, true); /// create inner table auto create_context = Context::createCopy(local_context); @@ -486,8 +499,8 @@ void StorageWindowView::alter( shutdown_called = false; - clean_cache_task = getContext()->getSchedulePool().createTask(getStorageID().getFullTableName(), [this] { threadFuncCleanup(); }); - fire_task = getContext()->getSchedulePool().createTask( + clean_cache_task = getCorrectContext()->getSchedulePool().createTask(getStorageID().getFullTableName(), [this] { threadFuncCleanup(); }); + fire_task = getCorrectContext()->getSchedulePool().createTask( getStorageID().getFullTableName(), [this] { is_proctime ? threadFuncFireProc() : threadFuncFireEvent(); }); clean_cache_task->deactivate(); fire_task->deactivate(); @@ -530,7 +543,7 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) auto inner_table = getInnerTable(); InterpreterSelectQuery fetch( inner_fetch_query, - getContext(), + getCorrectContext(), inner_table, inner_table->getInMemoryMetadataPtr(), SelectQueryOptions(QueryProcessingStage::FetchColumns)); @@ -564,8 +577,8 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) filter_function = makeASTFunction("has", func_array, std::make_shared(window_id_name)); } - auto syntax_result = TreeRewriter(getContext()).analyze(filter_function, builder.getHeader().getNamesAndTypesList()); - auto filter_expression = ExpressionAnalyzer(filter_function, syntax_result, getContext()).getActionsDAG(false); + auto syntax_result = TreeRewriter(getCorrectContext()).analyze(filter_function, builder.getHeader().getNamesAndTypesList()); + auto filter_expression = ExpressionAnalyzer(filter_function, syntax_result, getCorrectContext()).getActionsDAG(false); auto filter_actions = std::make_shared(std::move(filter_expression)); builder.addSimpleTransform([&](const Block & header) @@ -581,7 +594,7 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) column.column = column.type->createColumnConst(0, Tuple{w_start, watermark}); auto adding_column_dag = ActionsDAG::makeAddingColumnActions(std::move(column)); auto adding_column_actions - = std::make_shared(std::move(adding_column_dag), ExpressionActionsSettings::fromContext(getContext())); + = std::make_shared(std::move(adding_column_dag), ExpressionActionsSettings::fromContext(getCorrectContext())); builder.addSimpleTransform([&](const Block & header) { return std::make_shared(header, adding_column_actions); @@ -595,7 +608,7 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) new_header.getColumnsWithTypeAndName(), ActionsDAG::MatchColumnsMode::Name); auto actions = std::make_shared( - std::move(convert_actions_dag), ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes)); + std::move(convert_actions_dag), ExpressionActionsSettings::fromContext(getCorrectContext(), CompileExpressions::yes)); builder.addSimpleTransform([&](const Block & stream_header) { return std::make_shared(stream_header, actions); @@ -619,11 +632,11 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) return StorageBlocks::createStorage(blocks_id_global, required_columns, std::move(pipes), QueryProcessingStage::WithMergeableState); }; - TemporaryTableHolder blocks_storage(getContext(), creator); + TemporaryTableHolder blocks_storage(getCorrectContext(), creator); InterpreterSelectQuery select( getFinalQuery(), - getContext(), + getCorrectContext(), blocks_storage.getTable(), blocks_storage.getTable()->getInMemoryMetadataPtr(), SelectQueryOptions(QueryProcessingStage::Complete)); @@ -638,8 +651,8 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) { return std::make_shared( current_header, - getContext()->getSettingsRef().min_insert_block_size_rows, - getContext()->getSettingsRef().min_insert_block_size_bytes); + getCorrectContext()->getSettingsRef().min_insert_block_size_rows, + getCorrectContext()->getSettingsRef().min_insert_block_size_bytes); }); auto header = builder.getHeader(); @@ -694,7 +707,7 @@ inline void StorageWindowView::fire(UInt32 watermark) insert->table_id = target_table->getStorageID(); InterpreterInsertQuery interpreter( insert, - getContext(), + getCorrectContext(), /* allow_materialized */ false, /* no_squash */ false, /* no_destination */ false, @@ -707,8 +720,8 @@ inline void StorageWindowView::fire(UInt32 watermark) pipe.getHeader(), block_io.pipeline.getHeader().getNamesAndTypesList(), getTargetTable()->getInMemoryMetadataPtr()->getColumns(), - getContext(), - getContext()->getSettingsRef().insert_null_as_default); + getCorrectContext(), + getCorrectContext()->getSettingsRef().insert_null_as_default); auto adding_missing_defaults_actions = std::make_shared(std::move(adding_missing_defaults_dag)); pipe.addSimpleTransform([&](const Block & stream_header) { @@ -721,7 +734,7 @@ inline void StorageWindowView::fire(UInt32 watermark) ActionsDAG::MatchColumnsMode::Position); auto actions = std::make_shared( std::move(convert_actions_dag), - ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes)); + ExpressionActionsSettings::fromContext(getCorrectContext(), CompileExpressions::yes)); pipe.addSimpleTransform([&](const Block & stream_header) { return std::make_shared(stream_header, actions); @@ -742,7 +755,7 @@ ASTPtr StorageWindowView::getSourceTableSelectQuery() if (hasJoin(modified_select)) { auto analyzer_res = TreeRewriterResult({}); - removeJoin(modified_select, analyzer_res, getContext()); + removeJoin(modified_select, analyzer_res, getCorrectContext()); } else { @@ -791,13 +804,13 @@ ASTPtr StorageWindowView::getInnerTableCreateQuery(const ASTPtr & inner_query, c Aliases aliases; QueryAliasesVisitor(aliases).visit(inner_query); auto inner_query_normalized = inner_query->clone(); - QueryNormalizer::Data normalizer_data(aliases, {}, false, getContext()->getSettingsRef(), false); + QueryNormalizer::Data normalizer_data(aliases, {}, false, getCorrectContext()->getSettingsRef(), false); QueryNormalizer(normalizer_data).visit(inner_query_normalized); auto inner_select_query = std::static_pointer_cast(inner_query_normalized); auto t_sample_block - = InterpreterSelectQuery(inner_select_query, getContext(), SelectQueryOptions(QueryProcessingStage::WithMergeableState)) + = InterpreterSelectQuery(inner_select_query, getCorrectContext(), SelectQueryOptions(QueryProcessingStage::WithMergeableState)) .getSampleBlock(); ASTPtr columns_list = InterpreterCreateQuery::formatColumns(t_sample_block.getNamesAndTypesList()); @@ -1015,7 +1028,7 @@ void StorageWindowView::cleanup() std::lock_guard mutex_lock(mutex); auto alter_query = getCleanupQuery(); - auto cleanup_context = Context::createCopy(getContext()); + auto cleanup_context = Context::createCopy(getCorrectContext()); cleanup_context->makeQueryContext(); cleanup_context->setCurrentQueryId(""); cleanup_context->setQueryKindReplicatedDatabaseInternal(); @@ -1214,7 +1227,6 @@ StorageWindowView::StorageWindowView( const String & comment, LoadingStrictnessLevel mode) : IStorage(table_id_) - , WithContext(context_->getGlobalContext()) , log(getLogger(fmt::format("StorageWindowView({}.{})", table_id_.database_name, table_id_.table_name))) , fire_signal_timeout_s(context_->getSettingsRef().wait_for_window_view_fire_signal_timeout.totalSeconds()) , clean_interval_usec(context_->getSettingsRef().window_view_clean_interval.totalMicroseconds()) @@ -1294,8 +1306,8 @@ StorageWindowView::StorageWindowView( if (disabled_due_to_analyzer) return; - clean_cache_task = getContext()->getSchedulePool().createTask(getStorageID().getFullTableName(), [this] { threadFuncCleanup(); }); - fire_task = getContext()->getSchedulePool().createTask( + clean_cache_task = getCorrectContext()->getSchedulePool().createTask(getStorageID().getFullTableName(), [this] { threadFuncCleanup(); }); + fire_task = getCorrectContext()->getSchedulePool().createTask( getStorageID().getFullTableName(), [this] { is_proctime ? threadFuncFireProc() : threadFuncFireEvent(); }); clean_cache_task->deactivate(); fire_task->deactivate(); @@ -1306,7 +1318,7 @@ ASTPtr StorageWindowView::initInnerQuery(ASTSelectQuery query, ContextPtr contex select_query = query.clone(); output_header.clear(); - String select_database_name = getContext()->getCurrentDatabase(); + String select_database_name = getCorrectContext()->getCurrentDatabase(); String select_table_name; auto select_query_tmp = query.clone(); extractDependentTable(context_, select_query_tmp, select_database_name, select_table_name); @@ -1723,7 +1735,7 @@ void StorageWindowView::drop() { /// Must be guaranteed at this point for database engine Atomic that has_inner_table == false, /// because otherwise will be a deadlock. - dropInnerTableIfAny(false, getContext()); + dropInnerTableIfAny(false, getCorrectContext()); } void StorageWindowView::dropInnerTableIfAny(bool sync, ContextPtr local_context) @@ -1734,10 +1746,10 @@ void StorageWindowView::dropInnerTableIfAny(bool sync, ContextPtr local_context) try { InterpreterDropQuery::executeDropQuery( - ASTDropQuery::Kind::Drop, getContext(), local_context, inner_table_id, sync); + ASTDropQuery::Kind::Drop, getCorrectContext(), local_context, inner_table_id, sync); if (has_inner_target_table) - InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, target_table_id, sync, /* ignore_sync_setting */ true); + InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getCorrectContext(), local_context, target_table_id, sync, /* ignore_sync_setting */ true); } catch (...) { @@ -1757,7 +1769,7 @@ const Block & StorageWindowView::getOutputHeader() const std::lock_guard lock(sample_block_lock); if (!output_header) { - output_header = InterpreterSelectQuery(select_query->clone(), getContext(), SelectQueryOptions(QueryProcessingStage::Complete)) + output_header = InterpreterSelectQuery(select_query->clone(), getCorrectContext(), SelectQueryOptions(QueryProcessingStage::Complete)) .getSampleBlock(); } return output_header; @@ -1765,17 +1777,17 @@ const Block & StorageWindowView::getOutputHeader() const StoragePtr StorageWindowView::getSourceTable() const { - return DatabaseCatalog::instance().getTable(select_table_id, getContext()); + return DatabaseCatalog::instance().getTable(select_table_id, getCorrectContext()); } StoragePtr StorageWindowView::getInnerTable() const { - return DatabaseCatalog::instance().getTable(inner_table_id, getContext()); + return DatabaseCatalog::instance().getTable(inner_table_id, getCorrectContext()); } StoragePtr StorageWindowView::getTargetTable() const { - return DatabaseCatalog::instance().getTable(target_table_id, getContext()); + return DatabaseCatalog::instance().getTable(target_table_id, getCorrectContext()); } void StorageWindowView::throwIfWindowViewIsDisabled(ContextPtr local_context) const diff --git a/src/Storages/WindowView/StorageWindowView.h b/src/Storages/WindowView/StorageWindowView.h index 38fca512ed9..3d48eaae548 100644 --- a/src/Storages/WindowView/StorageWindowView.h +++ b/src/Storages/WindowView/StorageWindowView.h @@ -100,7 +100,7 @@ using ASTPtr = std::shared_ptr; * Users need to take these duplicated results into account. */ -class StorageWindowView final : public IStorage, WithContext +class StorageWindowView final : public IStorage { friend class WindowViewSource; friend class WatermarkTransform; diff --git a/tests/queries/0_stateless/03036_udf_user_defined_directory_in_client.reference b/tests/queries/0_stateless/03036_udf_user_defined_directory_in_client.reference index 251d054748a..76800809c34 100644 --- a/tests/queries/0_stateless/03036_udf_user_defined_directory_in_client.reference +++ b/tests/queries/0_stateless/03036_udf_user_defined_directory_in_client.reference @@ -1 +1,2 @@ Unknown function +Function with name 'xyz' does not exist. diff --git a/tests/queries/0_stateless/03036_udf_user_defined_directory_in_client.sh b/tests/queries/0_stateless/03036_udf_user_defined_directory_in_client.sh index e0a145d8456..3e93920e94c 100755 --- a/tests/queries/0_stateless/03036_udf_user_defined_directory_in_client.sh +++ b/tests/queries/0_stateless/03036_udf_user_defined_directory_in_client.sh @@ -8,7 +8,8 @@ ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test" ${CLICKHOUSE_CLIENT} --query "CREATE TABLE test (s String) ENGINE = Memory" # Calling an unknown function should not lead to creation of a 'user_defined' directory in the current directory -${CLICKHOUSE_CLIENT} --query "INSERT INTO test VALUES (xyz('abc'))" 2>&1 | grep -o -F 'Unknown function' +${CLICKHOUSE_CLIENT} --query "SET allow_experimental_analyzer = 0;INSERT INTO test VALUES (xyz('abc'))" 2>&1 | grep -o -F 'Unknown function' +${CLICKHOUSE_CLIENT} --query "SET allow_experimental_analyzer = 1;INSERT INTO test VALUES (xyz('abc'))" 2>&1 | grep -o -F "Function with name 'xyz' does not exist." ls -ld user_defined 2> /dev/null From f44960bb63e28b30dc4d459fb048b13b8ffd936b Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 17 Sep 2024 18:23:43 +0200 Subject: [PATCH 09/10] Add logs when disabling analyzer --- src/Interpreters/IInterpreterUnionOrSelectQuery.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp b/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp index b7931491704..27716aeca57 100644 --- a/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp +++ b/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace DB @@ -28,7 +29,13 @@ IInterpreterUnionOrSelectQuery::IInterpreterUnionOrSelectQuery(const ASTPtr & qu /// it's possible that new analyzer will be enabled in ::getQueryProcessingStage method /// of the underlying storage when all other parts of infrastructure are not ready for it /// (built with old analyzer). - // context->setSetting("allow_experimental_analyzer", false); + if (context->getSettingsRef().allow_experimental_analyzer) + { + LOG_TRACE(getLogger("IInterpreterUnionOrSelectQuery"), + "The new analyzer is enabled, but the old interpreter is used. It can be a bug, please report it. Will disable 'allow_experimental_analyzer' setting (for query: {})", + query_ptr->formatForLogging()); + context->setSetting("allow_experimental_analyzer", false); + } if (options.shard_num) context->addSpecialScalar( From 1ca79602ad014615960938b97c208ec177bd7ea0 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 17 Sep 2024 22:41:59 +0200 Subject: [PATCH 10/10] Revert changes in WindowView --- src/Storages/WindowView/StorageWindowView.cpp | 76 ++++++++----------- src/Storages/WindowView/StorageWindowView.h | 2 +- 2 files changed, 33 insertions(+), 45 deletions(-) diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index 11259b73576..4a20a07ae89 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -79,19 +79,6 @@ namespace ErrorCodes extern const int UNSUPPORTED_METHOD; } -namespace -{ - -ContextPtr getCorrectContext() -{ - auto result = Context::createCopy(Context::getGlobalContextInstance()); - result->setSetting("allow_experimental_analyzer", false); - return result; -} - -} - - namespace { /// Fetch all window info and replace tumble or hop node names with windowID @@ -441,7 +428,7 @@ ASTPtr StorageWindowView::getCleanupQuery() void StorageWindowView::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr local_context, TableExclusiveLockHolder &) { - InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Truncate, getCorrectContext(), local_context, inner_table_id, true); + InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Truncate, getContext(), local_context, inner_table_id, true); } bool StorageWindowView::optimize( @@ -486,7 +473,7 @@ void StorageWindowView::alter( output_header.clear(); InterpreterDropQuery::executeDropQuery( - ASTDropQuery::Kind::Drop, getCorrectContext(), local_context, inner_table_id, true); + ASTDropQuery::Kind::Drop, getContext(), local_context, inner_table_id, true); /// create inner table auto create_context = Context::createCopy(local_context); @@ -499,8 +486,8 @@ void StorageWindowView::alter( shutdown_called = false; - clean_cache_task = getCorrectContext()->getSchedulePool().createTask(getStorageID().getFullTableName(), [this] { threadFuncCleanup(); }); - fire_task = getCorrectContext()->getSchedulePool().createTask( + clean_cache_task = getContext()->getSchedulePool().createTask(getStorageID().getFullTableName(), [this] { threadFuncCleanup(); }); + fire_task = getContext()->getSchedulePool().createTask( getStorageID().getFullTableName(), [this] { is_proctime ? threadFuncFireProc() : threadFuncFireEvent(); }); clean_cache_task->deactivate(); fire_task->deactivate(); @@ -543,7 +530,7 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) auto inner_table = getInnerTable(); InterpreterSelectQuery fetch( inner_fetch_query, - getCorrectContext(), + getContext(), inner_table, inner_table->getInMemoryMetadataPtr(), SelectQueryOptions(QueryProcessingStage::FetchColumns)); @@ -577,8 +564,8 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) filter_function = makeASTFunction("has", func_array, std::make_shared(window_id_name)); } - auto syntax_result = TreeRewriter(getCorrectContext()).analyze(filter_function, builder.getHeader().getNamesAndTypesList()); - auto filter_expression = ExpressionAnalyzer(filter_function, syntax_result, getCorrectContext()).getActionsDAG(false); + auto syntax_result = TreeRewriter(getContext()).analyze(filter_function, builder.getHeader().getNamesAndTypesList()); + auto filter_expression = ExpressionAnalyzer(filter_function, syntax_result, getContext()).getActionsDAG(false); auto filter_actions = std::make_shared(std::move(filter_expression)); builder.addSimpleTransform([&](const Block & header) @@ -594,7 +581,7 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) column.column = column.type->createColumnConst(0, Tuple{w_start, watermark}); auto adding_column_dag = ActionsDAG::makeAddingColumnActions(std::move(column)); auto adding_column_actions - = std::make_shared(std::move(adding_column_dag), ExpressionActionsSettings::fromContext(getCorrectContext())); + = std::make_shared(std::move(adding_column_dag), ExpressionActionsSettings::fromContext(getContext())); builder.addSimpleTransform([&](const Block & header) { return std::make_shared(header, adding_column_actions); @@ -608,7 +595,7 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) new_header.getColumnsWithTypeAndName(), ActionsDAG::MatchColumnsMode::Name); auto actions = std::make_shared( - std::move(convert_actions_dag), ExpressionActionsSettings::fromContext(getCorrectContext(), CompileExpressions::yes)); + std::move(convert_actions_dag), ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes)); builder.addSimpleTransform([&](const Block & stream_header) { return std::make_shared(stream_header, actions); @@ -632,11 +619,11 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) return StorageBlocks::createStorage(blocks_id_global, required_columns, std::move(pipes), QueryProcessingStage::WithMergeableState); }; - TemporaryTableHolder blocks_storage(getCorrectContext(), creator); + TemporaryTableHolder blocks_storage(getContext(), creator); InterpreterSelectQuery select( getFinalQuery(), - getCorrectContext(), + getContext(), blocks_storage.getTable(), blocks_storage.getTable()->getInMemoryMetadataPtr(), SelectQueryOptions(QueryProcessingStage::Complete)); @@ -651,8 +638,8 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) { return std::make_shared( current_header, - getCorrectContext()->getSettingsRef().min_insert_block_size_rows, - getCorrectContext()->getSettingsRef().min_insert_block_size_bytes); + getContext()->getSettingsRef().min_insert_block_size_rows, + getContext()->getSettingsRef().min_insert_block_size_bytes); }); auto header = builder.getHeader(); @@ -707,7 +694,7 @@ inline void StorageWindowView::fire(UInt32 watermark) insert->table_id = target_table->getStorageID(); InterpreterInsertQuery interpreter( insert, - getCorrectContext(), + getContext(), /* allow_materialized */ false, /* no_squash */ false, /* no_destination */ false, @@ -720,8 +707,8 @@ inline void StorageWindowView::fire(UInt32 watermark) pipe.getHeader(), block_io.pipeline.getHeader().getNamesAndTypesList(), getTargetTable()->getInMemoryMetadataPtr()->getColumns(), - getCorrectContext(), - getCorrectContext()->getSettingsRef().insert_null_as_default); + getContext(), + getContext()->getSettingsRef().insert_null_as_default); auto adding_missing_defaults_actions = std::make_shared(std::move(adding_missing_defaults_dag)); pipe.addSimpleTransform([&](const Block & stream_header) { @@ -734,7 +721,7 @@ inline void StorageWindowView::fire(UInt32 watermark) ActionsDAG::MatchColumnsMode::Position); auto actions = std::make_shared( std::move(convert_actions_dag), - ExpressionActionsSettings::fromContext(getCorrectContext(), CompileExpressions::yes)); + ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes)); pipe.addSimpleTransform([&](const Block & stream_header) { return std::make_shared(stream_header, actions); @@ -755,7 +742,7 @@ ASTPtr StorageWindowView::getSourceTableSelectQuery() if (hasJoin(modified_select)) { auto analyzer_res = TreeRewriterResult({}); - removeJoin(modified_select, analyzer_res, getCorrectContext()); + removeJoin(modified_select, analyzer_res, getContext()); } else { @@ -804,13 +791,13 @@ ASTPtr StorageWindowView::getInnerTableCreateQuery(const ASTPtr & inner_query, c Aliases aliases; QueryAliasesVisitor(aliases).visit(inner_query); auto inner_query_normalized = inner_query->clone(); - QueryNormalizer::Data normalizer_data(aliases, {}, false, getCorrectContext()->getSettingsRef(), false); + QueryNormalizer::Data normalizer_data(aliases, {}, false, getContext()->getSettingsRef(), false); QueryNormalizer(normalizer_data).visit(inner_query_normalized); auto inner_select_query = std::static_pointer_cast(inner_query_normalized); auto t_sample_block - = InterpreterSelectQuery(inner_select_query, getCorrectContext(), SelectQueryOptions(QueryProcessingStage::WithMergeableState)) + = InterpreterSelectQuery(inner_select_query, getContext(), SelectQueryOptions(QueryProcessingStage::WithMergeableState)) .getSampleBlock(); ASTPtr columns_list = InterpreterCreateQuery::formatColumns(t_sample_block.getNamesAndTypesList()); @@ -1028,7 +1015,7 @@ void StorageWindowView::cleanup() std::lock_guard mutex_lock(mutex); auto alter_query = getCleanupQuery(); - auto cleanup_context = Context::createCopy(getCorrectContext()); + auto cleanup_context = Context::createCopy(getContext()); cleanup_context->makeQueryContext(); cleanup_context->setCurrentQueryId(""); cleanup_context->setQueryKindReplicatedDatabaseInternal(); @@ -1227,6 +1214,7 @@ StorageWindowView::StorageWindowView( const String & comment, LoadingStrictnessLevel mode) : IStorage(table_id_) + , WithContext(context_->getGlobalContext()) , log(getLogger(fmt::format("StorageWindowView({}.{})", table_id_.database_name, table_id_.table_name))) , fire_signal_timeout_s(context_->getSettingsRef().wait_for_window_view_fire_signal_timeout.totalSeconds()) , clean_interval_usec(context_->getSettingsRef().window_view_clean_interval.totalMicroseconds()) @@ -1306,8 +1294,8 @@ StorageWindowView::StorageWindowView( if (disabled_due_to_analyzer) return; - clean_cache_task = getCorrectContext()->getSchedulePool().createTask(getStorageID().getFullTableName(), [this] { threadFuncCleanup(); }); - fire_task = getCorrectContext()->getSchedulePool().createTask( + clean_cache_task = getContext()->getSchedulePool().createTask(getStorageID().getFullTableName(), [this] { threadFuncCleanup(); }); + fire_task = getContext()->getSchedulePool().createTask( getStorageID().getFullTableName(), [this] { is_proctime ? threadFuncFireProc() : threadFuncFireEvent(); }); clean_cache_task->deactivate(); fire_task->deactivate(); @@ -1318,7 +1306,7 @@ ASTPtr StorageWindowView::initInnerQuery(ASTSelectQuery query, ContextPtr contex select_query = query.clone(); output_header.clear(); - String select_database_name = getCorrectContext()->getCurrentDatabase(); + String select_database_name = getContext()->getCurrentDatabase(); String select_table_name; auto select_query_tmp = query.clone(); extractDependentTable(context_, select_query_tmp, select_database_name, select_table_name); @@ -1735,7 +1723,7 @@ void StorageWindowView::drop() { /// Must be guaranteed at this point for database engine Atomic that has_inner_table == false, /// because otherwise will be a deadlock. - dropInnerTableIfAny(false, getCorrectContext()); + dropInnerTableIfAny(false, getContext()); } void StorageWindowView::dropInnerTableIfAny(bool sync, ContextPtr local_context) @@ -1746,10 +1734,10 @@ void StorageWindowView::dropInnerTableIfAny(bool sync, ContextPtr local_context) try { InterpreterDropQuery::executeDropQuery( - ASTDropQuery::Kind::Drop, getCorrectContext(), local_context, inner_table_id, sync); + ASTDropQuery::Kind::Drop, getContext(), local_context, inner_table_id, sync); if (has_inner_target_table) - InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getCorrectContext(), local_context, target_table_id, sync, /* ignore_sync_setting */ true); + InterpreterDropQuery::executeDropQuery(ASTDropQuery::Kind::Drop, getContext(), local_context, target_table_id, sync, /* ignore_sync_setting */ true); } catch (...) { @@ -1769,7 +1757,7 @@ const Block & StorageWindowView::getOutputHeader() const std::lock_guard lock(sample_block_lock); if (!output_header) { - output_header = InterpreterSelectQuery(select_query->clone(), getCorrectContext(), SelectQueryOptions(QueryProcessingStage::Complete)) + output_header = InterpreterSelectQuery(select_query->clone(), getContext(), SelectQueryOptions(QueryProcessingStage::Complete)) .getSampleBlock(); } return output_header; @@ -1777,17 +1765,17 @@ const Block & StorageWindowView::getOutputHeader() const StoragePtr StorageWindowView::getSourceTable() const { - return DatabaseCatalog::instance().getTable(select_table_id, getCorrectContext()); + return DatabaseCatalog::instance().getTable(select_table_id, getContext()); } StoragePtr StorageWindowView::getInnerTable() const { - return DatabaseCatalog::instance().getTable(inner_table_id, getCorrectContext()); + return DatabaseCatalog::instance().getTable(inner_table_id, getContext()); } StoragePtr StorageWindowView::getTargetTable() const { - return DatabaseCatalog::instance().getTable(target_table_id, getCorrectContext()); + return DatabaseCatalog::instance().getTable(target_table_id, getContext()); } void StorageWindowView::throwIfWindowViewIsDisabled(ContextPtr local_context) const diff --git a/src/Storages/WindowView/StorageWindowView.h b/src/Storages/WindowView/StorageWindowView.h index 3d48eaae548..38fca512ed9 100644 --- a/src/Storages/WindowView/StorageWindowView.h +++ b/src/Storages/WindowView/StorageWindowView.h @@ -100,7 +100,7 @@ using ASTPtr = std::shared_ptr; * Users need to take these duplicated results into account. */ -class StorageWindowView final : public IStorage +class StorageWindowView final : public IStorage, WithContext { friend class WindowViewSource; friend class WatermarkTransform;