mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-29 21:20:49 +00:00
Try fix test.
This commit is contained in:
parent
c481401b0b
commit
fc415952ce
@ -246,7 +246,9 @@ public:
|
|||||||
// static std::vector<ssize_t> getInputsPositions(const Block & block, const NameToNodeMap & inputs_mapping);
|
// static std::vector<ssize_t> getInputsPositions(const Block & block, const NameToNodeMap & inputs_mapping);
|
||||||
// void transformBlock(Block & block, std::vector<ssize_t> inputs_pos, ColumnsWithTypeAndName result_columns) const;
|
// void transformBlock(Block & block, std::vector<ssize_t> inputs_pos, ColumnsWithTypeAndName result_columns) const;
|
||||||
|
|
||||||
|
#if USE_EMBEDDED_COMPILER
|
||||||
void compileExpressions(size_t min_count_to_compile_expression);
|
void compileExpressions(size_t min_count_to_compile_expression);
|
||||||
|
#endif
|
||||||
|
|
||||||
ActionsDAGPtr clone() const;
|
ActionsDAGPtr clone() const;
|
||||||
|
|
||||||
@ -328,7 +330,9 @@ private:
|
|||||||
void removeUnusedActions(bool allow_remove_inputs = true);
|
void removeUnusedActions(bool allow_remove_inputs = true);
|
||||||
void addAliases(const NamesWithAliases & aliases, bool project);
|
void addAliases(const NamesWithAliases & aliases, bool project);
|
||||||
|
|
||||||
|
#if USE_EMBEDDED_COMPILER
|
||||||
void compileFunctions(size_t min_count_to_compile_expression);
|
void compileFunctions(size_t min_count_to_compile_expression);
|
||||||
|
#endif
|
||||||
|
|
||||||
ActionsDAGPtr cloneActionsForConjunction(NodeRawConstPtrs conjunction);
|
ActionsDAGPtr cloneActionsForConjunction(NodeRawConstPtrs conjunction);
|
||||||
};
|
};
|
||||||
|
@ -623,11 +623,10 @@ void ExpressionActionsChain::finalize()
|
|||||||
/// Finalize all steps. Right to left to define unnecessary input columns.
|
/// Finalize all steps. Right to left to define unnecessary input columns.
|
||||||
for (int i = static_cast<int>(steps.size()) - 1; i >= 0; --i)
|
for (int i = static_cast<int>(steps.size()) - 1; i >= 0; --i)
|
||||||
{
|
{
|
||||||
Names required_output = steps[i]->required_output;
|
auto & required_output = steps[i]->required_output;
|
||||||
std::unordered_map<String, size_t> required_output_indexes;
|
Names required_names;
|
||||||
for (size_t j = 0; j < required_output.size(); ++j)
|
for (const auto & output : required_output)
|
||||||
required_output_indexes[required_output[j]] = j;
|
required_names.push_back(output.first);
|
||||||
auto & can_remove_required_output = steps[i]->can_remove_required_output;
|
|
||||||
|
|
||||||
if (i + 1 < static_cast<int>(steps.size()))
|
if (i + 1 < static_cast<int>(steps.size()))
|
||||||
{
|
{
|
||||||
@ -636,15 +635,15 @@ void ExpressionActionsChain::finalize()
|
|||||||
{
|
{
|
||||||
if (additional_input.count(it.name) == 0)
|
if (additional_input.count(it.name) == 0)
|
||||||
{
|
{
|
||||||
auto iter = required_output_indexes.find(it.name);
|
auto iter = required_output.find(it.name);
|
||||||
if (iter == required_output_indexes.end())
|
if (iter == required_output.end())
|
||||||
required_output.push_back(it.name);
|
required_names.push_back(it.name);
|
||||||
else if (!can_remove_required_output.empty())
|
else
|
||||||
can_remove_required_output[iter->second] = false;
|
iter->second = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
steps[i]->finalize(required_output);
|
steps[i]->finalize(required_names);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Adding the ejection of unnecessary columns to the beginning of each step.
|
/// Adding the ejection of unnecessary columns to the beginning of each step.
|
||||||
@ -668,8 +667,8 @@ std::string ExpressionActionsChain::dumpChain() const
|
|||||||
{
|
{
|
||||||
ss << "step " << i << "\n";
|
ss << "step " << i << "\n";
|
||||||
ss << "required output:\n";
|
ss << "required output:\n";
|
||||||
for (const std::string & name : steps[i]->required_output)
|
for (const auto & it : steps[i]->required_output)
|
||||||
ss << name << "\n";
|
ss << it.first << "\n";
|
||||||
ss << "\n" << steps[i]->dump() << "\n";
|
ss << "\n" << steps[i]->dump() << "\n";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -140,16 +140,21 @@ struct ExpressionActionsChain
|
|||||||
struct Step
|
struct Step
|
||||||
{
|
{
|
||||||
virtual ~Step() = default;
|
virtual ~Step() = default;
|
||||||
explicit Step(Names required_output_) : required_output(std::move(required_output_)) {}
|
explicit Step(Names required_output_)
|
||||||
|
{
|
||||||
|
for (const auto & name : required_output_)
|
||||||
|
required_output[name] = true;
|
||||||
|
}
|
||||||
|
|
||||||
/// Columns were added to the block before current step in addition to prev step output.
|
/// Columns were added to the block before current step in addition to prev step output.
|
||||||
NameSet additional_input;
|
NameSet additional_input;
|
||||||
/// Columns which are required in the result of current step.
|
/// Columns which are required in the result of current step.
|
||||||
Names required_output;
|
/// Flag is true if column from required_output is needed only for current step and not used in next actions
|
||||||
/// True if column from required_output is needed only for current step and not used in next actions
|
|
||||||
/// (and can be removed from block). Example: filter column for where actions.
|
/// (and can be removed from block). Example: filter column for where actions.
|
||||||
/// If not empty, has the same size with required_output; is filled in finalize().
|
/// If not empty, has the same size with required_output; is filled in finalize().
|
||||||
std::vector<bool> can_remove_required_output;
|
std::unordered_map<std::string, bool> required_output;
|
||||||
|
|
||||||
|
void addRequiredOutput(const std::string & name) { required_output[name] = true; }
|
||||||
|
|
||||||
virtual NamesAndTypesList getRequiredColumns() const = 0;
|
virtual NamesAndTypesList getRequiredColumns() const = 0;
|
||||||
virtual ColumnsWithTypeAndName getResultColumns() const = 0;
|
virtual ColumnsWithTypeAndName getResultColumns() const = 0;
|
||||||
|
@ -701,7 +701,7 @@ ArrayJoinActionPtr ExpressionAnalyzer::addMultipleArrayJoinAction(ActionsDAGPtr
|
|||||||
/// Assign new names to columns, if needed.
|
/// Assign new names to columns, if needed.
|
||||||
if (result_source.first != result_source.second)
|
if (result_source.first != result_source.second)
|
||||||
{
|
{
|
||||||
const auto & index = actions->getIndex();
|
auto & index = actions->getIndex();
|
||||||
auto it = index.begin();
|
auto it = index.begin();
|
||||||
for (; it != index.end(); ++it)
|
for (; it != index.end(); ++it)
|
||||||
if ((*it)->result_name == result_source.second)
|
if ((*it)->result_name == result_source.second)
|
||||||
@ -710,7 +710,7 @@ ArrayJoinActionPtr ExpressionAnalyzer::addMultipleArrayJoinAction(ActionsDAGPtr
|
|||||||
if (it == index.end())
|
if (it == index.end())
|
||||||
throw Exception("Unknown identifier: '" + result_source.second + "'", ErrorCodes::UNKNOWN_IDENTIFIER);
|
throw Exception("Unknown identifier: '" + result_source.second + "'", ErrorCodes::UNKNOWN_IDENTIFIER);
|
||||||
|
|
||||||
actions->addAlias(**it, result_source.first);
|
index.push_back(&actions->addAlias(**it, result_source.first));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Make ARRAY JOIN (replace arrays with their insides) for the columns in these new names.
|
/// Make ARRAY JOIN (replace arrays with their insides) for the columns in these new names.
|
||||||
@ -888,8 +888,7 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendPrewhere(
|
|||||||
auto & step = chain.lastStep(sourceColumns());
|
auto & step = chain.lastStep(sourceColumns());
|
||||||
getRootActions(select_query->prewhere(), only_types, step.actions());
|
getRootActions(select_query->prewhere(), only_types, step.actions());
|
||||||
String prewhere_column_name = select_query->prewhere()->getColumnName();
|
String prewhere_column_name = select_query->prewhere()->getColumnName();
|
||||||
step.required_output.push_back(prewhere_column_name);
|
step.addRequiredOutput(prewhere_column_name);
|
||||||
step.can_remove_required_output.push_back(true);
|
|
||||||
|
|
||||||
const auto & index = step.actions()->getIndex();
|
const auto & index = step.actions()->getIndex();
|
||||||
auto it = index.begin();
|
auto it = index.begin();
|
||||||
@ -919,10 +918,7 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendPrewhere(
|
|||||||
for (const auto & column : additional_required_columns)
|
for (const auto & column : additional_required_columns)
|
||||||
{
|
{
|
||||||
if (required_source_columns.count(column))
|
if (required_source_columns.count(column))
|
||||||
{
|
step.addRequiredOutput(column);
|
||||||
step.required_output.push_back(column);
|
|
||||||
step.can_remove_required_output.push_back(true);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
auto names = step.actions()->getNames();
|
auto names = step.actions()->getNames();
|
||||||
@ -979,8 +975,7 @@ void SelectQueryExpressionAnalyzer::appendPreliminaryFilter(ExpressionActionsCha
|
|||||||
// FIXME: assert(filter_info);
|
// FIXME: assert(filter_info);
|
||||||
auto * expression_step = typeid_cast<ExpressionActionsChain::ExpressionActionsStep *>(&step);
|
auto * expression_step = typeid_cast<ExpressionActionsChain::ExpressionActionsStep *>(&step);
|
||||||
expression_step->actions_dag = std::move(actions_dag);
|
expression_step->actions_dag = std::move(actions_dag);
|
||||||
step.required_output.push_back(std::move(column_name));
|
step.addRequiredOutput(column_name);
|
||||||
step.can_remove_required_output = {true};
|
|
||||||
|
|
||||||
chain.addStep();
|
chain.addStep();
|
||||||
}
|
}
|
||||||
@ -997,8 +992,7 @@ bool SelectQueryExpressionAnalyzer::appendWhere(ExpressionActionsChain & chain,
|
|||||||
getRootActions(select_query->where(), only_types, step.actions());
|
getRootActions(select_query->where(), only_types, step.actions());
|
||||||
|
|
||||||
auto where_column_name = select_query->where()->getColumnName();
|
auto where_column_name = select_query->where()->getColumnName();
|
||||||
step.required_output.push_back(where_column_name);
|
step.addRequiredOutput(where_column_name);
|
||||||
step.can_remove_required_output = {true};
|
|
||||||
|
|
||||||
const auto & index = step.actions()->getIndex();
|
const auto & index = step.actions()->getIndex();
|
||||||
auto it = index.begin();
|
auto it = index.begin();
|
||||||
@ -1030,7 +1024,7 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain
|
|||||||
ASTs asts = select_query->groupBy()->children;
|
ASTs asts = select_query->groupBy()->children;
|
||||||
for (const auto & ast : asts)
|
for (const auto & ast : asts)
|
||||||
{
|
{
|
||||||
step.required_output.emplace_back(ast->getColumnName());
|
step.addRequiredOutput(ast->getColumnName());
|
||||||
getRootActions(ast, only_types, step.actions());
|
getRootActions(ast, only_types, step.actions());
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1055,7 +1049,7 @@ void SelectQueryExpressionAnalyzer::appendAggregateFunctionsArguments(Expression
|
|||||||
|
|
||||||
for (const auto & desc : aggregate_descriptions)
|
for (const auto & desc : aggregate_descriptions)
|
||||||
for (const auto & name : desc.argument_names)
|
for (const auto & name : desc.argument_names)
|
||||||
step.required_output.emplace_back(name);
|
step.addRequiredOutput(name);
|
||||||
|
|
||||||
/// Collect aggregates removing duplicates by node.getColumnName()
|
/// Collect aggregates removing duplicates by node.getColumnName()
|
||||||
/// It's not clear why we recollect aggregates (for query parts) while we're able to use previously collected ones (for entire query)
|
/// It's not clear why we recollect aggregates (for query parts) while we're able to use previously collected ones (for entire query)
|
||||||
@ -1112,14 +1106,14 @@ void SelectQueryExpressionAnalyzer::appendWindowFunctionsArguments(
|
|||||||
// (2b) Required function argument columns.
|
// (2b) Required function argument columns.
|
||||||
for (const auto & a : f.function_node->arguments->children)
|
for (const auto & a : f.function_node->arguments->children)
|
||||||
{
|
{
|
||||||
step.required_output.push_back(a->getColumnName());
|
step.addRequiredOutput(a->getColumnName());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// (2a) Required PARTITION BY and ORDER BY columns.
|
// (2a) Required PARTITION BY and ORDER BY columns.
|
||||||
for (const auto & c : w.full_sort_description)
|
for (const auto & c : w.full_sort_description)
|
||||||
{
|
{
|
||||||
step.required_output.push_back(c.column_name);
|
step.addRequiredOutput(c.column_name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -1134,7 +1128,7 @@ bool SelectQueryExpressionAnalyzer::appendHaving(ExpressionActionsChain & chain,
|
|||||||
ExpressionActionsChain::Step & step = chain.lastStep(aggregated_columns);
|
ExpressionActionsChain::Step & step = chain.lastStep(aggregated_columns);
|
||||||
|
|
||||||
getRootActionsForHaving(select_query->having(), only_types, step.actions());
|
getRootActionsForHaving(select_query->having(), only_types, step.actions());
|
||||||
step.required_output.push_back(select_query->having()->getColumnName());
|
step.addRequiredOutput(select_query->having()->getColumnName());
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@ -1158,7 +1152,7 @@ void SelectQueryExpressionAnalyzer::appendSelect(ExpressionActionsChain & chain,
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
step.required_output.push_back(child->getColumnName());
|
step.addRequiredOutput(child->getColumnName());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1186,7 +1180,7 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChai
|
|||||||
if (!ast || ast->children.empty())
|
if (!ast || ast->children.empty())
|
||||||
throw Exception("Bad order expression AST", ErrorCodes::UNKNOWN_TYPE_OF_AST_NODE);
|
throw Exception("Bad order expression AST", ErrorCodes::UNKNOWN_TYPE_OF_AST_NODE);
|
||||||
ASTPtr order_expression = ast->children.at(0);
|
ASTPtr order_expression = ast->children.at(0);
|
||||||
step.required_output.push_back(order_expression->getColumnName());
|
step.addRequiredOutput(order_expression->getColumnName());
|
||||||
|
|
||||||
if (ast->with_fill)
|
if (ast->with_fill)
|
||||||
with_fill = true;
|
with_fill = true;
|
||||||
@ -1230,7 +1224,7 @@ bool SelectQueryExpressionAnalyzer::appendLimitBy(ExpressionActionsChain & chain
|
|||||||
NameSet aggregated_names;
|
NameSet aggregated_names;
|
||||||
for (const auto & column : aggregated_columns)
|
for (const auto & column : aggregated_columns)
|
||||||
{
|
{
|
||||||
step.required_output.push_back(column.name);
|
step.addRequiredOutput(column.name);
|
||||||
aggregated_names.insert(column.name);
|
aggregated_names.insert(column.name);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1238,7 +1232,7 @@ bool SelectQueryExpressionAnalyzer::appendLimitBy(ExpressionActionsChain & chain
|
|||||||
{
|
{
|
||||||
auto child_name = child->getColumnName();
|
auto child_name = child->getColumnName();
|
||||||
if (!aggregated_names.count(child_name))
|
if (!aggregated_names.count(child_name))
|
||||||
step.required_output.push_back(std::move(child_name));
|
step.addRequiredOutput(std::move(child_name));
|
||||||
}
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
@ -1286,7 +1280,7 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendProjectResult(ExpressionActio
|
|||||||
}
|
}
|
||||||
|
|
||||||
result_columns.emplace_back(source_name, result_name);
|
result_columns.emplace_back(source_name, result_name);
|
||||||
step.required_output.push_back(result_columns.back().second);
|
step.addRequiredOutput(result_columns.back().second);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1300,7 +1294,7 @@ void ExpressionAnalyzer::appendExpression(ExpressionActionsChain & chain, const
|
|||||||
{
|
{
|
||||||
ExpressionActionsChain::Step & step = chain.lastStep(sourceColumns());
|
ExpressionActionsChain::Step & step = chain.lastStep(sourceColumns());
|
||||||
getRootActions(expr, only_types, step.actions());
|
getRootActions(expr, only_types, step.actions());
|
||||||
step.required_output.push_back(expr->getColumnName());
|
step.addRequiredOutput(expr->getColumnName());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -1413,7 +1407,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
|
|||||||
|
|
||||||
if (!finalized)
|
if (!finalized)
|
||||||
{
|
{
|
||||||
finalize(chain, where_step_num);
|
finalize(chain, where_step_num, query);
|
||||||
finalized = true;
|
finalized = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1586,11 +1580,14 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
|
|||||||
const auto * select_query = query_analyzer.getSelectQuery();
|
const auto * select_query = query_analyzer.getSelectQuery();
|
||||||
for (const auto & child : select_query->select()->children)
|
for (const auto & child : select_query->select()->children)
|
||||||
{
|
{
|
||||||
step.required_output.push_back(child->getColumnName());
|
step.addRequiredOutput(child->getColumnName());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
selected_columns = chain.getLastStep().required_output;
|
selected_columns.clear();
|
||||||
|
selected_columns.reserve(chain.getLastStep().required_output.size());
|
||||||
|
for (const auto & it : chain.getLastStep().required_output)
|
||||||
|
selected_columns.emplace_back(it.first);
|
||||||
|
|
||||||
has_order_by = query.orderBy() != nullptr;
|
has_order_by = query.orderBy() != nullptr;
|
||||||
before_order_by = query_analyzer.appendOrderBy(
|
before_order_by = query_analyzer.appendOrderBy(
|
||||||
@ -1616,18 +1613,19 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
|
|||||||
checkActions();
|
checkActions();
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExpressionAnalysisResult::finalize(const ExpressionActionsChain & chain, size_t where_step_num)
|
void ExpressionAnalysisResult::finalize(const ExpressionActionsChain & chain, size_t where_step_num, const ASTSelectQuery & query)
|
||||||
{
|
{
|
||||||
if (hasPrewhere())
|
if (hasPrewhere())
|
||||||
{
|
{
|
||||||
const ExpressionActionsChain::Step & step = *chain.steps.at(0);
|
const ExpressionActionsChain::Step & step = *chain.steps.at(0);
|
||||||
prewhere_info->remove_prewhere_column = step.can_remove_required_output.at(0);
|
|
||||||
|
|
||||||
NameSet columns_to_remove;
|
NameSet columns_to_remove;
|
||||||
for (size_t i = 1; i < step.required_output.size(); ++i)
|
for (const auto & [name, can_remove] : step.required_output)
|
||||||
{
|
{
|
||||||
if (step.can_remove_required_output[i])
|
if (name == prewhere_info->prewhere_column_name)
|
||||||
columns_to_remove.insert(step.required_output[i]);
|
prewhere_info->remove_prewhere_column = can_remove;
|
||||||
|
else if (can_remove)
|
||||||
|
columns_to_remove.insert(name);
|
||||||
}
|
}
|
||||||
|
|
||||||
columns_to_remove_after_prewhere = std::move(columns_to_remove);
|
columns_to_remove_after_prewhere = std::move(columns_to_remove);
|
||||||
@ -1635,10 +1633,13 @@ void ExpressionAnalysisResult::finalize(const ExpressionActionsChain & chain, si
|
|||||||
else if (hasFilter())
|
else if (hasFilter())
|
||||||
{
|
{
|
||||||
/// Can't have prewhere and filter set simultaneously
|
/// Can't have prewhere and filter set simultaneously
|
||||||
filter_info->do_remove_column = chain.steps.at(0)->can_remove_required_output.at(0);
|
filter_info->do_remove_column = chain.steps.at(0)->required_output.find(filter_info->column_name)->second;
|
||||||
}
|
}
|
||||||
if (hasWhere())
|
if (hasWhere())
|
||||||
remove_where_filter = chain.steps.at(where_step_num)->can_remove_required_output.at(0);
|
{
|
||||||
|
auto where_column_name = query.where()->getColumnName();
|
||||||
|
remove_where_filter = chain.steps.at(where_step_num)->required_output.find(where_column_name)->second;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void ExpressionAnalysisResult::removeExtraColumns() const
|
void ExpressionAnalysisResult::removeExtraColumns() const
|
||||||
|
@ -245,7 +245,7 @@ struct ExpressionAnalysisResult
|
|||||||
|
|
||||||
void removeExtraColumns() const;
|
void removeExtraColumns() const;
|
||||||
void checkActions() const;
|
void checkActions() const;
|
||||||
void finalize(const ExpressionActionsChain & chain, size_t where_step_num);
|
void finalize(const ExpressionActionsChain & chain, size_t where_step_num, const ASTSelectQuery & query);
|
||||||
};
|
};
|
||||||
|
|
||||||
/// SelectQuery specific ExpressionAnalyzer part.
|
/// SelectQuery specific ExpressionAnalyzer part.
|
||||||
|
@ -694,7 +694,8 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector<Stage> &
|
|||||||
|
|
||||||
/// Remove all intermediate columns.
|
/// Remove all intermediate columns.
|
||||||
actions_chain.addStep();
|
actions_chain.addStep();
|
||||||
actions_chain.getLastStep().required_output.assign(stage.output_columns.begin(), stage.output_columns.end());
|
for (const auto & name : stage.output_columns)
|
||||||
|
actions_chain.getLastStep().addRequiredOutput(name);
|
||||||
actions_chain.getLastActions();
|
actions_chain.getLastActions();
|
||||||
|
|
||||||
actions_chain.finalize();
|
actions_chain.finalize();
|
||||||
|
Loading…
Reference in New Issue
Block a user