thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-18 21:51:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing build and style.

Browse Source
This commit is contained in:
Nikolai Kochetov 2023-02-20 14:21:20 +00:00
parent 2a8460d962
commit 84e442e620
7 changed files with 46 additions and 301 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/Processors/QueryPlan/AggregatingStep.cpp
View File

@ -25,6 +25,11 @@
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
static bool memoryBoundMergingWillBeUsed(
bool should_produce_results_in_order_of_bucket_number,
bool memory_bound_merging_of_aggregation_results_enabled,

42
src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp
View File

@ -2,6 +2,7 @@
#include <Core/Field.h>
#include <Functions/IFunction.h>
#include <Columns/ColumnConst.h>
#include <stack>
@ -11,7 +12,7 @@ MatchedTrees::Matches matchTrees(const ActionsDAG & inner_dag, const ActionsDAG
{
using Parents = std::set<const ActionsDAG::Node *>;
std::unordered_map<const ActionsDAG::Node *, Parents> inner_parents;
std::unordered_map<std::string_view, const ActionsDAG::Node *> inner_inputs_and_constants;
std::unordered_map<std::string_view, const ActionsDAG::Node *> inner_inputs;
{
std::stack<const ActionsDAG::Node *> stack;
@ -27,8 +28,8 @@ MatchedTrees::Matches matchTrees(const ActionsDAG & inner_dag, const ActionsDAG
const auto * node = stack.top();
stack.pop();
if (node->type == ActionsDAG::ActionType::INPUT || node->type == ActionsDAG::ActionType::COLUMN)
inner_inputs_and_constants.emplace(node->result_name, node);
if (node->type == ActionsDAG::ActionType::INPUT)
inner_inputs.emplace(node->result_name, node);
for (const auto * child : node->children)
{
@ -84,10 +85,10 @@ MatchedTrees::Matches matchTrees(const ActionsDAG & inner_dag, const ActionsDAG
/// match.node will be set if match is found.
auto & match = matches[frame.node];
if (frame.node->type == ActionsDAG::ActionType::INPUT || frame.node->type == ActionsDAG::ActionType::COLUMN)
if (frame.node->type == ActionsDAG::ActionType::INPUT)
{
const ActionsDAG::Node * mapped = nullptr;
if (auto it = inner_inputs_and_constants.find(frame.node->result_name); it != inner_inputs_and_constants.end())
if (auto it = inner_inputs.find(frame.node->result_name); it != inner_inputs.end())
mapped = it->second;
match.node = mapped;
@ -101,14 +102,20 @@ MatchedTrees::Matches matchTrees(const ActionsDAG & inner_dag, const ActionsDAG
//std::cerr << "... Processing " << frame.node->function_base->getName() << std::endl;
bool found_all_children = true;
for (const auto * child : frame.mapped_children)
if (!child)
const ActionsDAG::Node * any_child = nullptr;
size_t num_children = frame.node->children.size();
for (size_t i = 0; i < num_children; ++i)
{
if (frame.mapped_children[i])
any_child = frame.mapped_children[i];
else if (!frame.node->children[i]->column || !isColumnConst(*frame.node->children[i]->column))
found_all_children = false;
}
if (found_all_children && !frame.mapped_children.empty())
if (found_all_children && any_child)
{
Parents container;
Parents * intersection = &inner_parents[frame.mapped_children[0]];
Parents * intersection = &inner_parents[any_child];
if (frame.mapped_children.size() > 1)
{
@ -116,7 +123,8 @@ MatchedTrees::Matches matchTrees(const ActionsDAG & inner_dag, const ActionsDAG
size_t mapped_children_size = frame.mapped_children.size();
other_parents.reserve(mapped_children_size);
for (size_t i = 1; i < mapped_children_size; ++i)
other_parents.push_back(&inner_parents[frame.mapped_children[i]]);
if (frame.mapped_children[i])
other_parents.push_back(&inner_parents[frame.mapped_children[i]]);
for (const auto * parent : *intersection)
{
@ -148,12 +156,19 @@ MatchedTrees::Matches matchTrees(const ActionsDAG & inner_dag, const ActionsDAG
if (parent->type == ActionsDAG::ActionType::FUNCTION && func_name == parent->function_base->getName())
{
const auto & children = parent->children;
size_t num_children = children.size();
if (frame.mapped_children.size() == num_children)
if (children.size() == num_children)
{
bool all_children_matched = true;
for (size_t i = 0; all_children_matched && i < num_children; ++i)
all_children_matched = frame.mapped_children[i] == children[i];
{
if (frame.mapped_children[i] == nullptr)
{
all_children_matched = children[i]->column && isColumnConst(*children[i]->column)
&& assert_cast<const ColumnConst &>(*children[i]->column).getField() == assert_cast<const ColumnConst &>(*frame.node->children[i]->column).getField();
}
else
all_children_matched = frame.mapped_children[i] == children[i];
}
if (all_children_matched)
{
@ -212,5 +227,4 @@ MatchedTrees::Matches matchTrees(const ActionsDAG & inner_dag, const ActionsDAG
return matches;
}
}

230
src/Processors/QueryPlan/Optimizations/matchTrees.cpp
View File

@ -1,230 +0,0 @@
#include <Processors/QueryPlan/Optimizations/matchTrees.h>
#include <Functions/IFunction.h>
#include <Core/Field.h>
#include <Columns/ColumnConst.h>
#include <stack>
namespace DB::QueryPlanOptimizations
{
MatchedTrees::Matches matchTrees(const ActionsDAG & inner_dag, const ActionsDAG & outer_dag)
{
using Parents = std::set<const ActionsDAG::Node *>;
std::unordered_map<const ActionsDAG::Node *, Parents> inner_parents;
std::unordered_map<std::string_view, const ActionsDAG::Node *> inner_inputs;
{
std::stack<const ActionsDAG::Node *> stack;
for (const auto * out : inner_dag.getOutputs())
{
if (inner_parents.contains(out))
continue;
stack.push(out);
inner_parents.emplace(out, Parents());
while (!stack.empty())
{
const auto * node = stack.top();
stack.pop();
if (node->type == ActionsDAG::ActionType::INPUT)
inner_inputs.emplace(node->result_name, node);
for (const auto * child : node->children)
{
auto [it, inserted] = inner_parents.emplace(child, Parents());
it->second.emplace(node);
if (inserted)
stack.push(child);
}
}
}
}
struct Frame
{
const ActionsDAG::Node * node;
ActionsDAG::NodeRawConstPtrs mapped_children;
};
MatchedTrees::Matches matches;
std::stack<Frame> stack;
for (const auto & node : outer_dag.getNodes())
{
if (matches.contains(&node))
continue;
stack.push(Frame{&node, {}});
while (!stack.empty())
{
auto & frame = stack.top();
frame.mapped_children.reserve(frame.node->children.size());
while (frame.mapped_children.size() < frame.node->children.size())
{
const auto * child = frame.node->children[frame.mapped_children.size()];
auto it = matches.find(child);
if (it == matches.end())
{
/// If match map does not contain a child, it was not visited.
stack.push(Frame{child, {}});
break;
}
/// A node from found match may be nullptr.
/// It means that node is visited, but no match was found.
frame.mapped_children.push_back(it->second.node);
}
if (frame.mapped_children.size() < frame.node->children.size())
continue;
/// Create an empty match for current node.
/// natch.node will be set if match is found.
auto & match = matches[frame.node];
if (frame.node->type == ActionsDAG::ActionType::INPUT)
{
const ActionsDAG::Node * mapped = nullptr;
if (auto it = inner_inputs.find(frame.node->result_name); it != inner_inputs.end())
mapped = it->second;
match.node = mapped;
}
else if (frame.node->type == ActionsDAG::ActionType::ALIAS)
{
match = matches[frame.node->children.at(0)];
}
else if (frame.node->type == ActionsDAG::ActionType::FUNCTION)
{
//std::cerr << "... Processing " << frame.node->function_base->getName() << std::endl;
bool found_all_children = true;
const ActionsDAG::Node * any_child = nullptr;
size_t num_children = frame.node->children.size();
for (size_t i = 0; i < num_children; ++i)
{
if (frame.mapped_children[i])
any_child = frame.mapped_children[i];
else if (!frame.node->children[i]->column || !isColumnConst(*frame.node->children[i]->column))
found_all_children = false;
}
if (found_all_children && any_child)
{
Parents container;
Parents * intersection = &inner_parents[any_child];
if (frame.mapped_children.size() > 1)
{
std::vector<Parents *> other_parents;
size_t mapped_children_size = frame.mapped_children.size();
other_parents.reserve(mapped_children_size);
for (size_t i = 1; i < mapped_children_size; ++i)
if (frame.mapped_children[i])
other_parents.push_back(&inner_parents[frame.mapped_children[i]]);
for (const auto * parent : *intersection)
{
bool is_common = true;
for (const auto * set : other_parents)
{
if (!set->contains(parent))
{
is_common = false;
break;
}
}
if (is_common)
container.insert(parent);
}
intersection = &container;
}
//std::cerr << ".. Candidate parents " << intersection->size() << std::endl;
if (!intersection->empty())
{
auto func_name = frame.node->function_base->getName();
for (const auto * parent : *intersection)
{
//std::cerr << ".. candidate " << parent->result_name << std::endl;
if (parent->type == ActionsDAG::ActionType::FUNCTION && func_name == parent->function_base->getName())
{
const auto & children = parent->children;
if (children.size() == num_children)
{
bool all_children_matched = true;
for (size_t i = 0; all_children_matched && i < num_children; ++i)
{
if (frame.mapped_children[i] == nullptr)
{
all_children_matched = children[i]->column && isColumnConst(*children[i]->column)
&& assert_cast<const ColumnConst &>(*children[i]->column).getField() == assert_cast<const ColumnConst &>(*frame.node->children[i]->column).getField();
}
else
all_children_matched = frame.mapped_children[i] == children[i];
}
if (all_children_matched)
{
match.node = parent;
break;
}
}
}
}
}
}
if (!match.node && frame.node->function_base->hasInformationAboutMonotonicity())
{
size_t num_const_args = 0;
const ActionsDAG::Node * monotonic_child = nullptr;
for (const auto * child : frame.node->children)
{
if (child->column)
++num_const_args;
else
monotonic_child = child;
}
if (monotonic_child && num_const_args + 1 == frame.node->children.size())
{
const auto & child_match = matches[monotonic_child];
if (child_match.node)
{
auto info = frame.node->function_base->getMonotonicityForRange(*monotonic_child->result_type, {}, {});
if (info.is_monotonic)
{
MatchedTrees::Monotonicity monotonicity;
monotonicity.direction *= info.is_positive ? 1 : -1;
monotonicity.strict = info.is_strict;
if (child_match.monotonicity)
{
monotonicity.direction *= child_match.monotonicity->direction;
if (!child_match.monotonicity->strict)
monotonicity.strict = false;
}
match.node = child_match.node;
match.monotonicity = monotonicity;
}
}
}
}
}
stack.pop();
}
}
return matches;
}
}

43
src/Processors/QueryPlan/Optimizations/matchTrees.h
View File

@ -1,43 +0,0 @@
#include <Interpreters/ActionsDAG.h>
namespace DB::QueryPlanOptimizations
{
/// This structure stores a node mapping from one DAG to another.
/// The rule is following:
/// * Input nodes are mapped by name.
/// * Function is mapped to function if all children are mapped and function names are same.
/// * Alias is mapped to it's children mapping.
/// * Monotonic function can be mapped to it's children mapping if direct mapping does not exist.
/// In this case, information about monotonicity is filled.
/// * Mapped node is nullptr if there is no mapping found.
///
/// Overall, directly mapped nodes represent equal calculations.
/// Notes:
/// * Mapped DAG can contain many nodes which represent the same calculation.
/// In this case mapping is ambiguous and only one node is mapped.
/// * Aliases for mapped DAG are not supported.
/// DAG for PK does not contain aliases and ambiguous nodes.
struct MatchedTrees
{
/// Monotonicity is calculated for monotonic functions chain.
/// Chain is not strict if there is any non-strict monotonic function.
struct Monotonicity
{
int direction = 1;
bool strict = true;
};
struct Match
{
const ActionsDAG::Node * node = nullptr;
std::optional<Monotonicity> monotonicity;
};
using Matches = std::unordered_map<const ActionsDAG::Node *, Match>;
};
MatchedTrees::Matches matchTrees(const ActionsDAG & inner_dag, const ActionsDAG & outer_dag);
}

4
src/Processors/QueryPlan/Optimizations/optimizeTree.cpp
View File

@ -159,8 +159,8 @@ void optimizeTreeSecondPass(const QueryPlanOptimizationSettings & optimization_s
if (optimization_settings.force_use_projection && has_reading_from_mt && num_applied_projection == 0)
throw Exception(
"No projection is used when allow_experimental_projection_optimization = 1 and force_optimize_projection = 1",
ErrorCodes::PROJECTION_NOT_USED);
ErrorCodes::PROJECTION_NOT_USED,
"No projection is used when allow_experimental_projection_optimization = 1 and force_optimize_projection = 1");
}
}

22
src/Processors/QueryPlan/Optimizations/optimizeUseProjections.cpp
View File

@ -1,5 +1,5 @@
#include <Processors/QueryPlan/Optimizations/Optimizations.h>
#include <Processors/QueryPlan/Optimizations/matchTrees.h>
#include <Processors/QueryPlan/Optimizations/actionsDAGUtils.h>
#include <Processors/QueryPlan/ITransformingStep.h>
#include <Processors/QueryPlan/AggregatingStep.h>
#include <Processors/QueryPlan/ExpressionStep.h>
@ -52,10 +52,10 @@ static bool buildAggregatingDAG(QueryPlan::Node & node, ActionsDAGPtr & dag, Act
IQueryPlanStep * step = node.step.get();
if (auto * reading = typeid_cast<ReadFromMergeTree *>(step))
{
std::cerr << "============ Found ReadFromMergeTreen";
//std::cerr << "============ Found ReadFromMergeTreen";
if (const auto * prewhere_info = reading->getPrewhereInfo())
{
std::cerr << "============ Found prewhere info\n";
//std::cerr << "============ Found prewhere info\n";
if (prewhere_info->row_level_filter)
{
appendExpression(dag, prewhere_info->row_level_filter);
@ -67,9 +67,9 @@ static bool buildAggregatingDAG(QueryPlan::Node & node, ActionsDAGPtr & dag, Act
if (prewhere_info->prewhere_actions)
{
std::cerr << "============ Found prewhere actions\n";
//std::cerr << "============ Found prewhere actions\n";
appendExpression(dag, prewhere_info->prewhere_actions);
std::cerr << "============ Cur dag \n" << dag->dumpDAG();
//std::cerr << "============ Cur dag \n" << dag->dumpDAG();
need_remove_column = prewhere_info->remove_prewhere_column;
if (const auto * filter_node = dag->tryFindInOutputs(prewhere_info->prewhere_column_name))
filter_nodes.push_back(filter_node);
@ -93,7 +93,7 @@ static bool buildAggregatingDAG(QueryPlan::Node & node, ActionsDAGPtr & dag, Act
return false;
appendExpression(dag, actions);
std::cerr << "============ Cur e dag \n" << dag->dumpDAG();
//std::cerr << "============ Cur e dag \n" << dag->dumpDAG();
need_remove_column = false;
return true;
}
@ -105,7 +105,7 @@ static bool buildAggregatingDAG(QueryPlan::Node & node, ActionsDAGPtr & dag, Act
return false;
appendExpression(dag, actions);
std::cerr << "============ Cur f dag \n" << dag->dumpDAG();
//std::cerr << "============ Cur f dag \n" << dag->dumpDAG();
need_remove_column = filter->removesFilterColumn();
const auto * filter_expression = dag->tryFindInOutputs(filter->getFilterColumnName());
if (!filter_expression)
@ -282,8 +282,8 @@ ActionsDAGPtr analyzeAggregateProjection(
const auto & candidate = info.aggregates[idx];
/// Note: this check is a bit strict.
/// We check that aggregate function names, arguemnt types and parameters are equal.
/// In some cases it's possilbe only to check that states are equal,
/// We check that aggregate function names, argument types and parameters are equal.
/// In some cases it's possible only to check that states are equal,
/// e.g. for quantile(0.3)(...) and quantile(0.5)(...).
/// But also functions sum(...) and sumIf(...) will have equal states,
/// and we can't replace one to another from projection.
@ -709,7 +709,7 @@ bool optimizeUseAggProjections(QueryPlan::Node & node, QueryPlan::Nodes & nodes)
storage_snapshot->storage, storage_snapshot->metadata, storage_snapshot->object_columns); //, storage_snapshot->data);
proj_snapshot->addProjection(best_candidate->projection);
LOG_TRACE(&Poco::Logger::get("optimizeUseProjections"), "Proj snapshot {}", proj_snapshot->getColumns(GetColumnsOptions::Kind::All).toString());
LOG_TRACE(&Poco::Logger::get("optimizeUseProjections"), "Proj snapshot {}", proj_snapshot->getColumns(GetColumnsOptions::Kind::All).toString());
projection_reading = reader.readFromParts(
{},
@ -966,7 +966,7 @@ bool optimizeUseNormalProjections(Stack & stack, QueryPlan::Nodes & nodes)
storage_snapshot->storage, storage_snapshot->metadata, storage_snapshot->object_columns); //, storage_snapshot->data);
proj_snapshot->addProjection(best_candidate->projection);
LOG_TRACE(&Poco::Logger::get("optimizeUseProjections"), "Proj snapshot {}", proj_snapshot->getColumns(GetColumnsOptions::Kind::All).toString());
LOG_TRACE(&Poco::Logger::get("optimizeUseProjections"), "Proj snapshot {}", proj_snapshot->getColumns(GetColumnsOptions::Kind::All).toString());
auto query_info_copy = query_info;
query_info_copy.prewhere_info = nullptr;

1
src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
View File

@ -56,7 +56,6 @@ namespace ErrorCodes
extern const int ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER;
extern const int ILLEGAL_COLUMN;
extern const int ARGUMENT_OUT_OF_BOUND;
extern const int TOO_MANY_ROWS;
extern const int CANNOT_PARSE_TEXT;
extern const int TOO_MANY_PARTITIONS;
extern const int DUPLICATED_PART_UUIDS;