diff --git a/src/Interpreters/AddIndexConstraintsOptimizer.cpp b/src/Interpreters/AddIndexConstraintsOptimizer.cpp index 63a24eba094..06a9e59d297 100644 --- a/src/Interpreters/AddIndexConstraintsOptimizer.cpp +++ b/src/Interpreters/AddIndexConstraintsOptimizer.cpp @@ -12,10 +12,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} AddIndexConstraintsOptimizer::AddIndexConstraintsOptimizer( const StorageMetadataPtr & metadata_snapshot_) @@ -124,8 +120,10 @@ namespace { Poco::Logger::get("INDEX_HINT_CREATE").information("CHECK"); const auto * func = atom.ast->as(); - if (func && func->arguments->children.size() == 2 && getRelationMap().contains(func->name)) { - auto check_and_insert = [&](const size_t index, const ComparisonGraph::CompareResult need_result) -> bool { + if (func && func->arguments->children.size() == 2 && getRelationMap().contains(func->name)) + { + auto check_and_insert = [&](const size_t index, const ComparisonGraph::CompareResult need_result) -> bool + { if (!onlyConstants(func->arguments->children[1 - index])) return false; @@ -167,13 +165,14 @@ void AddIndexConstraintsOptimizer::perform(CNFQuery & cnf_query) const std::unordered_set primary_key_set(std::begin(primary_key), std::end(primary_key)); ASTs primary_key_only_asts; - for (const auto & vertex : graph.getVertexes()) + for (const auto & vertex : graph.getVertices()) for (const auto & ast : vertex) if (hasIndexColumns(ast, primary_key_set) && onlyIndexColumns(ast, primary_key_set)) primary_key_only_asts.push_back(ast); CNFQuery::AndGroup and_group; - cnf_query.iterateGroups([&and_group, &graph, &primary_key_only_asts](const auto & or_group) { + cnf_query.iterateGroups([&and_group, &graph, &primary_key_only_asts](const auto & or_group) + { auto add_group = createIndexHintGroup(or_group, graph, primary_key_only_asts); if (!add_group.empty()) and_group.emplace(std::move(add_group)); diff --git a/src/Interpreters/ComparisonGraph.cpp b/src/Interpreters/ComparisonGraph.cpp index 2e5703dd58f..83a887ce2bc 100644 --- a/src/Interpreters/ComparisonGraph.cpp +++ b/src/Interpreters/ComparisonGraph.cpp @@ -36,25 +36,29 @@ ASTPtr ComparisonGraph::normalizeAtom(const ASTPtr & atom) const ComparisonGraph::ComparisonGraph(const std::vector & atomic_formulas) { - static const std::map relation_to_enum = { + static const std::unordered_map relation_to_enum = + { {"equals", Edge::Type::EQUAL}, {"less", Edge::Type::LESS}, {"lessOrEquals", Edge::Type::LESS_OR_EQUAL}, }; Graph g; - for (const auto & atom_raw : atomic_formulas) { + for (const auto & atom_raw : atomic_formulas) + { const auto atom = normalizeAtom(atom_raw); const auto bad_term = std::numeric_limits::max(); - auto get_index = [](const ASTPtr & ast, Graph & asts_graph) -> std::size_t { + auto get_index = [](const ASTPtr & ast, Graph & asts_graph) -> std::size_t + { const auto it = asts_graph.ast_hash_to_component.find(ast->getTreeHash()); if (it != std::end(asts_graph.ast_hash_to_component)) { if (!std::any_of( - std::cbegin(asts_graph.vertexes[it->second].asts), - std::cend(asts_graph.vertexes[it->second].asts), - [ast](const ASTPtr & constraint_ast) { + std::cbegin(asts_graph.vertices[it->second].asts), + std::cend(asts_graph.vertices[it->second].asts), + [ast](const ASTPtr & constraint_ast) + { return constraint_ast->getTreeHash() == ast->getTreeHash() && constraint_ast->getColumnName() == ast->getColumnName(); })) @@ -66,10 +70,10 @@ ComparisonGraph::ComparisonGraph(const std::vector & atomic_formulas) } else { - asts_graph.ast_hash_to_component[ast->getTreeHash()] = asts_graph.vertexes.size(); - asts_graph.vertexes.push_back(EqualComponent{{ast}}); + asts_graph.ast_hash_to_component[ast->getTreeHash()] = asts_graph.vertices.size(); + asts_graph.vertices.push_back(EqualComponent{{ast}}); asts_graph.edges.emplace_back(); - return asts_graph.vertexes.size() - 1; + return asts_graph.vertices.size() - 1; } }; @@ -263,15 +267,18 @@ std::optional ComparisonGraph::getComponentId(const ASTPtr & ast) c return {}; const size_t index = hash_it->second; if (std::any_of( - std::cbegin(graph.vertexes[index].asts), - std::cend(graph.vertexes[index].asts), + std::cbegin(graph.vertices[index].asts), + std::cend(graph.vertices[index].asts), [ast](const ASTPtr & constraint_ast) { return constraint_ast->getTreeHash() == ast->getTreeHash() && constraint_ast->getColumnName() == ast->getColumnName(); - })) { + })) + { return index; - } else { + } + else + { return {}; } } @@ -283,18 +290,21 @@ bool ComparisonGraph::hasPath(const size_t left, const size_t right) const std::vector ComparisonGraph::getComponent(const std::size_t id) const { - return graph.vertexes[id].asts; + return graph.vertices[id].asts; } -bool ComparisonGraph::EqualComponent::hasConstant() const { +bool ComparisonGraph::EqualComponent::hasConstant() const +{ return constant_index != -1; } -ASTPtr ComparisonGraph::EqualComponent::getConstant() const { +ASTPtr ComparisonGraph::EqualComponent::getConstant() const +{ return asts[constant_index]; } -void ComparisonGraph::EqualComponent::buildConstants() { +void ComparisonGraph::EqualComponent::buildConstants() +{ constant_index = -1; for (size_t i = 0; i < asts.size(); ++i) { @@ -308,7 +318,8 @@ void ComparisonGraph::EqualComponent::buildConstants() { ComparisonGraph::CompareResult ComparisonGraph::getCompareResult(const std::string & name) { - static const std::unordered_map relation_to_compare = { + static const std::unordered_map relation_to_compare = + { {"equals", CompareResult::EQUAL}, {"notEquals", CompareResult::NOT_EQUAL}, {"less", CompareResult::LESS}, @@ -323,7 +334,8 @@ ComparisonGraph::CompareResult ComparisonGraph::getCompareResult(const std::stri ComparisonGraph::CompareResult ComparisonGraph::inverseCompareResult(const CompareResult result) { - static const std::unordered_map inverse_relations = { + static const std::unordered_map inverse_relations = + { {CompareResult::NOT_EQUAL, CompareResult::EQUAL}, {CompareResult::EQUAL, CompareResult::NOT_EQUAL}, {CompareResult::GREATER_OR_EQUAL, CompareResult::LESS}, @@ -341,8 +353,8 @@ std::optional ComparisonGraph::getEqualConst(const ASTPtr & ast) const if (hash_it == std::end(graph.ast_hash_to_component)) return std::nullopt; const size_t index = hash_it->second; - return graph.vertexes[index].hasConstant() - ? std::optional{graph.vertexes[index].getConstant()} + return graph.vertices[index].hasConstant() + ? std::optional{graph.vertices[index].getConstant()} : std::nullopt; } @@ -360,7 +372,7 @@ std::optional> ComparisonGraph::getConstUpperBound(const const ssize_t from = ast_const_upper_bound[to]; if (from == -1) return std::nullopt; - return std::make_pair(graph.vertexes[from].getConstant()->as()->value, dists.at({from, to}) == Path::LESS); + return std::make_pair(graph.vertices[from].getConstant()->as()->value, dists.at({from, to}) == Path::LESS); } std::optional> ComparisonGraph::getConstLowerBound(const ASTPtr & ast) const @@ -377,7 +389,7 @@ std::optional> ComparisonGraph::getConstLowerBound(const const ssize_t to = ast_const_lower_bound[from]; if (to == -1) return std::nullopt; - return std::make_pair(graph.vertexes[to].getConstant()->as()->value, dists.at({from, to}) == Path::LESS); + return std::make_pair(graph.vertices[to].getConstant()->as()->value, dists.at({from, to}) == Path::LESS); } void ComparisonGraph::dfsOrder(const Graph & asts_graph, size_t v, std::vector & visited, std::vector & order) const @@ -397,9 +409,9 @@ ComparisonGraph::Graph ComparisonGraph::reverseGraph(const Graph & asts_graph) c { Graph g; g.ast_hash_to_component = asts_graph.ast_hash_to_component; - g.vertexes = asts_graph.vertexes; - g.edges.resize(g.vertexes.size()); - for (size_t v = 0; v < asts_graph.vertexes.size(); ++v) + g.vertices = asts_graph.vertices; + g.edges.resize(g.vertices.size()); + for (size_t v = 0; v < asts_graph.vertices.size(); ++v) { for (const auto & edge : asts_graph.edges[v]) { @@ -409,10 +421,10 @@ ComparisonGraph::Graph ComparisonGraph::reverseGraph(const Graph & asts_graph) c return asts_graph; } -std::vector ComparisonGraph::getVertexes() const +std::vector ComparisonGraph::getVertices() const { std::vector result; - for (const auto & vertex : graph.vertexes) + for (const auto & vertex : graph.vertices) { result.emplace_back(); for (const auto & ast : vertex.asts) @@ -438,7 +450,7 @@ ComparisonGraph::Graph ComparisonGraph::BuildGraphFromAstsGraph(const Graph & as { Poco::Logger::get("Graph").information("building"); /// Find strongly connected component - const auto n = asts_graph.vertexes.size(); + const auto n = asts_graph.vertices.size(); std::vector order; { @@ -466,19 +478,19 @@ ComparisonGraph::Graph ComparisonGraph::BuildGraphFromAstsGraph(const Graph & as } Graph result; - result.vertexes.resize(component); + result.vertices.resize(component); result.edges.resize(component); for (const auto & [hash, index] : asts_graph.ast_hash_to_component) { result.ast_hash_to_component[hash] = components[index]; - result.vertexes[components[index]].asts.insert( - std::end(result.vertexes[components[index]].asts), - std::begin(asts_graph.vertexes[index].asts), - std::end(asts_graph.vertexes[index].asts)); // asts_graph has only one ast per vertex + result.vertices[components[index]].asts.insert( + std::end(result.vertices[components[index]].asts), + std::begin(asts_graph.vertices[index].asts), + std::end(asts_graph.vertices[index].asts)); // asts_graph has only one ast per vertex } /// Calculate constants - for (auto & vertex : result.vertexes) + for (auto & vertex : result.vertices) { vertex.buildConstants(); } @@ -494,16 +506,16 @@ ComparisonGraph::Graph ComparisonGraph::BuildGraphFromAstsGraph(const Graph & as // TODO: make edges unique (left most strict) } - for (size_t v = 0; v < result.vertexes.size(); ++v) + for (size_t v = 0; v < result.vertices.size(); ++v) { - for (size_t u = 0; u < result.vertexes.size(); ++u) + for (size_t u = 0; u < result.vertices.size(); ++u) { if (v == u) continue; - if (result.vertexes[v].hasConstant() && result.vertexes[u].hasConstant()) + if (result.vertices[v].hasConstant() && result.vertices[u].hasConstant()) { - const auto * left = result.vertexes[v].getConstant()->as(); - const auto * right = result.vertexes[u].getConstant()->as(); + const auto * left = result.vertices[v].getConstant()->as(); + const auto * right = result.vertices[u].getConstant()->as(); /// Only less. Equal constant fields = equal literals so it was already considered above. if (left->value > right->value) @@ -521,7 +533,7 @@ std::map, ComparisonGraph::Path> ComparisonGraph::Buil { // min path : < = -1, =< = 0 const auto inf = std::numeric_limits::max(); - const size_t n = graph.vertexes.size(); + const size_t n = graph.vertices.size(); std::vector> results(n, std::vector(n, inf)); for (size_t v = 0; v < n; ++v) { @@ -547,24 +559,27 @@ std::map, ComparisonGraph::Path> ComparisonGraph::Buil std::pair, std::vector> ComparisonGraph::buildConstBounds() const { - const size_t n = graph.vertexes.size(); + const size_t n = graph.vertices.size(); std::vector lower(n, -1); std::vector upper(n, -1); - auto get_value = [this](const size_t vertex) -> Field { - return graph.vertexes[vertex].getConstant()->as()->value; + auto get_value = [this](const size_t vertex) -> Field + { + return graph.vertices[vertex].getConstant()->as()->value; }; for (const auto & [edge, path] : dists) { const auto [from, to] = edge; - if (graph.vertexes[to].hasConstant()) { + if (graph.vertices[to].hasConstant()) + { if (lower[from] == -1 || get_value(lower[from]) > get_value(to) || (get_value(lower[from]) >= get_value(to) && dists.at({from, to}) == Path::LESS)) lower[from] = to; } - if (graph.vertexes[from].hasConstant()) { + if (graph.vertices[from].hasConstant()) + { if (upper[to] == -1 || get_value(upper[to]) < get_value(from) || (get_value(upper[to]) <= get_value(from) && dists.at({from, to}) == Path::LESS)) diff --git a/src/Interpreters/ComparisonGraph.h b/src/Interpreters/ComparisonGraph.h index 101294d3e6b..55d0e3c6d27 100644 --- a/src/Interpreters/ComparisonGraph.h +++ b/src/Interpreters/ComparisonGraph.h @@ -53,7 +53,7 @@ public: std::optional> getConstUpperBound(const ASTPtr & ast) const; std::optional> getConstLowerBound(const ASTPtr & ast) const; - std::vector getVertexes() const; + std::vector getVertices() const; private: /// strongly connected component @@ -86,14 +86,16 @@ private: struct Graph { - struct ASTHash { - size_t operator() (const IAST::Hash & hash) const { + struct ASTHash + { + size_t operator() (const IAST::Hash & hash) const + { return hash.first; } }; std::unordered_map ast_hash_to_component; - std::vector vertexes; + std::vector vertices; std::vector> edges; }; diff --git a/src/Interpreters/ConstraintMatcherVisitor.cpp b/src/Interpreters/ConstraintMatcherVisitor.cpp index 61b1c436b3a..b3050ea9eed 100644 --- a/src/Interpreters/ConstraintMatcherVisitor.cpp +++ b/src/Interpreters/ConstraintMatcherVisitor.cpp @@ -1 +1,38 @@ #include "ConstraintMatcherVisitor.h" + +#include +#include +#include +#include + +namespace DB +{ + +bool ConstraintMatcher::needChildVisit(const ASTPtr & node, const ASTPtr &) +{ + return node->as() || node->as(); +} + +std::optional ConstraintMatcher::getASTValue(const ASTPtr & node, Data & data) +{ + const auto it = data.constraints.find(node->getTreeHash().second); + if (it != std::end(data.constraints)) + { + for (const auto & ast : it->second) + { + if (node->getColumnName() == ast->getColumnName()) + { + return true; + } + } + } + return std::nullopt; +} + +void ConstraintMatcher::visit(ASTPtr & ast, Data & data) +{ + if (const auto always_value = getASTValue(ast, data); always_value) + ast = std::make_shared(static_cast(*always_value)); +} + +} diff --git a/src/Interpreters/ConstraintMatcherVisitor.h b/src/Interpreters/ConstraintMatcherVisitor.h index 24f3e7037e5..be0abe91fec 100644 --- a/src/Interpreters/ConstraintMatcherVisitor.h +++ b/src/Interpreters/ConstraintMatcherVisitor.h @@ -1,13 +1,6 @@ #pragma once #include -#include -#include -#include -#include -#include -#include -#include namespace DB { @@ -21,25 +14,9 @@ struct ConstraintMatcher using Visitor = InDepthNodeVisitor; - static bool needChildVisit(const ASTPtr & node, const ASTPtr &) { return node->as() || node->as(); } - - static std::optional getASTValue(const ASTPtr & node, Data & data) { - const auto it = data.constraints.find(node->getTreeHash().second); - if (it != std::end(data.constraints)) { - for (const auto & ast : it->second) { - if (node->getColumnName() == ast->getColumnName()) { - return true; - } - } - } - return std::nullopt; - } - - static void visit(ASTPtr & ast, Data & data) - { - if (const auto always_value = getASTValue(ast, data); always_value) - ast = std::make_shared(static_cast(*always_value)); - } + static bool needChildVisit(const ASTPtr & node, const ASTPtr &); + static std::optional getASTValue(const ASTPtr & node, Data & data); + static void visit(ASTPtr & ast, Data & data); }; using ConstraintMatcherVisitor = InDepthNodeVisitor; diff --git a/src/Interpreters/TreeCNFConverter.cpp b/src/Interpreters/TreeCNFConverter.cpp index ad991b64299..fac3007f394 100644 --- a/src/Interpreters/TreeCNFConverter.cpp +++ b/src/Interpreters/TreeCNFConverter.cpp @@ -1,7 +1,6 @@ #include #include #include - #include namespace DB @@ -37,7 +36,8 @@ void traversePushNot(ASTPtr & node, bool add_negation) if (func && (func->name == "and" || func->name == "or")) { - if (add_negation) { + if (add_negation) + { /// apply De Morgan's Law node = makeASTFunction( (func->name == "and" ? "or" : "and"), @@ -302,7 +302,7 @@ CNFQuery & CNFQuery::pushNotInFuntions() std::string CNFQuery::dump() const { - std::stringstream res; + WriteBufferFromOwnString res; bool first = true; for (const auto & group : statements) { diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index 92ca45259eb..6637792f8da 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -511,23 +511,23 @@ void optimizeLimitBy(const ASTSelectQuery * select_query) } /// Use constraints to get rid of useless parts of query -void optimizeWithConstraints(ASTSelectQuery * select_query, Aliases & aliases, const NameSet & source_columns_set, - const std::vector & tables_with_columns, +void optimizeWithConstraints(ASTSelectQuery * select_query, Aliases & /*aliases*/, const NameSet & /*source_columns_set*/, + const std::vector & /*tables_with_columns*/, const StorageMetadataPtr & metadata_snapshot) { - WhereConstraintsOptimizer(select_query, aliases, source_columns_set, tables_with_columns, metadata_snapshot).perform(); + WhereConstraintsOptimizer(select_query, metadata_snapshot).perform(); if (select_query->where()) Poco::Logger::get("CNF").information(select_query->where()->dumpTree()); else Poco::Logger::get("CNF").information("NO WHERE"); } -void optimizeSubstituteColumn(ASTSelectQuery * select_query, Aliases & aliases, const NameSet & source_columns_set, - const std::vector & tables_with_columns, +void optimizeSubstituteColumn(ASTSelectQuery * select_query, Aliases & /*aliases*/, const NameSet & /*source_columns_set*/, + const std::vector & /*tables_with_columns*/, const StorageMetadataPtr & metadata_snapshot, const ConstStoragePtr & storage) { - SubstituteColumnOptimizer(select_query, aliases, source_columns_set, tables_with_columns, metadata_snapshot, storage).perform(); + SubstituteColumnOptimizer(select_query, metadata_snapshot, storage).perform(); } /// transform where to CNF for more convenient optimization diff --git a/src/Interpreters/WhereConstraintsOptimizer.cpp b/src/Interpreters/WhereConstraintsOptimizer.cpp index a548eae2eed..bfdc88f34f2 100644 --- a/src/Interpreters/WhereConstraintsOptimizer.cpp +++ b/src/Interpreters/WhereConstraintsOptimizer.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -13,21 +12,10 @@ namespace DB { -namespace ErrorCodes -{ -extern const int LOGICAL_ERROR; -} - WhereConstraintsOptimizer::WhereConstraintsOptimizer( ASTSelectQuery * select_query_, - Aliases & /*aliases_*/, - const NameSet & /*source_columns_set_*/, - const std::vector & /*tables_with_columns_*/, const StorageMetadataPtr & metadata_snapshot_) : select_query(select_query_) - /* , aliases(aliases_) - , source_columns_set(source_columns_set_) - , tables_with_columns(tables_with_columns_)*/ , metadata_snapshot(metadata_snapshot_) { } @@ -181,13 +169,18 @@ void WhereConstraintsOptimizer::perform() auto cnf = TreeCNFConverter::toCNF(select_query->where()); Poco::Logger::get("BEFORE OPT").information(cnf.dump()); cnf.pullNotOutFunctions() - .filterAlwaysTrueGroups([&constraint_data, &compare_graph](const auto & group) { /// remove always true groups from CNF + .filterAlwaysTrueGroups([&constraint_data, &compare_graph](const auto & group) + { + /// remove always true groups from CNF return !checkIfGroupAlwaysTrueFullMatch(group, constraint_data) && !checkIfGroupAlwaysTrueGraph(group, compare_graph); }) - .filterAlwaysFalseAtoms([&constraint_data, &compare_graph](const auto & atom) { /// remove always false atoms from CNF + .filterAlwaysFalseAtoms([&constraint_data, &compare_graph](const auto & atom) + { + /// remove always false atoms from CNF return !checkIfAtomAlwaysFalseFullMatch(atom, constraint_data) && !checkIfAtomAlwaysFalseGraph(atom, compare_graph); }) - .transformAtoms([&compare_graph](const auto & atom) { + .transformAtoms([&compare_graph](const auto & atom) + { return replaceTermsToConstants(atom, compare_graph); }) .pushNotInFuntions(); diff --git a/src/Interpreters/WhereConstraintsOptimizer.h b/src/Interpreters/WhereConstraintsOptimizer.h index fe13c943ef4..8a58aa41fa5 100644 --- a/src/Interpreters/WhereConstraintsOptimizer.h +++ b/src/Interpreters/WhereConstraintsOptimizer.h @@ -17,17 +17,14 @@ using StorageMetadataPtr = std::shared_ptr; class WhereConstraintsOptimizer final { public: - WhereConstraintsOptimizer(ASTSelectQuery * select_query, Aliases & /* aliases */, const NameSet & /* source_columns_set */, - const std::vector & /* tables_with_columns */, - const StorageMetadataPtr & metadata_snapshot); + WhereConstraintsOptimizer( + ASTSelectQuery * select_query, + const StorageMetadataPtr & metadata_snapshot); void perform(); private: ASTSelectQuery * select_query; - /*Aliases & aliases; - const NameSet & source_columns_set; - const std::vector & tables_with_columns;*/ const StorageMetadataPtr & metadata_snapshot; }; diff --git a/src/Parsers/ASTConstraintDeclaration.h b/src/Parsers/ASTConstraintDeclaration.h index bdba7bd577a..5db8df43a76 100644 --- a/src/Parsers/ASTConstraintDeclaration.h +++ b/src/Parsers/ASTConstraintDeclaration.h @@ -10,7 +10,8 @@ namespace DB class ASTConstraintDeclaration : public IAST { public: - enum class Type { + enum class Type + { CHECK, ASSUME, }; diff --git a/src/Storages/ConstraintsDescription.cpp b/src/Storages/ConstraintsDescription.cpp index 4830ad2cd9e..042d7a344e5 100644 --- a/src/Storages/ConstraintsDescription.cpp +++ b/src/Storages/ConstraintsDescription.cpp @@ -53,13 +53,15 @@ ASTs ConstraintsDescription::filterConstraints(ConstraintType selection) const case ASTConstraintDeclaration::Type::ASSUME: return static_cast(ConstraintType::ASSUME); } + throw Exception("Unknown constraint type.", ErrorCodes::LOGICAL_ERROR); }; ASTs res; res.reserve(constraints.size()); for (const auto & constraint : constraints) { - if ((ast_to_decr_constraint_type(constraint->as()->type) & static_cast(selection)) != 0) { + if ((ast_to_decr_constraint_type(constraint->as()->type) & static_cast(selection)) != 0) + { res.push_back(constraint); } } @@ -88,7 +90,8 @@ std::vector ConstraintsDescription::getAtomicConstraint Poco::Logger::get("atomic_formula: initial:").information(constraint->as()->expr->ptr()->dumpTree()); const auto cnf = TreeCNFConverter::toCNF(constraint->as()->expr->ptr()) .pullNotOutFunctions(); - for (const auto & group : cnf.getStatements()) { + for (const auto & group : cnf.getStatements()) + { if (group.size() == 1) constraint_data.push_back(*group.begin()); } diff --git a/src/Storages/ConstraintsDescription.h b/src/Storages/ConstraintsDescription.h index 8f8cb750c52..00b5f6b6e4b 100644 --- a/src/Storages/ConstraintsDescription.h +++ b/src/Storages/ConstraintsDescription.h @@ -20,7 +20,8 @@ public: static ConstraintsDescription parse(const String & str); - enum class ConstraintType { + enum class ConstraintType + { CHECK = 1, ASSUME = 2, ALWAYS_TRUE = CHECK | ASSUME, diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index ae692439038..a2dbc6fc47f 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -2008,8 +2008,13 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingMergedIndex( for (size_t index_mark = index_range.begin; index_mark < index_range.end; ++index_mark) { if (index_mark != index_range.begin || !granules_filled || last_index_mark != index_range.begin) + { for (size_t i = 0; i < readers.size(); ++i) + { granules[i] = readers[i].read(); + granules_filled = true; + } + } MarkRange data_range( std::max(range.begin, index_mark * index_granularity), diff --git a/src/Storages/MergeTree/MergeTreeIndexHypothesis.cpp b/src/Storages/MergeTree/MergeTreeIndexHypothesis.cpp index 263d9057159..57ef75824d4 100644 --- a/src/Storages/MergeTree/MergeTreeIndexHypothesis.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexHypothesis.cpp @@ -83,7 +83,7 @@ MergeTreeIndexAggregatorPtr MergeTreeIndexHypothesis::createIndexAggregator() co MergeTreeIndexConditionPtr MergeTreeIndexHypothesis::createIndexCondition( const SelectQueryInfo &, ContextPtr) const { - return nullptr; + throw Exception("Not supported", ErrorCodes::LOGICAL_ERROR); } bool MergeTreeIndexHypothesis::mayBenefitFromIndexForIn(const ASTPtr &) const diff --git a/src/Storages/MergeTree/MergeTreeIndexMergedCondition.cpp b/src/Storages/MergeTree/MergeTreeIndexMergedCondition.cpp index 8352e1c0ff4..82e745bce1a 100644 --- a/src/Storages/MergeTree/MergeTreeIndexMergedCondition.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexMergedCondition.cpp @@ -13,7 +13,6 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; - extern const int INCORRECT_QUERY; } MergeTreeIndexMergedCondition::MergeTreeIndexMergedCondition( @@ -103,7 +102,8 @@ ComparisonGraph::CompareResult getExpectedCompare(const CNFQuery::AtomicFormula } /// Replaces < -> <=, > -> >= and assumes that all hypotheses are true then checks if path exists -bool MergeTreeIndexMergedCondition::alwaysUnknownOrTrue() const { +bool MergeTreeIndexMergedCondition::alwaysUnknownOrTrue() const +{ std::vector active_atomic_formulas(atomic_constraints); for (size_t i = 0; i < index_to_compare_atomic_hypotheses.size(); ++i) { diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index d28e4ecae90..0734ec33f67 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -189,8 +189,8 @@ void MergeTreeWhereOptimizer::optimize(ASTSelectQuery & select) const if (select.where()) { - Poco::Logger::get("MTPRWHERE WHE").information(select.where()->getColumnName()); - Poco::Logger::get("MTPRWHERE WHE").information(select.where()->dumpTree()); + Poco::Logger::get("MTPRWHERE WHERE").information(select.where()->getColumnName()); + Poco::Logger::get("MTPRWHERE WHERE").information(select.where()->dumpTree()); } if(select.prewhere()) { diff --git a/src/Storages/MergeTree/SubstituteColumnOptimizer.cpp b/src/Storages/MergeTree/SubstituteColumnOptimizer.cpp index 8dcea9b852e..13535360fd8 100644 --- a/src/Storages/MergeTree/SubstituteColumnOptimizer.cpp +++ b/src/Storages/MergeTree/SubstituteColumnOptimizer.cpp @@ -13,10 +13,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} namespace { @@ -224,15 +220,9 @@ void bruteforce( SubstituteColumnOptimizer::SubstituteColumnOptimizer( ASTSelectQuery * select_query_, - Aliases & /*aliases_*/, - const NameSet & /*source_columns_set_*/, - const std::vector & /*tables_with_columns_*/, const StorageMetadataPtr & metadata_snapshot_, const ConstStoragePtr & storage_) : select_query(select_query_) - /* , aliases(aliases_) - , source_columns_set(source_columns_set_) - , tables_with_columns(tables_with_columns_)*/ , metadata_snapshot(metadata_snapshot_) , storage(storage_) { @@ -261,7 +251,8 @@ void SubstituteColumnOptimizer::perform() ast->setAlias(ast->getAliasOrColumnName()); } - auto run_for_all = [&](const auto func) { + auto run_for_all = [&](const auto func) + { if (select_query->where()) func(select_query->refWhere(), false); if (select_query->prewhere()) @@ -279,7 +270,8 @@ void SubstituteColumnOptimizer::perform() ComponentVisitor::Data component_data( compare_graph, components, old_name, name_to_component, counter_id); std::unordered_set identifiers; - auto preprocess = [&](ASTPtr & ast, bool) { + auto preprocess = [&](ASTPtr & ast, bool) + { ComponentVisitor(component_data).visit(ast); collectIdentifiers(ast, identifiers); }; @@ -320,7 +312,8 @@ void SubstituteColumnOptimizer::perform() for (size_t i = 0; i < min_expressions.size(); ++i) id_to_expression_map[components_list[i]] = min_expressions[i]; - auto process = [&](ASTPtr & ast, bool is_select) { + auto process = [&](ASTPtr & ast, bool is_select) + { SubstituteColumnVisitor::Data substitute_data{id_to_expression_map, name_to_component, old_name, is_select}; SubstituteColumnVisitor(substitute_data).visit(ast); }; diff --git a/src/Storages/MergeTree/SubstituteColumnOptimizer.h b/src/Storages/MergeTree/SubstituteColumnOptimizer.h index b7835385b45..6fc7c215afd 100644 --- a/src/Storages/MergeTree/SubstituteColumnOptimizer.h +++ b/src/Storages/MergeTree/SubstituteColumnOptimizer.h @@ -18,19 +18,14 @@ class SubstituteColumnOptimizer { public: SubstituteColumnOptimizer( - ASTSelectQuery * select_query, - Aliases & /* aliases */, const NameSet & /* source_columns_set */, - const std::vector & /* tables_with_columns */, - const StorageMetadataPtr & /* metadata_snapshot */, - const ConstStoragePtr & storage); + ASTSelectQuery * select_query, + const StorageMetadataPtr & metadata_snapshot, + const ConstStoragePtr & storage); void perform(); private: ASTSelectQuery * select_query; - /*Aliases & aliases; - const NameSet & source_columns_set; - const std::vector & tables_with_columns;*/ const StorageMetadataPtr & metadata_snapshot; ConstStoragePtr storage; }; diff --git a/tests/queries/0_stateless/01624_soft_constraints.sh b/tests/queries/0_stateless/01624_soft_constraints.sh index 45eb122d05b..3625d4f0e5f 100755 --- a/tests/queries/0_stateless/01624_soft_constraints.sh +++ b/tests/queries/0_stateless/01624_soft_constraints.sh @@ -6,9 +6,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -SETTINGS="SET convert_query_to_cnf = 1; SET optimize_using_constraints = 1; SET optimize_move_to_prewhere = 1;" +SETTINGS="SET convert_query_to_cnf = 1; SET optimize_using_constraints = 1; SET optimize_move_to_prewhere = 1" $CLICKHOUSE_CLIENT -n --query=" +$SETTINGS; DROP DATABASE IF EXISTS hypothesis_test; DROP TABLE IF EXISTS hypothesis_test.test; DROP TABLE IF EXISTS hypothesis_test.test2; @@ -25,30 +26,28 @@ CREATE TABLE hypothesis_test.test ( ) ENGINE = MergeTree() ORDER BY i SETTINGS index_granularity=1; " -$CLICKHOUSE_CLIENT --query="INSERT INTO hypothesis_test.test VALUES -(0, 1, 2, 2), -(1, 2, 1, 2), -(2, 2, 2, 1), -(3, 1, 2, 3)" +$CLICKHOUSE_CLIENT -n --query="$SETTINGS; +INSERT INTO hypothesis_test.test VALUES (0, 1, 2, 2), (1, 2, 1, 2), (2, 2, 2, 1), (3, 1, 2, 3)" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM hypothesis_test.test WHERE b > a FORMAT JSON" | grep "rows_read" # 4 +$CLICKHOUSE_CLIENT -n --query="$SETTINGS; SELECT count() FROM hypothesis_test.test WHERE b > a FORMAT JSON" | grep "rows_read" # 4 -$CLICKHOUSE_CLIENT --query="SELECT count() FROM hypothesis_test.test WHERE b <= a FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT -n --query="$SETTINGS; SELECT count() FROM hypothesis_test.test WHERE b <= a FORMAT JSON" | grep "rows_read" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM hypothesis_test.test WHERE b >= a FORMAT JSON" | grep "rows_read" # 4 +$CLICKHOUSE_CLIENT -n --query="$SETTINGS; SELECT count() FROM hypothesis_test.test WHERE b >= a FORMAT JSON" | grep "rows_read" # 4 -$CLICKHOUSE_CLIENT --query="SELECT count() FROM hypothesis_test.test WHERE b = a FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT -n --query="$SETTINGS; SELECT count() FROM hypothesis_test.test WHERE b = a FORMAT JSON" | grep "rows_read" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM hypothesis_test.test WHERE c < a FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT -n --query="$SETTINGS; SELECT count() FROM hypothesis_test.test WHERE c < a FORMAT JSON" | grep "rows_read" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM hypothesis_test.test WHERE c = a FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT -n --query="$SETTINGS; SELECT count() FROM hypothesis_test.test WHERE c = a FORMAT JSON" | grep "rows_read" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM hypothesis_test.test WHERE c > a FORMAT JSON" | grep "rows_read" # 4 +$CLICKHOUSE_CLIENT -n --query="$SETTINGS; SELECT count() FROM hypothesis_test.test WHERE c > a FORMAT JSON" | grep "rows_read" # 4 -$CLICKHOUSE_CLIENT --query="SELECT count() FROM hypothesis_test.test WHERE c < a FORMAT JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT -n --query="$SETTINGS; SELECT count() FROM hypothesis_test.test WHERE c < a FORMAT JSON" | grep "rows_read" $CLICKHOUSE_CLIENT -n --query=" +$SETTINGS; CREATE TABLE hypothesis_test.test2 ( i UInt64, a UInt64, @@ -57,22 +56,20 @@ CREATE TABLE hypothesis_test.test2 ( ) ENGINE = MergeTree() ORDER BY i SETTINGS index_granularity=1; " -$CLICKHOUSE_CLIENT --query="INSERT INTO hypothesis_test.test2 VALUES -(0, 1, 2), -(1, 2, 1), -(2, 2, 2), -(3, 1, 0)" +$CLICKHOUSE_CLIENT -n --query="$SETTINGS; +INSERT INTO hypothesis_test.test2 VALUES (0, 1, 2), (1, 2, 1), (2, 2, 2), (3, 1, 0)" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM hypothesis_test.test2 WHERE a < b FORMAT JSON" | grep "rows_read" # 4 +$CLICKHOUSE_CLIENT -n --query="$SETTINGS; SELECT count() FROM hypothesis_test.test2 WHERE a < b FORMAT JSON" | grep "rows_read" # 4 -$CLICKHOUSE_CLIENT --query="SELECT count() FROM hypothesis_test.test2 WHERE a <= b FORMAT JSON" | grep "rows_read" # 4 +$CLICKHOUSE_CLIENT -n --query="$SETTINGS; SELECT count() FROM hypothesis_test.test2 WHERE a <= b FORMAT JSON" | grep "rows_read" # 4 -$CLICKHOUSE_CLIENT --query="SELECT count() FROM hypothesis_test.test2 WHERE a = b FORMAT JSON" | grep "rows_read" # 1 +$CLICKHOUSE_CLIENT -n --query="$SETTINGS; SELECT count() FROM hypothesis_test.test2 WHERE a = b FORMAT JSON" | grep "rows_read" # 1 -$CLICKHOUSE_CLIENT --query="SELECT count() FROM hypothesis_test.test2 WHERE a != b FORMAT JSON" | grep "rows_read" # 4 +$CLICKHOUSE_CLIENT -n --query="$SETTINGS; SELECT count() FROM hypothesis_test.test2 WHERE a != b FORMAT JSON" | grep "rows_read" # 4 $CLICKHOUSE_CLIENT -n --query=" +$SETTINGS; CREATE TABLE hypothesis_test.test3 ( i UInt64, a UInt64, @@ -81,22 +78,21 @@ CREATE TABLE hypothesis_test.test3 ( ) ENGINE = MergeTree() ORDER BY i SETTINGS index_granularity=1; " -$CLICKHOUSE_CLIENT --query="INSERT INTO hypothesis_test.test3 VALUES -(0, 1, 2), -(1, 2, 1), -(2, 2, 2), -(3, 1, 0)" +$CLICKHOUSE_CLIENT -n --query=" +$SETTINGS; +INSERT INTO hypothesis_test.test3 VALUES (0, 1, 2), (1, 2, 1), (2, 2, 2), (3, 1, 0)" -$CLICKHOUSE_CLIENT --query="SELECT count() FROM hypothesis_test.test3 WHERE a < b FORMAT JSON" | grep "rows_read" # 3 +$CLICKHOUSE_CLIENT -n --query="$SETTINGS;SELECT count() FROM hypothesis_test.test3 WHERE a < b FORMAT JSON" | grep "rows_read" # 3 -$CLICKHOUSE_CLIENT --query="SELECT count() FROM hypothesis_test.test3 WHERE a <= b FORMAT JSON" | grep "rows_read" # 4 +$CLICKHOUSE_CLIENT -n --query="$SETTINGS;SELECT count() FROM hypothesis_test.test3 WHERE a <= b FORMAT JSON" | grep "rows_read" # 4 -$CLICKHOUSE_CLIENT --query="SELECT count() FROM hypothesis_test.test3 WHERE a = b FORMAT JSON" | grep "rows_read" # 4 +$CLICKHOUSE_CLIENT -n --query="$SETTINGS;SELECT count() FROM hypothesis_test.test3 WHERE a = b FORMAT JSON" | grep "rows_read" # 4 -$CLICKHOUSE_CLIENT --query="SELECT count() FROM hypothesis_test.test3 WHERE a != b FORMAT JSON" | grep "rows_read" # 3 +$CLICKHOUSE_CLIENT -n --query="$SETTINGS;SELECT count() FROM hypothesis_test.test3 WHERE a != b FORMAT JSON" | grep "rows_read" # 3 $CLICKHOUSE_CLIENT -n --query=" +$SETTINGS; DROP TABLE hypothesis_test.test; DROP TABLE hypothesis_test.test2; DROP TABLE hypothesis_test.test3;