From 449e11f1ea3ad8fee6a7a4cdacc6a5f5ed1e840e Mon Sep 17 00:00:00 2001 From: Nikita Vasilev Date: Tue, 4 May 2021 22:18:37 +0300 Subject: [PATCH] settings --- src/CMakeLists.txt | 2 +- src/Interpreters/TreeOptimizer.cpp | 20 ++++++++++++------- .../WhereConstraintsOptimizer.cpp | 8 ++++++-- src/Interpreters/WhereConstraintsOptimizer.h | 4 +++- .../01622_constraints_simple_optimization.sql | 2 ++ .../01623_constraints_column_swap.sql | 2 ++ .../0_stateless/01624_soft_constraints.sh | 2 +- .../01625_constraints_index_append.sql | 2 ++ 8 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f6f4d5b6199..69a84dbeb2c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -69,7 +69,7 @@ add_subdirectory (Coordination) set(dbms_headers) -set(dbms_sources Interpreters/ConstraintMatcherVisitor.cpp Interpreters/ConstraintMatcherVisitor.h Interpreters/WhereConstraintsOptimizer.cpp Interpreters/WhereConstraintsOptimizer.h Interpreters/TreeCNFConverter.cpp Interpreters/TreeCNFConverter.h Interpreters/ComparisonGraph.cpp Interpreters/ComparisonGraph.h Storages/MergeTree/SubstituteColumnOptimizer.cpp Storages/MergeTree/SubstituteColumnOptimizer.h Storages/MergeTree/MergeTreeIndexHypothesis.cpp Storages/MergeTree/MergeTreeIndexHypothesis.h Interpreters/AddIndexConstraintsOptimizer.cpp Interpreters/AddIndexConstraintsOptimizer.h Storages/MergeTree/MergeTreeIndexMergedCondition.cpp Storages/MergeTree/MergeTreeIndexMergedCondition.h) +set(dbms_sources) add_headers_and_sources(clickhouse_common_io Common) add_headers_and_sources(clickhouse_common_io Common/HashTable) diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index 6637792f8da..c82c908998d 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -511,18 +511,23 @@ void optimizeLimitBy(const ASTSelectQuery * select_query) } /// Use constraints to get rid of useless parts of query -void optimizeWithConstraints(ASTSelectQuery * select_query, Aliases & /*aliases*/, const NameSet & /*source_columns_set*/, - const std::vector & /*tables_with_columns*/, - const StorageMetadataPtr & metadata_snapshot) +void optimizeWithConstraints(ASTSelectQuery * select_query, + Aliases & /*aliases*/, + const NameSet & /*source_columns_set*/, + const std::vector & /*tables_with_columns*/, + const StorageMetadataPtr & metadata_snapshot, + const bool optimize_append_index) { - WhereConstraintsOptimizer(select_query, metadata_snapshot).perform(); + WhereConstraintsOptimizer(select_query, metadata_snapshot, optimize_append_index).perform(); if (select_query->where()) Poco::Logger::get("CNF").information(select_query->where()->dumpTree()); else Poco::Logger::get("CNF").information("NO WHERE"); } -void optimizeSubstituteColumn(ASTSelectQuery * select_query, Aliases & /*aliases*/, const NameSet & /*source_columns_set*/, +void optimizeSubstituteColumn(ASTSelectQuery * select_query, + Aliases & /*aliases*/, + const NameSet & /*source_columns_set*/, const std::vector & /*tables_with_columns*/, const StorageMetadataPtr & metadata_snapshot, const ConstStoragePtr & storage) @@ -647,8 +652,9 @@ void TreeOptimizer::apply(ASTPtr & query, Aliases & aliases, const NameSet & sou if (settings.convert_query_to_cnf && settings.optimize_using_constraints) { - optimizeWithConstraints(select_query, aliases, source_columns_set, tables_with_columns, metadata_snapshot); - optimizeSubstituteColumn(select_query, aliases, source_columns_set, tables_with_columns, metadata_snapshot, storage); + optimizeWithConstraints(select_query, aliases, source_columns_set, tables_with_columns, metadata_snapshot, settings.optimize_append_index); + if (settings.optimize_substitute_columns) + optimizeSubstituteColumn(select_query, aliases, source_columns_set, tables_with_columns, metadata_snapshot, storage); } if (select_query->where()) { diff --git a/src/Interpreters/WhereConstraintsOptimizer.cpp b/src/Interpreters/WhereConstraintsOptimizer.cpp index bfdc88f34f2..13e09cd844c 100644 --- a/src/Interpreters/WhereConstraintsOptimizer.cpp +++ b/src/Interpreters/WhereConstraintsOptimizer.cpp @@ -14,9 +14,11 @@ namespace DB WhereConstraintsOptimizer::WhereConstraintsOptimizer( ASTSelectQuery * select_query_, - const StorageMetadataPtr & metadata_snapshot_) + const StorageMetadataPtr & metadata_snapshot_, + const bool optimize_append_index_) : select_query(select_query_) , metadata_snapshot(metadata_snapshot_) + , optimize_append_index(optimize_append_index_) { } @@ -184,7 +186,9 @@ void WhereConstraintsOptimizer::perform() return replaceTermsToConstants(atom, compare_graph); }) .pushNotInFuntions(); - AddIndexConstraintsOptimizer(metadata_snapshot).perform(cnf); + + if (optimize_append_index) + AddIndexConstraintsOptimizer(metadata_snapshot).perform(cnf); Poco::Logger::get("AFTER OPT").information(cnf.dump()); select_query->setExpression(ASTSelectQuery::Expression::WHERE, TreeCNFConverter::fromCNF(cnf)); diff --git a/src/Interpreters/WhereConstraintsOptimizer.h b/src/Interpreters/WhereConstraintsOptimizer.h index 8a58aa41fa5..75160983a76 100644 --- a/src/Interpreters/WhereConstraintsOptimizer.h +++ b/src/Interpreters/WhereConstraintsOptimizer.h @@ -19,13 +19,15 @@ class WhereConstraintsOptimizer final public: WhereConstraintsOptimizer( ASTSelectQuery * select_query, - const StorageMetadataPtr & metadata_snapshot); + const StorageMetadataPtr & metadata_snapshot, + const bool optimize_append_index_); void perform(); private: ASTSelectQuery * select_query; const StorageMetadataPtr & metadata_snapshot; + bool optimize_append_index; }; } diff --git a/tests/queries/0_stateless/01622_constraints_simple_optimization.sql b/tests/queries/0_stateless/01622_constraints_simple_optimization.sql index e9370dc8492..0ab71cd8161 100644 --- a/tests/queries/0_stateless/01622_constraints_simple_optimization.sql +++ b/tests/queries/0_stateless/01622_constraints_simple_optimization.sql @@ -6,6 +6,8 @@ DROP TABLE IF EXISTS constraint_test.transitivity2; SET convert_query_to_cnf = 1; SET optimize_using_constraints = 1; SET optimize_move_to_prewhere = 1; +SET optimize_substitute_columns = 1; +SET optimize_append_index = 1; CREATE DATABASE constraint_test; CREATE TABLE constraint_test.assumption (URL String, a Int32, CONSTRAINT c1 ASSUME domainWithoutWWW(URL) = 'yandex.ru', CONSTRAINT c2 ASSUME URL > 'zzz' AND startsWith(URL, 'test') = True) ENGINE = TinyLog; diff --git a/tests/queries/0_stateless/01623_constraints_column_swap.sql b/tests/queries/0_stateless/01623_constraints_column_swap.sql index 96073007505..a6c5c5b6bc5 100644 --- a/tests/queries/0_stateless/01623_constraints_column_swap.sql +++ b/tests/queries/0_stateless/01623_constraints_column_swap.sql @@ -1,6 +1,8 @@ SET convert_query_to_cnf = 1; SET optimize_using_constraints = 1; SET optimize_move_to_prewhere = 1; +SET optimize_substitute_columns = 1; +SET optimize_append_index = 1; DROP DATABASE IF EXISTS column_swap_test; DROP TABLE IF EXISTS column_swap_test.test; diff --git a/tests/queries/0_stateless/01624_soft_constraints.sh b/tests/queries/0_stateless/01624_soft_constraints.sh index 3625d4f0e5f..d27d51079ae 100755 --- a/tests/queries/0_stateless/01624_soft_constraints.sh +++ b/tests/queries/0_stateless/01624_soft_constraints.sh @@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -SETTINGS="SET convert_query_to_cnf = 1; SET optimize_using_constraints = 1; SET optimize_move_to_prewhere = 1" +SETTINGS="SET convert_query_to_cnf = 1; SET optimize_using_constraints = 1; SET optimize_move_to_prewhere = 1; SET optimize_substitute_columns = 1; SET optimize_append_index = 1" $CLICKHOUSE_CLIENT -n --query=" $SETTINGS; diff --git a/tests/queries/0_stateless/01625_constraints_index_append.sql b/tests/queries/0_stateless/01625_constraints_index_append.sql index 28c18334369..7d75c57218a 100644 --- a/tests/queries/0_stateless/01625_constraints_index_append.sql +++ b/tests/queries/0_stateless/01625_constraints_index_append.sql @@ -1,6 +1,8 @@ SET convert_query_to_cnf = 1; SET optimize_using_constraints = 1; SET optimize_move_to_prewhere = 1; +SET optimize_substitute_columns = 1; +SET optimize_append_index = 1; DROP DATABASE IF EXISTS index_append_test; DROP TABLE IF EXISTS index_append_test.test;