diff --git a/dbms/src/Interpreters/AnalyzedJoin.cpp b/dbms/src/Interpreters/AnalyzedJoin.cpp index 8357ab80aba..44e145dba8a 100644 --- a/dbms/src/Interpreters/AnalyzedJoin.cpp +++ b/dbms/src/Interpreters/AnalyzedJoin.cpp @@ -80,7 +80,7 @@ ExpressionActionsPtr AnalyzedJoin::createJoinedBlockActions( ASTPtr query = expression_list; auto syntax_result = SyntaxAnalyzer(context).analyze(query, columns_from_joined_table, required_columns); - ExpressionAnalyzer analyzer(query, syntax_result, context, {}, required_columns_set); + ExpressionAnalyzer analyzer(query, syntax_result, context, required_columns_set); return analyzer.getActions(true, false); } diff --git a/dbms/src/Interpreters/AnalyzedJoin.h b/dbms/src/Interpreters/AnalyzedJoin.h index d820cb0da7b..db76acd5ac3 100644 --- a/dbms/src/Interpreters/AnalyzedJoin.h +++ b/dbms/src/Interpreters/AnalyzedJoin.h @@ -33,6 +33,7 @@ struct AnalyzedJoin private: friend class SyntaxAnalyzer; + friend struct SyntaxAnalyzerResult; friend class ExpressionAnalyzer; Names key_names_left; diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index fdc8226a42a..9405f7cedeb 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -58,7 +58,6 @@ #include #include #include -#include namespace DB { @@ -77,28 +76,15 @@ ExpressionAnalyzer::ExpressionAnalyzer( const ASTPtr & query_, const SyntaxAnalyzerResultPtr & syntax_analyzer_result_, const Context & context_, - const NamesAndTypesList & additional_source_columns, const NameSet & required_result_columns_, size_t subquery_depth_, bool do_global_, const SubqueriesForSets & subqueries_for_sets_) - : ExpressionAnalyzerData(syntax_analyzer_result_->source_columns, required_result_columns_, subqueries_for_sets_) + : ExpressionAnalyzerData(required_result_columns_, subqueries_for_sets_) , query(query_), context(context_), settings(context.getSettings()) , subquery_depth(subquery_depth_), do_global(do_global_) , syntax(syntax_analyzer_result_) { - storage = syntax->storage; - rewrite_subqueries = syntax->rewrite_subqueries; - - if (!additional_source_columns.empty()) - { - source_columns.insert(source_columns.end(), additional_source_columns.begin(), additional_source_columns.end()); - removeDuplicateColumns(source_columns); - } - - /// Delete the unnecessary from `source_columns` list. Form `columns_added_by_join`. - collectUsedColumns(); - /// external_tables, subqueries_for_sets for global subqueries. /// Replaces global subqueries with the generated names of temporary tables that will be sent to remote servers. initGlobalSubqueriesAndExternalTables(); @@ -115,7 +101,7 @@ ExpressionAnalyzer::ExpressionAnalyzer( bool ExpressionAnalyzer::isRemoteStorage() const { - return storage && storage->isRemote(); + return storage() && storage()->isRemote(); } @@ -133,7 +119,7 @@ void ExpressionAnalyzer::analyzeAggregation() if (select_query && (select_query->groupBy() || select_query->having())) has_aggregation = true; - ExpressionActionsPtr temp_actions = std::make_shared(source_columns, context); + ExpressionActionsPtr temp_actions = std::make_shared(sourceColumns(), context); if (select_query) { @@ -256,7 +242,7 @@ void ExpressionAnalyzer::makeSetsForIndex() { const auto * select_query = query->as(); - if (storage && select_query && storage->supportsIndexForIn()) + if (storage() && select_query && storage()->supportsIndexForIn()) { if (select_query->where()) makeSetsForIndexImpl(select_query->where()); @@ -312,7 +298,7 @@ void ExpressionAnalyzer::makeSetsForIndexImpl(const ASTPtr & node) { const IAST & args = *func->arguments; - if (storage && storage->mayBenefitFromIndexForIn(args.children.at(0), context)) + if (storage() && storage()->mayBenefitFromIndexForIn(args.children.at(0), context)) { const ASTPtr & arg = args.children.at(1); if (arg->as() || arg->as()) @@ -322,9 +308,9 @@ void ExpressionAnalyzer::makeSetsForIndexImpl(const ASTPtr & node) } else { - NamesAndTypesList temp_columns = source_columns; + NamesAndTypesList temp_columns = sourceColumns(); temp_columns.insert(temp_columns.end(), array_join_columns.begin(), array_join_columns.end()); - for (const auto & joined_column : columns_added_by_join) + for (const auto & joined_column : columnsAddedByJoin()) temp_columns.push_back(joined_column); ExpressionActionsPtr temp_actions = std::make_shared(temp_columns, context); getRootActions(func->arguments->children.at(0), true, temp_actions); @@ -343,7 +329,7 @@ void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_subqueries, { LogAST log; ActionsVisitor actions_visitor(context, settings.size_limits_for_set, subquery_depth, - source_columns, actions, prepared_sets, subqueries_for_sets, + sourceColumns(), actions, prepared_sets, subqueries_for_sets, no_subqueries, only_consts, !isRemoteStorage(), log.stream()); actions_visitor.visit(ast); actions = actions_visitor.popActionsLevel(); @@ -356,7 +342,7 @@ void ExpressionAnalyzer::getActionsFromJoinKeys(const ASTTableJoin & table_join, LogAST log; ActionsVisitor actions_visitor(context, settings.size_limits_for_set, subquery_depth, - source_columns, actions, prepared_sets, subqueries_for_sets, + sourceColumns(), actions, prepared_sets, subqueries_for_sets, no_subqueries, only_consts, !isRemoteStorage(), log.stream()); if (table_join.using_expression_list) @@ -494,7 +480,7 @@ bool ExpressionAnalyzer::appendArrayJoin(ExpressionActionsChain & chain, bool on if (!array_join_expression_list) return false; - initChain(chain, source_columns); + initChain(chain, sourceColumns()); ExpressionActionsChain::Step & step = chain.steps.back(); getRootActions(array_join_expression_list, only_types, step.actions); @@ -507,12 +493,12 @@ bool ExpressionAnalyzer::appendArrayJoin(ExpressionActionsChain & chain, bool on void ExpressionAnalyzer::addJoinAction(ExpressionActionsPtr & actions, bool only_types) const { if (only_types) - actions->add(ExpressionAction::ordinaryJoin(nullptr, analyzedJoin().key_names_left, columns_added_by_join)); + actions->add(ExpressionAction::ordinaryJoin(nullptr, analyzedJoin().key_names_left, columnsAddedByJoin())); else for (auto & subquery_for_set : subqueries_for_sets) if (subquery_for_set.second.join) actions->add(ExpressionAction::ordinaryJoin(subquery_for_set.second.join, analyzedJoin().key_names_left, - columns_added_by_join)); + columnsAddedByJoin())); } static void appendRequiredColumns( @@ -536,7 +522,7 @@ bool ExpressionAnalyzer::appendJoin(ExpressionActionsChain & chain, bool only_ty if (!select_query->join()) return false; - initChain(chain, source_columns); + initChain(chain, sourceColumns()); ExpressionActionsChain::Step & step = chain.steps.back(); const auto & join_element = select_query->join()->as(); @@ -588,7 +574,7 @@ bool ExpressionAnalyzer::appendJoin(ExpressionActionsChain & chain, bool only_ty auto & analyzed_join = analyzedJoin(); /// Actions which need to be calculated on joined block. ExpressionActionsPtr joined_block_actions = - analyzed_join.createJoinedBlockActions(columns_added_by_join, select_query, context); + analyzed_join.createJoinedBlockActions(columnsAddedByJoin(), select_query, context); /** For GLOBAL JOINs (in the case, for example, of the push method for executing GLOBAL subqueries), the following occurs * - in the addExternalStorage function, the JOIN (SELECT ...) subquery is replaced with JOIN _data1, @@ -610,7 +596,7 @@ bool ExpressionAnalyzer::appendJoin(ExpressionActionsChain & chain, bool only_ty NameSet required_columns(action_columns.begin(), action_columns.end()); appendRequiredColumns( - required_columns, joined_block_actions->getSampleBlock(), analyzed_join.key_names_right, columns_added_by_join); + required_columns, joined_block_actions->getSampleBlock(), analyzed_join.key_names_right, columnsAddedByJoin()); auto original_map = analyzed_join.getOriginalColumnsMap(required_columns); Names original_columns; @@ -647,7 +633,7 @@ bool ExpressionAnalyzer::appendPrewhere( if (!select_query->prewhere()) return false; - initChain(chain, source_columns); + initChain(chain, sourceColumns()); auto & step = chain.getLastStep(); getRootActions(select_query->prewhere(), only_types, step.actions); String prewhere_column_name = select_query->prewhere()->getColumnName(); @@ -656,7 +642,7 @@ bool ExpressionAnalyzer::appendPrewhere( { /// Remove unused source_columns from prewhere actions. - auto tmp_actions = std::make_shared(source_columns, context); + auto tmp_actions = std::make_shared(sourceColumns(), context); getRootActions(select_query->prewhere(), only_types, tmp_actions); tmp_actions->finalize({prewhere_column_name}); auto required_columns = tmp_actions->getRequiredColumns(); @@ -676,7 +662,7 @@ bool ExpressionAnalyzer::appendPrewhere( auto names = step.actions->getSampleBlock().getNames(); NameSet name_set(names.begin(), names.end()); - for (const auto & column : source_columns) + for (const auto & column : sourceColumns()) if (required_source_columns.count(column.name) == 0) name_set.erase(column.name); @@ -697,7 +683,7 @@ bool ExpressionAnalyzer::appendPrewhere( NameSet prewhere_input_names(required_columns.begin(), required_columns.end()); NameSet unused_source_columns; - for (const auto & column : source_columns) + for (const auto & column : sourceColumns()) { if (prewhere_input_names.count(column.name) == 0) { @@ -722,7 +708,7 @@ bool ExpressionAnalyzer::appendWhere(ExpressionActionsChain & chain, bool only_t if (!select_query->where()) return false; - initChain(chain, source_columns); + initChain(chain, sourceColumns()); ExpressionActionsChain::Step & step = chain.steps.back(); step.required_output.push_back(select_query->where()->getColumnName()); @@ -742,7 +728,7 @@ bool ExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain, bool only if (!select_query->groupBy()) return false; - initChain(chain, source_columns); + initChain(chain, sourceColumns()); ExpressionActionsChain::Step & step = chain.steps.back(); ASTs asts = select_query->groupBy()->children; @@ -761,7 +747,7 @@ void ExpressionAnalyzer::appendAggregateFunctionsArguments(ExpressionActionsChai assertAggregation(); - initChain(chain, source_columns); + initChain(chain, sourceColumns()); ExpressionActionsChain::Step & step = chain.steps.back(); for (size_t i = 0; i < aggregate_descriptions.size(); ++i) @@ -899,7 +885,7 @@ void ExpressionAnalyzer::appendProjectResult(ExpressionActionsChain & chain) con void ExpressionAnalyzer::appendExpression(ExpressionActionsChain & chain, const ASTPtr & expr, bool only_types) { - initChain(chain, source_columns); + initChain(chain, sourceColumns()); ExpressionActionsChain::Step & step = chain.steps.back(); getRootActions(expr, only_types, step.actions); step.required_output.push_back(expr->getColumnName()); @@ -921,7 +907,7 @@ void ExpressionAnalyzer::getActionsBeforeAggregation(const ASTPtr & ast, Express ExpressionActionsPtr ExpressionAnalyzer::getActions(bool add_aliases, bool project_result) { - ExpressionActionsPtr actions = std::make_shared(source_columns, context); + ExpressionActionsPtr actions = std::make_shared(sourceColumns(), context); NamesWithAliases result_columns; Names result_names; @@ -956,7 +942,7 @@ ExpressionActionsPtr ExpressionAnalyzer::getActions(bool add_aliases, bool proje if (!(add_aliases && project_result)) { /// We will not delete the original columns. - for (const auto & column_name_type : source_columns) + for (const auto & column_name_type : sourceColumns()) result_names.push_back(column_name_type.name); } @@ -982,164 +968,4 @@ void ExpressionAnalyzer::getAggregateInfo(Names & key_names, AggregateDescriptio aggregates = aggregate_descriptions; } -void ExpressionAnalyzer::collectUsedColumns() -{ - /** Calculate which columns are required to execute the expression. - * Then, delete all other columns from the list of available columns. - * After execution, columns will only contain the list of columns needed to read from the table. - */ - - RequiredSourceColumnsVisitor::Data columns_context; - RequiredSourceColumnsVisitor(columns_context).visit(query); - - NameSet source_column_names; - for (const auto & column : source_columns) - source_column_names.insert(column.name); - - NameSet required = columns_context.requiredColumns(); - - if (columns_context.has_table_join) - { - const AnalyzedJoin & analyzed_join = analyzedJoin(); - NameSet avaliable_columns; - for (const auto & name : source_columns) - avaliable_columns.insert(name.name); - - /// Add columns obtained by JOIN (if needed). - columns_added_by_join.clear(); - for (const auto & joined_column : analyzed_join.available_joined_columns) - { - auto & name = joined_column.name; - if (avaliable_columns.count(name)) - continue; - - if (required.count(name)) - { - /// Optimisation: do not add columns needed only in JOIN ON section. - if (columns_context.nameInclusion(name) > analyzed_join.rightKeyInclusion(name)) - columns_added_by_join.push_back(joined_column); - required.erase(name); - } - } - } - - NameSet array_join_sources; - if (columns_context.has_array_join) - { - /// Insert the columns required for the ARRAY JOIN calculation into the required columns list. - for (const auto & result_source : syntax->array_join_result_to_source) - array_join_sources.insert(result_source.second); - - for (const auto & column_name_type : source_columns) - if (array_join_sources.count(column_name_type.name)) - required.insert(column_name_type.name); - } - - const auto * select_query = query->as(); - - /// You need to read at least one column to find the number of rows. - if (select_query && required.empty()) - { - /// We will find a column with minimum . - /// Because it is the column that is cheapest to read. - struct ColumnSizeTuple - { - size_t compressed_size; - size_t type_size; - size_t uncompressed_size; - String name; - bool operator<(const ColumnSizeTuple & that) const - { - return std::tie(compressed_size, type_size, uncompressed_size) - < std::tie(that.compressed_size, that.type_size, that.uncompressed_size); - } - }; - std::vector columns; - if (storage) - { - auto column_sizes = storage->getColumnSizes(); - for (auto & source_column : source_columns) - { - auto c = column_sizes.find(source_column.name); - if (c == column_sizes.end()) - continue; - size_t type_size = source_column.type->haveMaximumSizeOfValue() ? source_column.type->getMaximumSizeOfValueInMemory() : 100; - columns.emplace_back(ColumnSizeTuple{c->second.data_compressed, type_size, c->second.data_uncompressed, source_column.name}); - } - } - if (columns.size()) - required.insert(std::min_element(columns.begin(), columns.end())->name); - else - /// If we have no information about columns sizes, choose a column of minimum size of its data type. - required.insert(ExpressionActions::getSmallestColumn(source_columns)); - } - - NameSet unknown_required_source_columns = required; - - for (NamesAndTypesList::iterator it = source_columns.begin(); it != source_columns.end();) - { - const String & column_name = it->name; - unknown_required_source_columns.erase(column_name); - - if (!required.count(column_name)) - source_columns.erase(it++); - else - ++it; - } - - /// If there are virtual columns among the unknown columns. Remove them from the list of unknown and add - /// in columns list, so that when further processing they are also considered. - if (storage) - { - for (auto it = unknown_required_source_columns.begin(); it != unknown_required_source_columns.end();) - { - if (storage->hasColumn(*it)) - { - source_columns.push_back(storage->getColumn(*it)); - unknown_required_source_columns.erase(it++); - } - else - ++it; - } - } - - if (!unknown_required_source_columns.empty()) - { - std::stringstream ss; - ss << "Missing columns:"; - for (const auto & name : unknown_required_source_columns) - ss << " '" << name << "'"; - ss << " while processing query: '" << query << "'"; - - ss << ", required columns:"; - for (const auto & name : columns_context.requiredColumns()) - ss << " '" << name << "'"; - - if (!source_column_names.empty()) - { - ss << ", source columns:"; - for (const auto & name : source_column_names) - ss << " '" << name << "'"; - } - else - ss << ", no source columns"; - - if (columns_context.has_table_join) - { - ss << ", joined columns:"; - for (const auto & column : analyzedJoin().available_joined_columns) - ss << " '" << column.name << "'"; - } - - if (!array_join_sources.empty()) - { - ss << ", arrayJoin columns:"; - for (const auto & name : array_join_sources) - ss << " '" << name << "'"; - } - - throw Exception(ss.str(), ErrorCodes::UNKNOWN_IDENTIFIER); - } -} - } diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.h b/dbms/src/Interpreters/ExpressionAnalyzer.h index 644d10da1be..3bb8a2bab07 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.h +++ b/dbms/src/Interpreters/ExpressionAnalyzer.h @@ -29,13 +29,8 @@ struct SyntaxAnalyzerResult; using SyntaxAnalyzerResultPtr = std::shared_ptr; /// ExpressionAnalyzer sources, intermediates and results. It splits data and logic, allows to test them separately. -/// If you are not writing a test you probably don't need it. Use ExpressionAnalyzer itself. struct ExpressionAnalyzerData { - /// Original columns. - /// First, all available columns of the table are placed here. Then (when analyzing the query), unused columns are deleted. - NamesAndTypesList source_columns; - /// If non-empty, ignore all expressions in not from this list. NameSet required_result_columns; @@ -58,15 +53,10 @@ struct ExpressionAnalyzerData /// Predicate optimizer overrides the sub queries bool rewrite_subqueries = false; - /// Columns will be added to block by join. - NamesAndTypesList columns_added_by_join; /// Subset of analyzed_join.available_joined_columns - protected: - ExpressionAnalyzerData(const NamesAndTypesList & source_columns_, - const NameSet & required_result_columns_, + ExpressionAnalyzerData(const NameSet & required_result_columns_, const SubqueriesForSets & subqueries_for_sets_) - : source_columns(source_columns_), - required_result_columns(required_result_columns_), + : required_result_columns(required_result_columns_), subqueries_for_sets(subqueries_for_sets_) {} }; @@ -102,7 +92,6 @@ public: const ASTPtr & query_, const SyntaxAnalyzerResultPtr & syntax_analyzer_result_, const Context & context_, - const NamesAndTypesList & additional_source_columns = {}, const NameSet & required_result_columns_ = {}, size_t subquery_depth_ = 0, bool do_global_ = false, @@ -114,11 +103,6 @@ public: /// Get a list of aggregation keys and descriptions of aggregate functions if the query contains GROUP BY. void getAggregateInfo(Names & key_names, AggregateDescriptions & aggregates) const; - /** Get a set of columns that are enough to read from the table to evaluate the expression. - * Columns added from another table by JOIN are not counted. - */ - Names getRequiredSourceColumns() const { return source_columns.getNames(); } - /** These methods allow you to build a chain of transformations over a block, that receives values in the desired sections of the query. * * Example usage: @@ -182,25 +166,21 @@ public: /// Create Set-s that we can from IN section to use the index on them. void makeSetsForIndex(); - bool isRewriteSubqueriesPredicate() { return rewrite_subqueries; } - bool hasGlobalSubqueries() { return has_global_subqueries; } private: ASTPtr query; const Context & context; const ExtractedSettings settings; - StoragePtr storage; /// The main table in FROM clause, if exists. size_t subquery_depth; bool do_global; /// Do I need to prepare for execution global subqueries when analyzing the query. SyntaxAnalyzerResultPtr syntax; - const AnalyzedJoin & analyzedJoin() const { return syntax->analyzed_join; } - /** Remove all unnecessary columns from the list of all available columns of the table (`columns`). - * At the same time, form a set of columns added by JOIN (`columns_added_by_join`). - */ - void collectUsedColumns(); + const StoragePtr & storage() const { return syntax->storage; } /// The main table in FROM clause, if exists. + const AnalyzedJoin & analyzedJoin() const { return syntax->analyzed_join; } + const NamesAndTypesList & sourceColumns() const { return syntax->required_source_columns; } + const NamesAndTypesList & columnsAddedByJoin() const { return syntax->columns_added_by_join; } /// Find global subqueries in the GLOBAL IN/JOIN sections. Fills in external_tables. void initGlobalSubqueriesAndExternalTables(); diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 9682d0e29e4..0335fbd2a81 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -292,9 +292,9 @@ InterpreterSelectQuery::InterpreterSelectQuery( table_lock = storage->lockStructureForShare(false, context.getCurrentQueryId()); syntax_analyzer_result = SyntaxAnalyzer(context, options).analyze( - query_ptr, source_header.getNamesAndTypesList(), required_result_column_names, storage); + query_ptr, source_header.getNamesAndTypesList(), required_result_column_names, storage, NamesAndTypesList()); query_analyzer = std::make_unique( - query_ptr, syntax_analyzer_result, context, NamesAndTypesList(), + query_ptr, syntax_analyzer_result, context, NameSet(required_result_column_names.begin(), required_result_column_names.end()), options.subquery_depth, !options.only_analyze); @@ -317,7 +317,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (!options.only_analyze || options.modify_inplace) { - if (query_analyzer->isRewriteSubqueriesPredicate()) + if (syntax_analyzer_result->rewrite_subqueries) { /// remake interpreter_subquery when PredicateOptimizer rewrites subqueries and main table is subquery if (is_subquery) @@ -336,7 +336,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( interpreter_subquery->ignoreWithTotals(); } - required_columns = query_analyzer->getRequiredSourceColumns(); + required_columns = syntax_analyzer_result->requiredSourceColumns(); if (storage) source_header = storage->getSampleBlockForColumns(required_columns); @@ -675,7 +675,16 @@ static SortingInfoPtr optimizeReadInOrder(const MergeTreeData & merge_tree, cons size_t prefix_size = std::min(order_descr.size(), sorting_key_columns.size()); auto order_by_expr = query.orderBy(); - auto syntax_result = SyntaxAnalyzer(context).analyze(order_by_expr, merge_tree.getColumns().getAllPhysical()); + SyntaxAnalyzerResultPtr syntax_result; + try + { + syntax_result = SyntaxAnalyzer(context).analyze(order_by_expr, merge_tree.getColumns().getAllPhysical()); + } + catch (const Exception &) + { + return {}; + } + for (size_t i = 0; i < prefix_size; ++i) { /// Read in pk order in case of exact match with order key element @@ -789,7 +798,7 @@ void InterpreterSelectQuery::executeImpl(TPipeline & pipeline, const BlockInputS /// Try transferring some condition from WHERE to PREWHERE if enabled and viable if (settings.optimize_move_to_prewhere && query.where() && !query.prewhere() && !query.final()) - MergeTreeWhereOptimizer{current_info, context, merge_tree, query_analyzer->getRequiredSourceColumns(), log}; + MergeTreeWhereOptimizer{current_info, context, merge_tree, syntax_analyzer_result->requiredSourceColumns(), log}; }; if (const MergeTreeData * merge_tree_data = dynamic_cast(storage.get())) diff --git a/dbms/src/Interpreters/MutationsInterpreter.cpp b/dbms/src/Interpreters/MutationsInterpreter.cpp index 69339f66712..b5a60666ae0 100644 --- a/dbms/src/Interpreters/MutationsInterpreter.cpp +++ b/dbms/src/Interpreters/MutationsInterpreter.cpp @@ -186,8 +186,7 @@ void MutationsInterpreter::prepare(bool dry_run) { auto query = column.default_desc.expression->clone(); auto syntax_result = SyntaxAnalyzer(context).analyze(query, all_columns); - ExpressionAnalyzer analyzer(query, syntax_result, context); - for (const String & dependency : analyzer.getRequiredSourceColumns()) + for (const String & dependency : syntax_result->requiredSourceColumns()) { if (updated_columns.count(dependency)) column_to_affected_materialized[dependency].push_back(column.name); diff --git a/dbms/src/Interpreters/RequiredSourceColumnsVisitor.cpp b/dbms/src/Interpreters/RequiredSourceColumnsVisitor.cpp index a6e581cb6a8..b5f9c83db50 100644 --- a/dbms/src/Interpreters/RequiredSourceColumnsVisitor.cpp +++ b/dbms/src/Interpreters/RequiredSourceColumnsVisitor.cpp @@ -40,7 +40,7 @@ static std::vector extractNamesFromLambda(const ASTFunction & node) return names; } -bool RequiredSourceColumnsMatcher::needChildVisit(ASTPtr & node, const ASTPtr & child) +bool RequiredSourceColumnsMatcher::needChildVisit(const ASTPtr & node, const ASTPtr & child) { if (child->as()) return false; @@ -60,7 +60,7 @@ bool RequiredSourceColumnsMatcher::needChildVisit(ASTPtr & node, const ASTPtr & return true; } -void RequiredSourceColumnsMatcher::visit(ASTPtr & ast, Data & data) +void RequiredSourceColumnsMatcher::visit(const ASTPtr & ast, Data & data) { /// results are columns @@ -111,7 +111,7 @@ void RequiredSourceColumnsMatcher::visit(ASTPtr & ast, Data & data) } } -void RequiredSourceColumnsMatcher::visit(ASTSelectQuery & select, const ASTPtr &, Data & data) +void RequiredSourceColumnsMatcher::visit(const ASTSelectQuery & select, const ASTPtr &, Data & data) { /// special case for top-level SELECT items: they are publics for (auto & node : select.select()->children) @@ -128,7 +128,7 @@ void RequiredSourceColumnsMatcher::visit(ASTSelectQuery & select, const ASTPtr & Visitor(data).visit(node); /// revisit select_expression_list (with children) when all the aliases are set - Visitor(data).visit(select.refSelect()); + Visitor(data).visit(select.select()); } void RequiredSourceColumnsMatcher::visit(const ASTIdentifier & node, const ASTPtr &, Data & data) @@ -158,7 +158,7 @@ void RequiredSourceColumnsMatcher::visit(const ASTFunction & node, const ASTPtr } } -void RequiredSourceColumnsMatcher::visit(ASTTablesInSelectQueryElement & node, const ASTPtr &, Data & data) +void RequiredSourceColumnsMatcher::visit(const ASTTablesInSelectQueryElement & node, const ASTPtr &, Data & data) { ASTTableExpression * expr = nullptr; ASTTableJoin * join = nullptr; @@ -177,7 +177,7 @@ void RequiredSourceColumnsMatcher::visit(ASTTablesInSelectQueryElement & node, c } /// ASTIdentifiers here are tables. Do not visit them as generic ones. -void RequiredSourceColumnsMatcher::visit(ASTTableExpression & node, const ASTPtr &, Data & data) +void RequiredSourceColumnsMatcher::visit(const ASTTableExpression & node, const ASTPtr &, Data & data) { if (node.database_and_table_name) data.addTableAliasIfAny(*node.database_and_table_name); diff --git a/dbms/src/Interpreters/RequiredSourceColumnsVisitor.h b/dbms/src/Interpreters/RequiredSourceColumnsVisitor.h index ed3ec75ddc9..b42a95f29ee 100644 --- a/dbms/src/Interpreters/RequiredSourceColumnsVisitor.h +++ b/dbms/src/Interpreters/RequiredSourceColumnsVisitor.h @@ -21,19 +21,19 @@ struct ASTTableExpression; class RequiredSourceColumnsMatcher { public: - using Visitor = InDepthNodeVisitor; + using Visitor = ConstInDepthNodeVisitor; using Data = ColumnNamesContext; - static bool needChildVisit(ASTPtr & node, const ASTPtr & child); - static void visit(ASTPtr & ast, Data & data); + static bool needChildVisit(const ASTPtr & node, const ASTPtr & child); + static void visit(const ASTPtr & ast, Data & data); private: static void visit(const ASTIdentifier & node, const ASTPtr &, Data & data); static void visit(const ASTFunction & node, const ASTPtr &, Data & data); - static void visit(ASTTablesInSelectQueryElement & node, const ASTPtr &, Data & data); - static void visit(ASTTableExpression & node, const ASTPtr &, Data & data); + static void visit(const ASTTablesInSelectQueryElement & node, const ASTPtr &, Data & data); + static void visit(const ASTTableExpression & node, const ASTPtr &, Data & data); static void visit(const ASTArrayJoin & node, const ASTPtr &, Data & data); - static void visit(ASTSelectQuery & select, const ASTPtr &, Data & data); + static void visit(const ASTSelectQuery & select, const ASTPtr &, Data & data); }; /// Extracts all the information about columns and tables from ASTSelectQuery block into ColumnNamesContext object. diff --git a/dbms/src/Interpreters/SyntaxAnalyzer.cpp b/dbms/src/Interpreters/SyntaxAnalyzer.cpp index 0cb833e9bc7..282b19991b1 100644 --- a/dbms/src/Interpreters/SyntaxAnalyzer.cpp +++ b/dbms/src/Interpreters/SyntaxAnalyzer.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -44,6 +45,7 @@ namespace ErrorCodes extern const int INVALID_JOIN_ON_EXPRESSION; extern const int EMPTY_LIST_OF_COLUMNS_QUERIED; extern const int NOT_IMPLEMENTED; + extern const int UNKNOWN_IDENTIFIER; } NameSet removeDuplicateColumns(NamesAndTypesList & columns) @@ -558,12 +560,181 @@ void checkJoin(const ASTTablesInSelectQueryElement * join) } +/// Calculate which columns are required to execute the expression. +/// Then, delete all other columns from the list of available columns. +/// After execution, columns will only contain the list of columns needed to read from the table. +void SyntaxAnalyzerResult::collectUsedColumns(const ASTPtr & query, const NamesAndTypesList & additional_source_columns) +{ + /// We caclulate required_source_columns with source_columns modifications and swap them on exit + required_source_columns = source_columns; + + if (!additional_source_columns.empty()) + { + source_columns.insert(source_columns.end(), additional_source_columns.begin(), additional_source_columns.end()); + removeDuplicateColumns(source_columns); + } + + RequiredSourceColumnsVisitor::Data columns_context; + RequiredSourceColumnsVisitor(columns_context).visit(query); + + NameSet source_column_names; + for (const auto & column : source_columns) + source_column_names.insert(column.name); + + NameSet required = columns_context.requiredColumns(); + + if (columns_context.has_table_join) + { + NameSet avaliable_columns; + for (const auto & name : source_columns) + avaliable_columns.insert(name.name); + + /// Add columns obtained by JOIN (if needed). + columns_added_by_join.clear(); + for (const auto & joined_column : analyzed_join.available_joined_columns) + { + auto & name = joined_column.name; + if (avaliable_columns.count(name)) + continue; + + if (required.count(name)) + { + /// Optimisation: do not add columns needed only in JOIN ON section. + if (columns_context.nameInclusion(name) > analyzed_join.rightKeyInclusion(name)) + columns_added_by_join.push_back(joined_column); + required.erase(name); + } + } + } + + NameSet array_join_sources; + if (columns_context.has_array_join) + { + /// Insert the columns required for the ARRAY JOIN calculation into the required columns list. + for (const auto & result_source : array_join_result_to_source) + array_join_sources.insert(result_source.second); + + for (const auto & column_name_type : source_columns) + if (array_join_sources.count(column_name_type.name)) + required.insert(column_name_type.name); + } + + const auto * select_query = query->as(); + + /// You need to read at least one column to find the number of rows. + if (select_query && required.empty()) + { + /// We will find a column with minimum . + /// Because it is the column that is cheapest to read. + struct ColumnSizeTuple + { + size_t compressed_size; + size_t type_size; + size_t uncompressed_size; + String name; + bool operator<(const ColumnSizeTuple & that) const + { + return std::tie(compressed_size, type_size, uncompressed_size) + < std::tie(that.compressed_size, that.type_size, that.uncompressed_size); + } + }; + std::vector columns; + if (storage) + { + auto column_sizes = storage->getColumnSizes(); + for (auto & source_column : source_columns) + { + auto c = column_sizes.find(source_column.name); + if (c == column_sizes.end()) + continue; + size_t type_size = source_column.type->haveMaximumSizeOfValue() ? source_column.type->getMaximumSizeOfValueInMemory() : 100; + columns.emplace_back(ColumnSizeTuple{c->second.data_compressed, type_size, c->second.data_uncompressed, source_column.name}); + } + } + if (columns.size()) + required.insert(std::min_element(columns.begin(), columns.end())->name); + else + /// If we have no information about columns sizes, choose a column of minimum size of its data type. + required.insert(ExpressionActions::getSmallestColumn(source_columns)); + } + + NameSet unknown_required_source_columns = required; + + for (NamesAndTypesList::iterator it = source_columns.begin(); it != source_columns.end();) + { + const String & column_name = it->name; + unknown_required_source_columns.erase(column_name); + + if (!required.count(column_name)) + source_columns.erase(it++); + else + ++it; + } + + /// If there are virtual columns among the unknown columns. Remove them from the list of unknown and add + /// in columns list, so that when further processing they are also considered. + if (storage) + { + for (auto it = unknown_required_source_columns.begin(); it != unknown_required_source_columns.end();) + { + if (storage->hasColumn(*it)) + { + source_columns.push_back(storage->getColumn(*it)); + unknown_required_source_columns.erase(it++); + } + else + ++it; + } + } + + if (!unknown_required_source_columns.empty()) + { + std::stringstream ss; + ss << "Missing columns:"; + for (const auto & name : unknown_required_source_columns) + ss << " '" << name << "'"; + ss << " while processing query: '" << queryToString(query) << "'"; + + ss << ", required columns:"; + for (const auto & name : columns_context.requiredColumns()) + ss << " '" << name << "'"; + + if (!source_column_names.empty()) + { + ss << ", source columns:"; + for (const auto & name : source_column_names) + ss << " '" << name << "'"; + } + else + ss << ", no source columns"; + + if (columns_context.has_table_join) + { + ss << ", joined columns:"; + for (const auto & column : analyzed_join.available_joined_columns) + ss << " '" << column.name << "'"; + } + + if (!array_join_sources.empty()) + { + ss << ", arrayJoin columns:"; + for (const auto & name : array_join_sources) + ss << " '" << name << "'"; + } + + throw Exception(ss.str(), ErrorCodes::UNKNOWN_IDENTIFIER); + } + + required_source_columns.swap(source_columns); +} + SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze( ASTPtr & query, const NamesAndTypesList & source_columns_, const Names & required_result_columns, - StoragePtr storage) const + StoragePtr storage, + const NamesAndTypesList & additional_source_columns) const { auto * select_query = query->as(); if (!storage && select_query) @@ -669,6 +840,7 @@ SyntaxAnalyzerResultPtr SyntaxAnalyzer::analyze( collectJoinedColumns(result.analyzed_join, *select_query, source_columns_set, result.aliases, settings.join_use_nulls); } + result.collectUsedColumns(query, additional_source_columns); return std::make_shared(result); } diff --git a/dbms/src/Interpreters/SyntaxAnalyzer.h b/dbms/src/Interpreters/SyntaxAnalyzer.h index 5896358eb85..c7addb03526 100644 --- a/dbms/src/Interpreters/SyntaxAnalyzer.h +++ b/dbms/src/Interpreters/SyntaxAnalyzer.h @@ -13,8 +13,13 @@ NameSet removeDuplicateColumns(NamesAndTypesList & columns); struct SyntaxAnalyzerResult { StoragePtr storage; + AnalyzedJoin analyzed_join; NamesAndTypesList source_columns; + /// Set of columns that are enough to read from the table to evaluate the expression. It does not include joined columns. + NamesAndTypesList required_source_columns; + /// Columns will be added to block by JOIN. It's a subset of analyzed_join.available_joined_columns + NamesAndTypesList columns_added_by_join; Aliases aliases; @@ -31,10 +36,11 @@ struct SyntaxAnalyzerResult /// Note: not used further. NameToNameMap array_join_name_to_alias; - AnalyzedJoin analyzed_join; - /// Predicate optimizer overrides the sub queries bool rewrite_subqueries = false; + + void collectUsedColumns(const ASTPtr & query, const NamesAndTypesList & additional_source_columns); + Names requiredSourceColumns() const { return required_source_columns.getNames(); } }; using SyntaxAnalyzerResultPtr = std::shared_ptr; @@ -64,7 +70,8 @@ public: ASTPtr & query, const NamesAndTypesList & source_columns_, const Names & required_result_columns = {}, - StoragePtr storage = {}) const; + StoragePtr storage = {}, + const NamesAndTypesList & additional_source_columns = {}) const; private: const Context & context; diff --git a/dbms/src/Interpreters/evaluateMissingDefaults.cpp b/dbms/src/Interpreters/evaluateMissingDefaults.cpp index 83af15d1924..bef41488793 100644 --- a/dbms/src/Interpreters/evaluateMissingDefaults.cpp +++ b/dbms/src/Interpreters/evaluateMissingDefaults.cpp @@ -61,7 +61,7 @@ void evaluateMissingDefaults(Block & block, auto syntax_result = SyntaxAnalyzer(context).analyze(default_expr_list, block.getNamesAndTypesList()); auto expression_analyzer = ExpressionAnalyzer{default_expr_list, syntax_result, context}; - auto required_source_columns = expression_analyzer.getRequiredSourceColumns(); + auto required_source_columns = syntax_result->requiredSourceColumns(); auto rows_was = copy_block.rows(); // Delete all not needed columns in DEFAULT expression. diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index dfc59654629..74e5df41217 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -134,8 +134,7 @@ MergeTreeData::MergeTreeData( throw Exception("Sampling expression must be present in the primary key", ErrorCodes::BAD_ARGUMENTS); auto syntax = SyntaxAnalyzer(global_context).analyze(sample_by_ast, getColumns().getAllPhysical()); - columns_required_for_sampling = ExpressionAnalyzer(sample_by_ast, syntax, global_context) - .getRequiredSourceColumns(); + columns_required_for_sampling = syntax->requiredSourceColumns(); } MergeTreeDataFormatVersion min_format_version(0); if (!date_column_name.empty()) @@ -295,8 +294,7 @@ void MergeTreeData::setPrimaryKeyIndicesAndColumns( if (!added_key_column_expr_list->children.empty()) { auto syntax = SyntaxAnalyzer(global_context).analyze(added_key_column_expr_list, all_columns); - Names used_columns = ExpressionAnalyzer(added_key_column_expr_list, syntax, global_context) - .getRequiredSourceColumns(); + Names used_columns = syntax->requiredSourceColumns(); NamesAndTypesList deleted_columns; NamesAndTypesList added_columns; diff --git a/dbms/src/Storages/transformQueryForExternalDatabase.cpp b/dbms/src/Storages/transformQueryForExternalDatabase.cpp index c0fcbabba42..55a0ef95200 100644 --- a/dbms/src/Storages/transformQueryForExternalDatabase.cpp +++ b/dbms/src/Storages/transformQueryForExternalDatabase.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -111,8 +110,7 @@ String transformQueryForExternalDatabase( { auto clone_query = query.clone(); auto syntax_result = SyntaxAnalyzer(context).analyze(clone_query, available_columns); - ExpressionAnalyzer analyzer(clone_query, syntax_result, context); - const Names & used_columns = analyzer.getRequiredSourceColumns(); + const Names used_columns = syntax_result->requiredSourceColumns(); auto select = std::make_shared();