diff --git a/src/DataStreams/AddingDefaultsBlockInputStream.cpp b/src/DataStreams/AddingDefaultsBlockInputStream.cpp index 160d1b4fb76..045151465b8 100644 --- a/src/DataStreams/AddingDefaultsBlockInputStream.cpp +++ b/src/DataStreams/AddingDefaultsBlockInputStream.cpp @@ -171,7 +171,12 @@ Block AddingDefaultsBlockInputStream::readImpl() if (!evaluate_block.columns()) evaluate_block.insert({ColumnConst::create(ColumnUInt8::create(1, 0), res.rows()), std::make_shared(), "_dummy"}); - evaluateMissingDefaults(evaluate_block, header.getNamesAndTypesList(), columns, context, false); + auto dag = createFillingMissingDefaultsExpression(evaluate_block, header.getNamesAndTypesList(), columns, context, false); + if (dag) + { + auto actions = std::make_shared(std::move(dag)); + actions->execute(evaluate_block); + } std::unordered_map mixed_columns; diff --git a/src/Functions/replicate.cpp b/src/Functions/replicate.cpp index adbb37a7c91..aa4a7d42c6c 100644 --- a/src/Functions/replicate.cpp +++ b/src/Functions/replicate.cpp @@ -11,6 +11,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_COLUMN; + extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION; } namespace @@ -35,33 +36,54 @@ public: size_t getNumberOfArguments() const override { - return 2; + return 0; } + bool isVariadic() const override { return true; } + bool useDefaultImplementationForNulls() const override { return false; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - const DataTypeArray * array_type = checkAndGetDataType(arguments[1].get()); - if (!array_type) - throw Exception("Second argument for function " + getName() + " must be array.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (arguments.size() < 2) + throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION, + "Function {} expect at leas two arguments, got {}", getName(), arguments.size()); + + for (size_t i = 1; i < arguments.size(); ++i) + { + const DataTypeArray * array_type = checkAndGetDataType(arguments[i].get()); + if (!array_type) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Argument {} for function {} must be array.", + i + 1, getName()); + } return std::make_shared(arguments[0]); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override { ColumnPtr first_column = arguments[0].column; - const ColumnArray * array_column = checkAndGetColumn(arguments[1].column.get()); - ColumnPtr temp_column; - if (!array_column) + ColumnPtr offsets; + + for (size_t i = 1; i < arguments.size(); ++i) { - const auto * const_array_column = checkAndGetColumnConst(arguments[1].column.get()); - if (!const_array_column) - throw Exception("Unexpected column for replicate", ErrorCodes::ILLEGAL_COLUMN); - temp_column = const_array_column->convertToFullColumn(); - array_column = checkAndGetColumn(temp_column.get()); + const ColumnArray * array_column = checkAndGetColumn(arguments[i].column.get()); + ColumnPtr temp_column; + if (!array_column) + { + const auto * const_array_column = checkAndGetColumnConst(arguments[i].column.get()); + if (!const_array_column) + throw Exception("Unexpected column for replicate", ErrorCodes::ILLEGAL_COLUMN); + temp_column = const_array_column->convertToFullColumn(); + array_column = checkAndGetColumn(temp_column.get()); + } + + if (!offsets || offsets->empty()) + offsets = array_column->getOffsetsPtr(); } - return ColumnArray::create(first_column->replicate(array_column->getOffsets())->convertToFullColumnIfConst(), array_column->getOffsetsPtr()); + + const auto & offsets_data = assert_cast(*offsets).getData(); + return ColumnArray::create(first_column->replicate(offsets_data)->convertToFullColumnIfConst(), offsets); } }; diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index d8c40ffda2f..ed6f8527bc5 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -618,6 +618,25 @@ bool ActionsDAG::trivial() const return true; } +void ActionsDAG::addMaterializingOutputActions() +{ + FunctionOverloadResolverPtr func_builder_materialize = + std::make_shared( + std::make_unique( + std::make_shared())); + + Index new_index; + for (auto * node : index) + { + auto & name = node->result_name; + node = &addFunction(func_builder_materialize, {node}, {}, true); + node = &addAlias(*node, name, true); + new_index.insert(node); + } + + index.swap(new_index); +} + ActionsDAGPtr ActionsDAG::makeConvertingActions( const ColumnsWithTypeAndName & source, const ColumnsWithTypeAndName & result, diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index b12da30e24f..c929f1a4a59 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -231,6 +231,9 @@ public: ActionsDAGPtr clone() const; + /// For apply materialize() function for every output. + /// Also add aliases so the result names remain unchanged. + void addMaterializingOutputActions(); enum class MatchColumnsMode { diff --git a/src/Interpreters/addMissingDefaults.cpp b/src/Interpreters/addMissingDefaults.cpp index 37a0812826b..7935ca04b13 100644 --- a/src/Interpreters/addMissingDefaults.cpp +++ b/src/Interpreters/addMissingDefaults.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB @@ -74,7 +75,12 @@ Block addMissingDefaults( } /// Computes explicitly specified values by default and materialized columns. - evaluateMissingDefaults(res, required_columns, columns, context); + auto dag = createFillingMissingDefaultsExpression(res, required_columns, columns, context); + if (dag) + { + auto actions = std::make_shared(std::move(dag)); + actions->execute(res); + } return res; } diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp index ab74aa7d631..3e2f055307a 100644 --- a/src/Interpreters/inplaceBlockConversions.cpp +++ b/src/Interpreters/inplaceBlockConversions.cpp @@ -24,7 +24,7 @@ namespace { /// Add all required expressions for missing columns calculation -void addDefaultRequiredExpressionsRecursively(Block & block, const String & required_column, const ColumnsDescription & columns, ASTPtr default_expr_list_accum, NameSet & added_columns) +void addDefaultRequiredExpressionsRecursively(const Block & block, const String & required_column, const ColumnsDescription & columns, ASTPtr default_expr_list_accum, NameSet & added_columns) { checkStackSize(); if (block.has(required_column) || added_columns.count(required_column)) @@ -52,7 +52,7 @@ void addDefaultRequiredExpressionsRecursively(Block & block, const String & requ } } -ASTPtr defaultRequiredExpressions(Block & block, const NamesAndTypesList & required_columns, const ColumnsDescription & columns) +ASTPtr defaultRequiredExpressions(const Block & block, const NamesAndTypesList & required_columns, const ColumnsDescription & columns) { ASTPtr default_expr_list = std::make_shared(); @@ -87,67 +87,32 @@ ASTPtr convertRequiredExpressions(Block & block, const NamesAndTypesList & requi return conversion_expr_list; } -void executeExpressionsOnBlock( - Block & block, +ActionsDAGPtr createFillingMissingDefaultsExpression( + const Block & header, ASTPtr expr_list, bool save_unneeded_columns, const NamesAndTypesList & required_columns, const Context & context) { if (!expr_list) - return; + return nullptr; - if (!save_unneeded_columns) - { - auto syntax_result = TreeRewriter(context).analyze(expr_list, block.getNamesAndTypesList()); - ExpressionAnalyzer{expr_list, syntax_result, context}.getActions(true)->execute(block); - return; - } - - /** ExpressionAnalyzer eliminates "unused" columns, in order to ensure their safety - * we are going to operate on a copy instead of the original block */ - Block copy_block{block}; - - auto syntax_result = TreeRewriter(context).analyze(expr_list, block.getNamesAndTypesList()); + auto syntax_result = TreeRewriter(context).analyze(expr_list, header.getNamesAndTypesList()); auto expression_analyzer = ExpressionAnalyzer{expr_list, syntax_result, context}; - auto required_source_columns = syntax_result->requiredSourceColumns(); - auto rows_was = copy_block.rows(); + auto dag = expression_analyzer.getActionsDAG(true, !save_unneeded_columns); - // Delete all not needed columns in DEFAULT expression. - // They can intersect with columns added in PREWHERE - // test 00950_default_prewhere - // CLICKHOUSE-4523 - for (const auto & delete_column : copy_block.getNamesAndTypesList()) + if (save_unneeded_columns) { - if (std::find(required_source_columns.begin(), required_source_columns.end(), delete_column.name) == required_source_columns.end()) - { - copy_block.erase(delete_column.name); - } + Names required_names; + required_names.resize(required_columns.size()); + for (const auto & column : required_columns) + required_names.push_back(column.name); + + dag->removeUnusedActions(required_names); + dag->addMaterializingOutputActions(); } - if (copy_block.columns() == 0) - { - // Add column to indicate block size in execute() - copy_block.insert({DataTypeUInt8().createColumnConst(rows_was, 0u), std::make_shared(), "__dummy"}); - } - - expression_analyzer.getActions(true)->execute(copy_block); - - /// move evaluated columns to the original block, materializing them at the same time - size_t pos = 0; - for (auto col = required_columns.begin(); col != required_columns.end(); ++col, ++pos) - { - if (copy_block.has(col->name)) - { - auto evaluated_col = copy_block.getByName(col->name); - evaluated_col.column = evaluated_col.column->convertToFullColumnIfConst(); - - if (block.has(col->name)) - block.getByName(col->name) = std::move(evaluated_col); - else - block.insert(pos, std::move(evaluated_col)); - } - } + return dag; } } @@ -157,19 +122,25 @@ void performRequiredConversions(Block & block, const NamesAndTypesList & require ASTPtr conversion_expr_list = convertRequiredExpressions(block, required_columns); if (conversion_expr_list->children.empty()) return; - executeExpressionsOnBlock(block, conversion_expr_list, true, required_columns, context); + + if (auto dag = createFillingMissingDefaultsExpression(block, conversion_expr_list, true, required_columns, context)) + { + auto expression = std::make_shared(std::move(dag)); + expression->execute(block); + } } -void evaluateMissingDefaults(Block & block, +ActionsDAGPtr createFillingMissingDefaultsExpression( + const Block & header, const NamesAndTypesList & required_columns, const ColumnsDescription & columns, const Context & context, bool save_unneeded_columns) { if (!columns.hasDefaults()) - return; + return nullptr; - ASTPtr default_expr_list = defaultRequiredExpressions(block, required_columns, columns); - executeExpressionsOnBlock(block, default_expr_list, save_unneeded_columns, required_columns, context); + ASTPtr expr_list = defaultRequiredExpressions(header, required_columns, columns); + return createFillingMissingDefaultsExpression(header, expr_list, save_unneeded_columns, required_columns, context); } } diff --git a/src/Interpreters/inplaceBlockConversions.h b/src/Interpreters/inplaceBlockConversions.h index 066975ab4bc..ffba4a45ba1 100644 --- a/src/Interpreters/inplaceBlockConversions.h +++ b/src/Interpreters/inplaceBlockConversions.h @@ -12,9 +12,13 @@ class Context; class NamesAndTypesList; class ColumnsDescription; +class ActionsDAG; +using ActionsDAGPtr = std::shared_ptr; + /// Adds missing defaults to block according to required_columns /// using columns description -void evaluateMissingDefaults(Block & block, +ActionsDAGPtr createFillingMissingDefaultsExpression( + const Block & header, const NamesAndTypesList & required_columns, const ColumnsDescription & columns, const Context & context, bool save_unneeded_columns = true); diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp new file mode 100644 index 00000000000..0d651897bf8 --- /dev/null +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -0,0 +1,22 @@ +#include +#include + +namespace DB::QueryPlanOptimizations +{ + +size_t tryPushDownLimit(QueryPlan::Node * node, QueryPlan::Nodes &) +{ + auto * filter_step = typeid_cast(node->step.get()); + if (!filter_step) + return 0; + + QueryPlan::Node * child_node = node->children.front(); + auto & child = child_node->step; + + if (const auto * adding_const_column = typeid_cast(child.get())) + { + + } +} + +} diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 0140b32e12c..052f6922cac 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -186,7 +186,13 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns additional_columns.insert({res_columns[pos], name_and_type->type, name_and_type->name}); } - DB::evaluateMissingDefaults(additional_columns, columns, metadata_snapshot->getColumns(), storage.global_context); + auto dag = DB::createFillingMissingDefaultsExpression( + additional_columns, columns, metadata_snapshot->getColumns(), storage.global_context); + if (dag) + { + auto actions = std::make_shared(std::move(dag)); + actions->execute(additional_columns); + } /// Move columns from block. name_and_type = columns.begin();