2018-10-11 19:28:59 +00:00
|
|
|
#include <Interpreters/ExecuteScalarSubqueriesVisitor.h>
|
|
|
|
|
2021-04-10 23:33:54 +00:00
|
|
|
#include <Columns/ColumnTuple.h>
|
2019-05-17 14:34:25 +00:00
|
|
|
#include <DataStreams/IBlockInputStream.h>
|
2019-10-19 20:36:35 +00:00
|
|
|
#include <DataStreams/materializeBlock.h>
|
|
|
|
#include <DataTypes/DataTypeTuple.h>
|
2019-12-12 08:57:25 +00:00
|
|
|
#include <IO/WriteHelpers.h>
|
2021-04-10 23:33:54 +00:00
|
|
|
#include <Interpreters/Context.h>
|
|
|
|
#include <Interpreters/InterpreterSelectWithUnionQuery.h>
|
|
|
|
#include <Interpreters/addTypeConversionToAST.h>
|
|
|
|
#include <Interpreters/misc.h>
|
|
|
|
#include <Parsers/ASTExpressionList.h>
|
|
|
|
#include <Parsers/ASTFunction.h>
|
|
|
|
#include <Parsers/ASTLiteral.h>
|
|
|
|
#include <Parsers/ASTSelectQuery.h>
|
|
|
|
#include <Parsers/ASTSubquery.h>
|
|
|
|
#include <Parsers/ASTTablesInSelectQuery.h>
|
|
|
|
#include <Parsers/ASTWithElement.h>
|
2021-02-16 08:26:24 +00:00
|
|
|
#include <Processors/Executors/PullingAsyncPipelineExecutor.h>
|
2021-01-13 18:58:00 +00:00
|
|
|
|
2018-10-11 19:28:59 +00:00
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
|
|
|
extern const int INCORRECT_RESULT_OF_SCALAR_SUBQUERY;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2019-01-09 16:16:59 +00:00
|
|
|
bool ExecuteScalarSubqueriesMatcher::needChildVisit(ASTPtr & node, const ASTPtr & child)
|
2018-12-10 13:19:09 +00:00
|
|
|
{
|
|
|
|
/// Processed
|
2019-03-11 13:22:51 +00:00
|
|
|
if (node->as<ASTSubquery>() || node->as<ASTFunction>())
|
2018-12-10 13:19:09 +00:00
|
|
|
return false;
|
|
|
|
|
|
|
|
/// Don't descend into subqueries in FROM section
|
2019-03-11 13:22:51 +00:00
|
|
|
if (node->as<ASTTableExpression>())
|
2018-12-10 13:19:09 +00:00
|
|
|
return false;
|
|
|
|
|
2020-09-12 17:00:04 +00:00
|
|
|
/// Do not go to subqueries defined in with statement
|
|
|
|
if (node->as<ASTWithElement>())
|
|
|
|
return false;
|
|
|
|
|
2019-03-11 13:22:51 +00:00
|
|
|
if (node->as<ASTSelectQuery>())
|
2019-01-09 16:16:59 +00:00
|
|
|
{
|
|
|
|
/// Do not go to FROM, JOIN, UNION.
|
2019-03-11 13:22:51 +00:00
|
|
|
if (child->as<ASTTableExpression>() || child->as<ASTSelectQuery>())
|
2019-01-09 16:16:59 +00:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2018-12-10 13:19:09 +00:00
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2019-02-22 13:33:56 +00:00
|
|
|
void ExecuteScalarSubqueriesMatcher::visit(ASTPtr & ast, Data & data)
|
2018-12-10 13:19:09 +00:00
|
|
|
{
|
2019-03-11 13:22:51 +00:00
|
|
|
if (const auto * t = ast->as<ASTSubquery>())
|
2018-12-10 13:19:09 +00:00
|
|
|
visit(*t, ast, data);
|
2019-03-11 13:22:51 +00:00
|
|
|
if (const auto * t = ast->as<ASTFunction>())
|
2019-02-22 13:33:56 +00:00
|
|
|
visit(*t, ast, data);
|
2018-12-10 13:19:09 +00:00
|
|
|
}
|
|
|
|
|
2019-10-19 20:36:35 +00:00
|
|
|
/// Converting to literal values might take a fair amount of overhead when the value is large, (e.g.
|
|
|
|
/// Array, BitMap, etc.), This conversion is required for constant folding, index lookup, branch
|
|
|
|
/// elimination. However, these optimizations should never be related to large values, thus we
|
|
|
|
/// blacklist them here.
|
|
|
|
static bool worthConvertingToLiteral(const Block & scalar)
|
|
|
|
{
|
2020-04-22 06:01:33 +00:00
|
|
|
const auto * scalar_type_name = scalar.safeGetByPosition(0).type->getFamilyName();
|
2019-10-19 20:36:35 +00:00
|
|
|
std::set<String> useless_literal_types = {"Array", "Tuple", "AggregateFunction", "Function", "Set", "LowCardinality"};
|
|
|
|
return !useless_literal_types.count(scalar_type_name);
|
|
|
|
}
|
|
|
|
|
2018-12-10 13:02:45 +00:00
|
|
|
void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr & ast, Data & data)
|
2018-10-11 19:28:59 +00:00
|
|
|
{
|
2019-10-19 20:36:35 +00:00
|
|
|
auto hash = subquery.getTreeHash();
|
|
|
|
auto scalar_query_hash_str = toString(hash.first) + "_" + toString(hash.second);
|
|
|
|
|
|
|
|
Block scalar;
|
2021-04-10 23:33:54 +00:00
|
|
|
if (data.getContext()->hasQueryContext() && data.getContext()->getQueryContext()->hasScalar(scalar_query_hash_str))
|
|
|
|
scalar = data.getContext()->getQueryContext()->getScalar(scalar_query_hash_str);
|
2019-10-19 20:36:35 +00:00
|
|
|
else if (data.scalars.count(scalar_query_hash_str))
|
|
|
|
scalar = data.scalars[scalar_query_hash_str];
|
|
|
|
else
|
2018-10-11 19:28:59 +00:00
|
|
|
{
|
2021-04-10 23:33:54 +00:00
|
|
|
auto subquery_context = Context::createCopy(data.getContext());
|
|
|
|
Settings subquery_settings = data.getContext()->getSettings();
|
2019-10-19 20:36:35 +00:00
|
|
|
subquery_settings.max_result_rows = 1;
|
2020-03-08 23:48:08 +00:00
|
|
|
subquery_settings.extremes = false;
|
2021-04-10 23:33:54 +00:00
|
|
|
subquery_context->setSettings(subquery_settings);
|
2019-10-19 20:36:35 +00:00
|
|
|
|
|
|
|
ASTPtr subquery_select = subquery.children.at(0);
|
|
|
|
|
2021-02-09 03:46:06 +00:00
|
|
|
auto options = SelectQueryOptions(QueryProcessingStage::Complete, data.subquery_depth + 1, true);
|
2020-04-30 17:07:34 +00:00
|
|
|
options.analyze(data.only_analyze);
|
|
|
|
|
|
|
|
auto interpreter = InterpreterSelectWithUnionQuery(subquery_select, subquery_context, options);
|
2019-10-19 20:36:35 +00:00
|
|
|
Block block;
|
2020-04-30 17:07:34 +00:00
|
|
|
|
|
|
|
if (data.only_analyze)
|
|
|
|
{
|
|
|
|
/// If query is only analyzed, then constants are not correct.
|
|
|
|
block = interpreter.getSampleBlock();
|
|
|
|
for (auto & column : block)
|
2020-05-08 11:19:27 +00:00
|
|
|
{
|
2020-04-30 17:07:34 +00:00
|
|
|
if (column.column->empty())
|
2020-05-08 11:19:27 +00:00
|
|
|
{
|
|
|
|
auto mut_col = column.column->cloneEmpty();
|
|
|
|
mut_col->insertDefault();
|
|
|
|
column.column = std::move(mut_col);
|
|
|
|
}
|
|
|
|
}
|
2020-04-30 17:07:34 +00:00
|
|
|
}
|
|
|
|
else
|
2018-10-11 19:28:59 +00:00
|
|
|
{
|
2021-01-13 18:58:00 +00:00
|
|
|
auto io = interpreter.execute();
|
2019-10-19 20:36:35 +00:00
|
|
|
|
2021-05-26 15:44:52 +00:00
|
|
|
PullingAsyncPipelineExecutor executor(io.pipeline);
|
|
|
|
while (block.rows() == 0 && executor.pull(block));
|
2021-02-16 08:26:24 +00:00
|
|
|
|
2021-05-26 15:44:52 +00:00
|
|
|
if (block.rows() == 0)
|
|
|
|
{
|
|
|
|
/// Interpret subquery with empty result as Null literal
|
|
|
|
auto ast_new = std::make_unique<ASTLiteral>(Null());
|
|
|
|
ast_new->setAlias(ast->tryGetAlias());
|
|
|
|
ast = std::move(ast_new);
|
|
|
|
return;
|
|
|
|
}
|
2020-04-30 17:07:34 +00:00
|
|
|
|
2021-05-26 15:44:52 +00:00
|
|
|
if (block.rows() != 1)
|
|
|
|
throw Exception("Scalar subquery returned more than one row", ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY);
|
2021-02-16 08:26:24 +00:00
|
|
|
|
2021-05-26 15:44:52 +00:00
|
|
|
Block tmp_block;
|
|
|
|
while (tmp_block.rows() == 0 && executor.pull(tmp_block));
|
2021-02-16 08:26:24 +00:00
|
|
|
|
2021-05-26 15:44:52 +00:00
|
|
|
if (tmp_block.rows() != 0)
|
|
|
|
throw Exception("Scalar subquery returned more than one row", ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY);
|
2018-10-11 19:28:59 +00:00
|
|
|
}
|
|
|
|
|
2019-10-19 20:36:35 +00:00
|
|
|
block = materializeBlock(block);
|
|
|
|
size_t columns = block.columns();
|
|
|
|
|
|
|
|
if (columns == 1)
|
|
|
|
scalar = block;
|
2018-10-11 19:28:59 +00:00
|
|
|
else
|
2019-10-19 20:36:35 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
ColumnWithTypeAndName ctn;
|
|
|
|
ctn.type = std::make_shared<DataTypeTuple>(block.getDataTypes());
|
|
|
|
ctn.column = ColumnTuple::create(block.getColumns());
|
|
|
|
scalar.insert(ctn);
|
|
|
|
}
|
2018-10-11 19:28:59 +00:00
|
|
|
}
|
|
|
|
|
2021-04-10 23:33:54 +00:00
|
|
|
const Settings & settings = data.getContext()->getSettingsRef();
|
2019-10-19 20:36:35 +00:00
|
|
|
|
|
|
|
// Always convert to literals when there is no query context.
|
2021-04-10 23:33:54 +00:00
|
|
|
if (data.only_analyze || !settings.enable_scalar_subquery_optimization || worthConvertingToLiteral(scalar)
|
|
|
|
|| !data.getContext()->hasQueryContext())
|
2018-10-11 19:28:59 +00:00
|
|
|
{
|
2021-07-16 22:11:44 +00:00
|
|
|
/// subquery and ast can be the same object and ast will be moved.
|
|
|
|
/// Save these fields to avoid use after move.
|
2021-07-16 14:17:22 +00:00
|
|
|
auto alias = subquery.alias;
|
|
|
|
auto prefer_alias_to_column_name = subquery.prefer_alias_to_column_name;
|
|
|
|
|
2019-10-19 20:36:35 +00:00
|
|
|
auto lit = std::make_unique<ASTLiteral>((*scalar.safeGetByPosition(0).column)[0]);
|
2021-07-16 14:17:22 +00:00
|
|
|
lit->alias = alias;
|
|
|
|
lit->prefer_alias_to_column_name = prefer_alias_to_column_name;
|
2019-10-19 20:36:35 +00:00
|
|
|
ast = addTypeConversionToAST(std::move(lit), scalar.safeGetByPosition(0).type->getName());
|
2020-12-23 19:36:10 +00:00
|
|
|
|
|
|
|
/// If only analyze was requested the expression is not suitable for constant folding, disable it.
|
|
|
|
if (data.only_analyze)
|
|
|
|
{
|
|
|
|
ast->as<ASTFunction>()->alias.clear();
|
|
|
|
auto func = makeASTFunction("identity", std::move(ast));
|
2021-07-16 14:17:22 +00:00
|
|
|
func->alias = alias;
|
|
|
|
func->prefer_alias_to_column_name = prefer_alias_to_column_name;
|
2020-12-23 19:36:10 +00:00
|
|
|
ast = std::move(func);
|
|
|
|
}
|
2018-10-11 19:28:59 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2019-10-19 20:36:35 +00:00
|
|
|
auto func = makeASTFunction("__getScalar", std::make_shared<ASTLiteral>(scalar_query_hash_str));
|
|
|
|
func->alias = subquery.alias;
|
|
|
|
func->prefer_alias_to_column_name = subquery.prefer_alias_to_column_name;
|
|
|
|
ast = std::move(func);
|
2018-10-11 19:28:59 +00:00
|
|
|
}
|
2019-10-19 20:36:35 +00:00
|
|
|
|
|
|
|
data.scalars[scalar_query_hash_str] = std::move(scalar);
|
2018-10-11 19:28:59 +00:00
|
|
|
}
|
|
|
|
|
2019-02-22 13:33:56 +00:00
|
|
|
void ExecuteScalarSubqueriesMatcher::visit(const ASTFunction & func, ASTPtr & ast, Data & data)
|
2018-10-11 19:28:59 +00:00
|
|
|
{
|
|
|
|
/// Don't descend into subqueries in arguments of IN operator.
|
2021-03-04 05:59:57 +00:00
|
|
|
/// But if an argument is not subquery, then deeper may be scalar subqueries and we need to descend in them.
|
2018-10-11 19:28:59 +00:00
|
|
|
|
2018-12-10 13:02:45 +00:00
|
|
|
std::vector<ASTPtr *> out;
|
2020-07-15 14:22:54 +00:00
|
|
|
if (checkFunctionIsInOrGlobalInOperator(func))
|
2018-10-11 19:28:59 +00:00
|
|
|
{
|
|
|
|
for (auto & child : ast->children)
|
|
|
|
{
|
2018-11-01 17:07:20 +00:00
|
|
|
if (child != func.arguments)
|
2018-12-10 13:02:45 +00:00
|
|
|
out.push_back(&child);
|
2018-10-11 19:28:59 +00:00
|
|
|
else
|
2018-11-01 17:07:20 +00:00
|
|
|
for (size_t i = 0, size = func.arguments->children.size(); i < size; ++i)
|
2019-03-11 13:22:51 +00:00
|
|
|
if (i != 1 || !func.arguments->children[i]->as<ASTSubquery>())
|
2018-12-10 13:02:45 +00:00
|
|
|
out.push_back(&func.arguments->children[i]);
|
2018-10-11 19:28:59 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
2018-12-10 13:02:45 +00:00
|
|
|
for (auto & child : ast->children)
|
|
|
|
out.push_back(&child);
|
|
|
|
|
2019-02-22 13:33:56 +00:00
|
|
|
for (ASTPtr * add_node : out)
|
2019-02-22 15:45:47 +00:00
|
|
|
Visitor(data).visit(*add_node);
|
2018-10-11 19:28:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
}
|