thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-23 16:12:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Create new StorageView with substituted parameters for every SELECT query of a parameterized view

Browse Source
This commit is contained in:
Smita Kulkarni 2023-07-25 14:04:55 +02:00
parent 045bb3e1f3
commit 22fec136c1
11 changed files with 33 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/Interpreters/Context.cpp
View File

@ -114,7 +114,10 @@
#include <re2/re2.h>
#include <Storages/StorageView.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/FunctionParameterValuesVisitor.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <base/find_symbols.h>
#include <Interpreters/InterpreterSelectWithUnionQuery.h>
#if USE_ROCKSDB
#include <rocksdb/table.h>
@ -1576,8 +1579,21 @@ StoragePtr Context::executeTableFunction(const ASTPtr & table_expression, const
{
if (table.get()->isView() && table->as<StorageView>() && table->as<StorageView>()->isParameterizedView())
{
auto query = table->getInMemoryMetadataPtr()->getSelectQuery().inner_query->clone();
NameToNameMap parameterized_view_values = analyzeFunctionParamValues(table_expression);
StorageView::replaceQueryParametersIfParametrizedView(query,parameterized_view_values);
ASTCreateQuery create;
create.select = query->as<ASTSelectWithUnionQuery>();
auto sample_block = InterpreterSelectWithUnionQuery::getSampleBlock(query, getQueryContext());
auto res = std::make_shared<StorageView>(StorageID(database_name, table_name),
create,
ColumnsDescription(sample_block.getNamesAndTypesList()),
/* comment */ "",
/* is_parameterized_view */ true);
res->startup();
function->prefer_subquery_to_function_formatting = true;
return table;
return res;
}
}
auto hash = table_expression->getTreeHash();

24
src/Interpreters/InterpreterSelectQuery.cpp
View File

@ -610,27 +610,10 @@ InterpreterSelectQuery::InterpreterSelectQuery(
{
/// Allow push down and other optimizations for VIEW: replace with subquery and rewrite it.
ASTPtr view_table;
NameToNameMap parameter_types;
if (view)
{
query_info.is_parameterized_view = view->isParameterizedView();
/// We need to fetch the parameters set for SELECT ... FROM parameterized_view(<params>) before the query is replaced.
/// replaceWithSubquery replaces the function child and adds the subquery in its place.
/// the parameters are children of function child, if function (which corresponds to parametrised view and has
/// parameters in its arguments: `parametrised_view(<params>)`) is replaced the parameters are also gone from tree
/// So we need to get the parameters before they are removed from the tree
/// and after query is replaced, we use these parameters to substitute in the parameterized view query
if (query_info.is_parameterized_view)
{
query_info.parameterized_view_values = analyzeFunctionParamValues(query_ptr);
parameter_types = view->getParameterTypes();
}
view->replaceWithSubquery(getSelectQuery(), view_table, metadata_snapshot, view->isParameterizedView());
if (query_info.is_parameterized_view)
{
view->replaceQueryParametersIfParametrizedView(query_ptr, query_info.parameterized_view_values);
}
}
syntax_analyzer_result = TreeRewriter(context).analyzeSelect(
@ -639,10 +622,7 @@ InterpreterSelectQuery::InterpreterSelectQuery(
options,
joined_tables.tablesWithColumns(),
required_result_column_names,
table_join,
query_info.is_parameterized_view,
query_info.parameterized_view_values,
parameter_types);
table_join);
query_info.syntax_analyzer_result = syntax_analyzer_result;
@ -793,7 +773,7 @@ InterpreterSelectQuery::InterpreterSelectQuery(
query_info.filter_asts.push_back(parallel_replicas_custom_filter_ast);
}
source_header = storage_snapshot->getSampleBlockForColumns(required_columns, query_info.parameterized_view_values);
source_header = storage_snapshot->getSampleBlockForColumns(required_columns);
}
/// Calculate structure of the result.

7
src/Interpreters/TranslateQualifiedNamesVisitor.cpp
View File

@ -249,13 +249,6 @@ void TranslateQualifiedNamesMatcher::visit(ASTExpressionList & node, const ASTPt
if (first_table || !data.join_using_columns.contains(column.name))
{
std::string column_name = column.name;
/// replaceQueryParameterWithValue is used for parameterized view (which are created using query parameters
/// and SELECT is used with substitution of these query parameters )
if (!data.parameter_values.empty())
column_name
= StorageView::replaceQueryParameterWithValue(column_name, data.parameter_values, data.parameter_types);
addIdentifier(columns, table.table, column_name);
}
}

6
src/Interpreters/TranslateQualifiedNamesVisitor.h
View File

@ -28,15 +28,11 @@ public:
const TablesWithColumns & tables;
std::unordered_set<String> join_using_columns;
bool has_columns;
NameToNameMap parameter_values;
NameToNameMap parameter_types;
Data(const NameSet & source_columns_, const TablesWithColumns & tables_, bool has_columns_ = true, const NameToNameMap & parameter_values_ = {}, const NameToNameMap & parameter_types_ = {})
Data(const NameSet & source_columns_, const TablesWithColumns & tables_, bool has_columns_ = true)
: source_columns(source_columns_)
, tables(tables_)
, has_columns(has_columns_)
, parameter_values(parameter_values_)
, parameter_types(parameter_types_)
{}
bool hasColumn(const String & name) const { return source_columns.count(name); }

21
src/Interpreters/TreeRewriter.cpp
View File

@ -299,11 +299,10 @@ using ReplacePositionalArgumentsVisitor = InDepthNodeVisitor<OneTypeMatcher<Repl
/// Expand asterisks and qualified asterisks with column names.
/// There would be columns in normal form & column aliases after translation. Column & column alias would be normalized in QueryNormalizer.
void translateQualifiedNames(ASTPtr & query, const ASTSelectQuery & select_query, const NameSet & source_columns_set,
const TablesWithColumns & tables_with_columns, const NameToNameMap & parameter_values = {},
const NameToNameMap & parameter_types = {})
const TablesWithColumns & tables_with_columns)
{
LogAST log;
TranslateQualifiedNamesVisitor::Data visitor_data(source_columns_set, tables_with_columns, true/* has_columns */, parameter_values, parameter_types);
TranslateQualifiedNamesVisitor::Data visitor_data(source_columns_set, tables_with_columns, true/* has_columns */);
TranslateQualifiedNamesVisitor visitor(visitor_data, log.stream());
visitor.visit(query);
@ -1160,10 +1159,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect(
const SelectQueryOptions & select_options,
const TablesWithColumns & tables_with_columns,
const Names & required_result_columns,
std::shared_ptr<TableJoin> table_join,
bool is_parameterized_view,
const NameToNameMap parameter_values,
const NameToNameMap parameter_types) const
std::shared_ptr<TableJoin> table_join) const
{
auto * select_query = query->as<ASTSelectQuery>();
if (!select_query)
@ -1201,7 +1197,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect(
result.analyzed_join->setColumnsFromJoinedTable(std::move(columns_from_joined_table), source_columns_set, right_table.table.getQualifiedNamePrefix());
}
translateQualifiedNames(query, *select_query, source_columns_set, tables_with_columns, parameter_values, parameter_types);
translateQualifiedNames(query, *select_query, source_columns_set, tables_with_columns);
/// Optimizes logical expressions.
LogicalExpressionsOptimizer(select_query, tables_with_columns, settings.optimize_min_equality_disjunction_chain_length.value).perform();
@ -1259,15 +1255,6 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect(
result.window_function_asts = getWindowFunctions(query, *select_query);
result.expressions_with_window_function = getExpressionsWithWindowFunctions(query);
/// replaceQueryParameterWithValue is used for parameterized view (which are created using query parameters
/// and SELECT is used with substitution of these query parameters )
/// the replaced column names will be used in the next steps
if (is_parameterized_view)
{
for (auto & column : result.source_columns)
column.name = StorageView::replaceQueryParameterWithValue(column.name, parameter_values, parameter_types);
}
result.collectUsedColumns(query, true, settings.query_plan_optimize_primary_key);
result.required_source_columns_before_expanding_alias_columns = result.required_source_columns.getNames();

5
src/Interpreters/TreeRewriter.h
View File

@ -128,10 +128,7 @@ public:
const SelectQueryOptions & select_options = {},
const std::vector<TableWithColumnNamesAndTypes> & tables_with_columns = {},
const Names & required_result_columns = {},
std::shared_ptr<TableJoin> table_join = {},
bool is_parameterized_view = false,
const NameToNameMap parameter_values = {},
const NameToNameMap parameter_types = {}) const;
std::shared_ptr<TableJoin> table_join = {}) const;
private:
static void normalize(ASTPtr & query, Aliases & aliases, const NameSet & source_columns_set, bool ignore_alias, const Settings & settings, bool allow_self_aliases, ContextPtr context_, bool is_create_parameterized_view = false);

1
src/Storages/SelectQueryInfo.h
View File

@ -258,7 +258,6 @@ struct SelectQueryInfo
bool parallel_replicas_disabled = false;
bool is_parameterized_view = false;
NameToNameMap parameterized_view_values;
// If limit is not 0, that means it's a trivial limit query.
UInt64 limit = 0;

8
src/Storages/StorageSnapshot.cpp
View File

@ -113,7 +113,7 @@ NameAndTypePair StorageSnapshot::getColumn(const GetColumnsOptions & options, co
return *column;
}
Block StorageSnapshot::getSampleBlockForColumns(const Names & column_names, const NameToNameMap & parameter_values) const
Block StorageSnapshot::getSampleBlockForColumns(const Names & column_names) const
{
Block res;
@ -121,12 +121,6 @@ Block StorageSnapshot::getSampleBlockForColumns(const Names & column_names, cons
for (const auto & column_name : column_names)
{
std::string substituted_column_name = column_name;
/// substituted_column_name is used for parameterized view (which are created using query parameters
/// and SELECT is used with substitution of these query parameters )
if (!parameter_values.empty())
substituted_column_name = StorageView::replaceValueWithQueryParameter(column_name, parameter_values);
auto column = columns.tryGetColumnOrSubcolumn(GetColumnsOptions::All, substituted_column_name);
auto object_column = object_columns.tryGetColumnOrSubcolumn(GetColumnsOptions::All, substituted_column_name);
if (column && !object_column)

2
src/Storages/StorageSnapshot.h
View File

@ -71,7 +71,7 @@ struct StorageSnapshot
NameAndTypePair getColumn(const GetColumnsOptions & options, const String & column_name) const;
/// Block with ordinary + materialized + aliases + virtuals + subcolumns.
Block getSampleBlockForColumns(const Names & column_names, const NameToNameMap & parameter_values = {}) const;
Block getSampleBlockForColumns(const Names & column_names) const;
ColumnsDescription getDescriptionForColumns(const Names & column_names) const;

44
src/Storages/StorageView.cpp
View File

@ -107,7 +107,8 @@ StorageView::StorageView(
const StorageID & table_id_,
const ASTCreateQuery & query,
const ColumnsDescription & columns_,
const String & comment)
const String & comment,
const bool is_parameterized_view_)
: IStorage(table_id_)
{
StorageInMemoryMetadata storage_metadata;
@ -123,8 +124,7 @@ StorageView::StorageView(
NormalizeSelectWithUnionQueryVisitor::Data data{SetOperationMode::Unspecified};
NormalizeSelectWithUnionQueryVisitor{data}.visit(description.inner_query);
is_parameterized_view = query.isParameterizedView();
view_parameter_types = analyzeReceiveQueryParamsWithType(description.inner_query);
is_parameterized_view = is_parameterized_view_ || query.isParameterizedView();
storage_metadata.setSelectQuery(description);
setInMemoryMetadata(storage_metadata);
}
@ -173,7 +173,7 @@ void StorageView::read(
query_plan.addStep(std::move(materializing));
/// And also convert to expected structure.
const auto & expected_header = storage_snapshot->getSampleBlockForColumns(column_names, query_info.parameterized_view_values);
const auto & expected_header = storage_snapshot->getSampleBlockForColumns(column_names);
const auto & header = query_plan.getCurrentDataStream().header;
const auto * select_with_union = current_inner_query->as<ASTSelectWithUnionQuery>();
@ -258,42 +258,6 @@ void StorageView::replaceWithSubquery(ASTSelectQuery & outer_query, ASTPtr view_
child = view_query;
}
String StorageView::replaceQueryParameterWithValue(const String & column_name, const NameToNameMap & parameter_values, const NameToNameMap & parameter_types)
{
std::string name = column_name;
std::string::size_type pos = 0u;
for (const auto & parameter : parameter_values)
{
if ((pos = name.find(parameter.first)) != std::string::npos)
{
auto parameter_datatype_iterator = parameter_types.find(parameter.first);
size_t parameter_end = pos + parameter.first.size();
if (parameter_datatype_iterator != parameter_types.end() && name.size() >= parameter_end && (name[parameter_end] == ',' || name[parameter_end] == ')'))
{
String parameter_name("_CAST(" + parameter.second + ", '" + parameter_datatype_iterator->second + "')");
name.replace(pos, parameter.first.size(), parameter_name);
break;
}
}
}
return name;
}
String StorageView::replaceValueWithQueryParameter(const String & column_name, const NameToNameMap & parameter_values)
{
String name = column_name;
std::string::size_type pos = 0u;
for (const auto & parameter : parameter_values)
{
if ((pos = name.find("_CAST(" + parameter.second)) != std::string::npos)
{
name = name.substr(0,pos) + parameter.first + ")";
break;
}
}
return name;
}
ASTPtr StorageView::restoreViewName(ASTSelectQuery & select_query, const ASTPtr & view_name)
{
ASTTableExpression * table_expression = getFirstTableExpression(select_query);

11
src/Storages/StorageView.h
View File

@ -15,7 +15,8 @@ public:
const StorageID & table_id_,
const ASTCreateQuery & query,
const ColumnsDescription & columns_,
const String & comment);
const String & comment,
const bool is_parameterized_view_=false);
std::string getName() const override { return "View"; }
bool isView() const override { return true; }
@ -44,17 +45,9 @@ public:
static void replaceWithSubquery(ASTSelectQuery & outer_query, ASTPtr view_query, ASTPtr & view_name, const bool parameterized_view);
static ASTPtr restoreViewName(ASTSelectQuery & select_query, const ASTPtr & view_name);
static String replaceQueryParameterWithValue (const String & column_name, const NameToNameMap & parameter_values, const NameToNameMap & parameter_types);
static String replaceValueWithQueryParameter (const String & column_name, const NameToNameMap & parameter_values);
const NameToNameMap & getParameterTypes() const
{
return view_parameter_types;
}
protected:
bool is_parameterized_view;
NameToNameMap view_parameter_types;
};
}