thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add EXCEPTSTRICT and REPLACESTRICT column transformers

Browse Source
This commit is contained in:
hexiaoting 2020-10-21 15:54:13 +08:00
parent 293d2f06fa
commit 91b1dab75b
6 changed files with 85 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
src/Parsers/ASTColumnsTransformers.cpp
View File

@ -46,7 +46,9 @@ void ASTColumnsApplyTransformer::transform(ASTs & nodes) const
void ASTColumnsExceptTransformer::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
{
settings.ostr << (settings.hilite ? hilite_keyword : "") << "EXCEPT" << (settings.hilite ? hilite_none : "") << "(";
settings.ostr << (settings.hilite ? hilite_keyword : "");
settings.ostr << (is_strict ? "EXCEPTSTRICT" : "EXCEPT");
settings.ostr << (settings.hilite ? hilite_none : "") << "(";
for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it)
{
@ -62,23 +64,42 @@ void ASTColumnsExceptTransformer::formatImpl(const FormatSettings & settings, Fo
void ASTColumnsExceptTransformer::transform(ASTs & nodes) const
{
ASTs unexcepted_columns(children);
nodes.erase(
std::remove_if(
nodes.begin(),
nodes.end(),
[this](const ASTPtr & node_child)
[&](const ASTPtr & node_child)
{
if (const auto * id = node_child->as<ASTIdentifier>())
{
for (const auto & except_child : children)
for (size_t i = 0; i < children.size(); i++)
{
if (except_child->as<const ASTIdentifier &>().name == id->shortName())
if (children[i]->as<const ASTIdentifier &>().name == id->shortName())
{
unexcepted_columns.erase(unexcepted_columns.begin() + i);
return true;
}
}
}
return false;
}),
nodes.end());
if (is_strict && !unexcepted_columns.empty())
{
String unexisted_columns;
for (size_t i = 0; i < unexcepted_columns.size(); ++i)
{
if (i > 0)
unexisted_columns += ", ";
unexisted_columns += unexcepted_columns[i]->as<const ASTIdentifier &>().name;
}
throw Exception(
"Columns transformer EXPCEPTSTRICT process unexist column : " + unexisted_columns,
ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
}
}
void ASTColumnsReplaceTransformer::Replacement::formatImpl(
@ -90,7 +111,9 @@ void ASTColumnsReplaceTransformer::Replacement::formatImpl(
void ASTColumnsReplaceTransformer::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
{
settings.ostr << (settings.hilite ? hilite_keyword : "") << "REPLACE" << (settings.hilite ? hilite_none : "") << "(";
settings.ostr << (settings.hilite ? hilite_keyword : "");
settings.ostr << (is_strict ? "REPLACESTRICT" : "REPLACE");
settings.ostr << (settings.hilite ? hilite_none : "") << "(";
for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it)
{
@ -131,7 +154,6 @@ void ASTColumnsReplaceTransformer::transform(ASTs & nodes) const
replace_map.emplace(replacement.name, replacement.expr);
}
UInt8 replace_column_sucess = 0;
for (auto & column : nodes)
{
if (const auto * id = column->as<ASTIdentifier>())
@ -141,7 +163,7 @@ void ASTColumnsReplaceTransformer::transform(ASTs & nodes) const
{
column = replace_it->second;
column->setAlias(replace_it->first);
++replace_column_sucess;
replace_map.erase(replace_it);
}
}
else if (auto * ast_with_alias = dynamic_cast<ASTWithAlias *>(column.get()))
@ -154,15 +176,25 @@ void ASTColumnsReplaceTransformer::transform(ASTs & nodes) const
replaceChildren(new_ast, column, replace_it->first);
column = new_ast;
column->setAlias(replace_it->first);
++replace_column_sucess;
replace_map.erase(replace_it);
}
}
}
if (replace_column_sucess < replace_map.size())
if (is_strict && !replace_map.empty())
{
String unexisted_columns = "";
for (auto it = replace_map.begin(); it != replace_map.end(); ++it)
{
if (unexisted_columns != "")
unexisted_columns += ", ";
unexisted_columns += it->first;
}
throw Exception(
"Expressions in columns transformer REPLACE should use same column name as original column",
"Columns transformer REPLACESTRICT process unexist column : " + unexisted_columns,
ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
}
}
}

2
src/Parsers/ASTColumnsTransformers.h
View File

@ -30,6 +30,7 @@ protected:
class ASTColumnsExceptTransformer : public IASTColumnsTransformer
{
public:
bool is_strict = false;
String getID(char) const override { return "ColumnsExceptTransformer"; }
ASTPtr clone() const override
{
@ -66,6 +67,7 @@ public:
void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override;
};
bool is_strict = false;
String getID(char) const override { return "ColumnsReplaceTransformer"; }
ASTPtr clone() const override
{

28
src/Parsers/ExpressionElementParsers.cpp
View File

@ -1236,6 +1236,10 @@ bool ParserColumnsTransformers::parseImpl(Pos & pos, ASTPtr & node, Expected & e
ParserKeyword except("EXCEPT");
ParserKeyword replace("REPLACE");
ParserKeyword as("AS");
ParserKeyword except_strict("EXCEPTSTRICT");
ParserKeyword replace_strict("REPLACESTRICT");
bool is_except = false;
bool is_replace = false;
if (apply.ignore(pos, expected))
{
@ -1256,7 +1260,26 @@ bool ParserColumnsTransformers::parseImpl(Pos & pos, ASTPtr & node, Expected & e
node = std::move(res);
return true;
}
else if (except_strict.ignore(pos, expected))
{
is_except = true;
is_strict = true;
}
else if (except.ignore(pos, expected))
{
is_except = true;
}
else if (replace_strict.ignore(pos, expected))
{
is_replace = true;
is_strict = true;
}
else if (replace.ignore(pos, expected))
{
is_replace = true;
}
if (is_except)
{
if (pos->type != TokenType::OpeningRoundBracket)
return false;
@ -1282,11 +1305,13 @@ bool ParserColumnsTransformers::parseImpl(Pos & pos, ASTPtr & node, Expected & e
auto res = std::make_shared<ASTColumnsExceptTransformer>();
res->children = std::move(identifiers);
res->is_strict = is_strict;
node = std::move(res);
return true;
}
else if (replace.ignore(pos, expected))
else if (is_replace)
{
if (pos->type != TokenType::OpeningRoundBracket)
return false;
++pos;
@ -1323,6 +1348,7 @@ bool ParserColumnsTransformers::parseImpl(Pos & pos, ASTPtr & node, Expected & e
auto res = std::make_shared<ASTColumnsReplaceTransformer>();
res->children = std::move(replacements);
res->is_strict = is_strict;
node = std::move(res);
return true;
}

3
src/Parsers/ExpressionElementParsers.h
View File

@ -92,9 +92,12 @@ protected:
*/
class ParserColumnsTransformers : public IParserBase
{
public:
ParserColumnsTransformers(bool is_strict_ = false): is_strict(is_strict_) {}
protected:
const char * getName() const override { return "COLUMNS transformers"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
bool is_strict;
};
/** A function, for example, f(x, y + 1, g(z)).

6
tests/queries/0_stateless/01470_columns_transformers.reference
View File

@ -8,6 +8,7 @@
1970-04-11 1970-01-11 1970-11-21
222 18 347
111 11 173.5
1970-04-11 1970-01-11 1970-11-21
SELECT
sum(i),
sum(j),
@ -50,6 +51,11 @@ SELECT
avg(j + 2 AS j),
avg(k)
FROM columns_transformers
SELECT
toDate(any(i)),
toDate(any(j)),
toDate(any(k))
FROM columns_transformers AS a
SELECT
(i + 1) + 1 AS i,
j,

7
tests/queries/0_stateless/01470_columns_transformers.sql
View File

@ -13,12 +13,14 @@ SELECT columns_transformers.* EXCEPT(j) APPLY(avg) from columns_transformers;
-- EXCEPT after APPLY will not match anything
SELECT a.* APPLY(toDate) EXCEPT(i, j) APPLY(any) from columns_transformers a;
SELECT * REPLACE(i + 1 AS col) from columns_transformers; -- { serverError 16 }
SELECT * EXCEPTSTRICT(i, j1) from columns_transformers; -- { serverError 16 }
SELECT * REPLACESTRICT(i + 1 AS col) from columns_transformers; -- { serverError 16 }
SELECT * REPLACE(i + 1 AS i) APPLY(sum) from columns_transformers;
SELECT columns_transformers.* REPLACE(j + 2 AS j, i + 1 AS i) APPLY(avg) from columns_transformers;
SELECT columns_transformers.* REPLACE(j + 1 AS j, j + 2 AS j) APPLY(avg) from columns_transformers; -- { serverError 43 }
-- REPLACE after APPLY will not match anything
SELECT a.* APPLY(toDate) REPLACE(i + 1 AS i) APPLY(any) from columns_transformers a; -- { serverError 16 }
SELECT a.* APPLY(toDate) REPLACE(i + 1 AS i) APPLY(any) from columns_transformers a;
SELECT a.* APPLY(toDate) REPLACESTRICT(i + 1 AS i) APPLY(any) from columns_transformers a; -- { serverError 16 }
EXPLAIN SYNTAX SELECT * APPLY(sum) from columns_transformers;
EXPLAIN SYNTAX SELECT columns_transformers.* APPLY(avg) from columns_transformers;
@ -29,6 +31,7 @@ EXPLAIN SYNTAX SELECT columns_transformers.* EXCEPT(j) APPLY(avg) from columns_t
EXPLAIN SYNTAX SELECT a.* APPLY(toDate) EXCEPT(i, j) APPLY(any) from columns_transformers a;
EXPLAIN SYNTAX SELECT * REPLACE(i + 1 AS i) APPLY(sum) from columns_transformers;
EXPLAIN SYNTAX SELECT columns_transformers.* REPLACE(j + 2 AS j, i + 1 AS i) APPLY(avg) from columns_transformers;
EXPLAIN SYNTAX SELECT a.* APPLY(toDate) REPLACE(i + 1 AS i) APPLY(any) from columns_transformers a;
-- Multiple REPLACE in a row
EXPLAIN SYNTAX SELECT * REPLACE(i + 1 AS i) REPLACE(i + 1 AS i) from columns_transformers;