Improvements

This commit is contained in:
Alexander Tokmakov 2019-05-18 06:21:39 +03:00 committed by Alexander Tokmakov
parent 6fd1319bba
commit 6e06658363
2 changed files with 49 additions and 17 deletions

View File

@ -1,4 +1,5 @@
#include <Columns/ColumnConst.h>
#include <DataTypes/DataTypeFactory.h>
#include <Formats/BlockInputStreamFromRowInputStream.h>
#include <Interpreters/ExpressionAnalyzer.h>
@ -11,6 +12,8 @@
#include <Parsers/ExpressionElementParsers.h>
#include <Parsers/ExpressionListParsers.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypeString.h>
namespace DB
{
@ -26,7 +29,7 @@ namespace ErrorCodes
ConstantExpressionTemplate::ConstantExpressionTemplate(const IDataType & result_column_type, TokenIterator begin, TokenIterator end,
const Context & context)
{
std::pair<String, NamesAndTypesList> expr_template = replaceLiteralsWithDummyIdentifiers(begin, end);
std::pair<String, NamesAndTypesList> expr_template = replaceLiteralsWithDummyIdentifiers(begin, end, result_column_type);
for (const auto & col : expr_template.second)
literals.insert({nullptr, col.type, col.name});
columns = literals.cloneEmptyColumns();
@ -43,6 +46,7 @@ ConstantExpressionTemplate::ConstantExpressionTemplate(const IDataType & result_
addNodesToCastResult(result_column_type, ast_template);
result_column_name = ast_template->getColumnName();
// TODO convert SyntaxAnalyzer and ExpressionAnalyzer exceptions to CANNOT_CREATE_EXPRESSION_TEMPLATE
auto syntax_result = SyntaxAnalyzer(context).analyze(ast_template, expr_template.second);
actions_on_literals = ExpressionAnalyzer(ast_template, syntax_result, context).getActions(false);
@ -73,6 +77,7 @@ void ConstantExpressionTemplate::parseExpression(ReadBuffer & istr, const Format
skipWhitespaceIfAny(istr);
assertString(tokens[cur_token++], istr);
}
++rows_count;
}
catch (DB::Exception & e)
{
@ -90,6 +95,8 @@ ColumnPtr ConstantExpressionTemplate::evaluateAll()
{
Block evaluated = literals.cloneWithColumns(std::move(columns));
columns = literals.cloneEmptyColumns();
if (!literals.columns())
evaluated.insert({ColumnConst::create(ColumnUInt8::create(1, 0), rows_count), std::make_shared<DataTypeUInt8>(), "_dummy"});
actions_on_literals->execute(evaluated);
if (!evaluated || evaluated.rows() == 0)
@ -104,10 +111,9 @@ ColumnPtr ConstantExpressionTemplate::evaluateAll()
}
std::pair<String, NamesAndTypesList>
ConstantExpressionTemplate::replaceLiteralsWithDummyIdentifiers(TokenIterator & begin, TokenIterator & end)
ConstantExpressionTemplate::replaceLiteralsWithDummyIdentifiers(TokenIterator & begin, TokenIterator & end, const IDataType & result_column_type)
{
NamesAndTypesList dummy_columns;
ParserLiteral parser;
String result;
size_t token_idx = 0;
while (begin != end)
@ -116,21 +122,46 @@ ConstantExpressionTemplate::replaceLiteralsWithDummyIdentifiers(TokenIterator &
if (t.isError())
throw DB::Exception("Error in tokens", ErrorCodes::CANNOT_CREATE_EXPRESSION_TEMPLATE);
// TODO don't convert constant string arguments of functions such as CAST(x, 'type')
// TODO process Array as one literal to make possible parsing constant arrays of different size
if (t.type == TokenType::Number || t.type == TokenType::StringLiteral)
{
Expected expected;
ASTPtr ast;
if (!parser.parse(begin, ast, expected))
throw DB::Exception("Cannot determine literal type", ErrorCodes::CANNOT_CREATE_EXPRESSION_TEMPLATE);
Expected expected;
ASTPtr ast;
DataTypePtr type;
// TODO use nullable type if necessary (e.g. value is not NULL, but result_column_type is nullable and next rows may contain NULLs)
// TODO parse numbers more carefully: sign is a separate token before number
// TODO don't convert constant arguments of functions such as CAST(x, 'type')
if (t.type == TokenType::Number)
{
ParserNumber parser;
if (!parser.parse(begin, ast, expected))
throw DB::Exception("Cannot determine literal type: " + String(t.begin, t.size()), ErrorCodes::CANNOT_CREATE_EXPRESSION_TEMPLATE);
Field & value = ast->as<ASTLiteral &>().value;
DataTypePtr type = DataTypeFactory::instance().get(value.getTypeName());
// TODO parse numbers more carefully: distinguish unary and binary sign
type = DataTypeFactory::instance().get(value.getTypeName());
}
else if (t.type == TokenType::StringLiteral)
{
type = std::make_shared<DataTypeString>();
}
else if (t.type == TokenType::OpeningSquareBracket)
{
ParserArrayOfLiterals parser;
if (parser.parse(begin, ast, expected))
{
Field & value = ast->as<ASTLiteral &>().value;
type = DataTypeFactory::instance().get(value.getTypeName());
}
}
if (type)
{
/// Allow literal to be NULL, if result column has nullable type
// TODO also allow NULL literals inside functions, which return not NULL for NULL arguments,
// even if result_column_type is not nullable
if (result_column_type.isNullable())
type = makeNullable(type);
// TODO ensure dummy_col_name is unique (there was no _dummy_x identifier in expression)
String dummy_col_name = "_dummy_" + std::to_string(dummy_columns.size());
dummy_columns.push_back(NameAndTypePair(dummy_col_name, type));
token_after_literal_idx.push_back(token_idx);
result.append(dummy_col_name);
@ -144,8 +175,6 @@ ConstantExpressionTemplate::replaceLiteralsWithDummyIdentifiers(TokenIterator &
}
result.append(" ");
}
if (dummy_columns.empty()) // TODO
throw DB::Exception("not implemented yet", ErrorCodes::CANNOT_CREATE_EXPRESSION_TEMPLATE);
return std::make_pair(result, dummy_columns);
}

View File

@ -18,7 +18,7 @@ public:
ColumnPtr evaluateAll();
private:
std::pair<String, NamesAndTypesList> replaceLiteralsWithDummyIdentifiers(TokenIterator & begin, TokenIterator & end);
std::pair<String, NamesAndTypesList> replaceLiteralsWithDummyIdentifiers(TokenIterator & begin, TokenIterator & end, const IDataType & result_column_type);
static void addNodesToCastResult(const IDataType & result_column_type, ASTPtr & expr);
@ -30,6 +30,9 @@ private:
Block literals;
MutableColumns columns;
/// For expressions without literals (e.g. "now()")
size_t rows_count = 0;
};
}