diff --git a/dbms/src/Formats/ConstantExpressionTemplate.cpp b/dbms/src/Formats/ConstantExpressionTemplate.cpp index 8ff31723a17..d18e1e2fd8f 100644 --- a/dbms/src/Formats/ConstantExpressionTemplate.cpp +++ b/dbms/src/Formats/ConstantExpressionTemplate.cpp @@ -1,4 +1,5 @@ +#include #include #include #include @@ -11,6 +12,8 @@ #include #include #include +#include +#include namespace DB { @@ -26,7 +29,7 @@ namespace ErrorCodes ConstantExpressionTemplate::ConstantExpressionTemplate(const IDataType & result_column_type, TokenIterator begin, TokenIterator end, const Context & context) { - std::pair expr_template = replaceLiteralsWithDummyIdentifiers(begin, end); + std::pair expr_template = replaceLiteralsWithDummyIdentifiers(begin, end, result_column_type); for (const auto & col : expr_template.second) literals.insert({nullptr, col.type, col.name}); columns = literals.cloneEmptyColumns(); @@ -43,6 +46,7 @@ ConstantExpressionTemplate::ConstantExpressionTemplate(const IDataType & result_ addNodesToCastResult(result_column_type, ast_template); result_column_name = ast_template->getColumnName(); + // TODO convert SyntaxAnalyzer and ExpressionAnalyzer exceptions to CANNOT_CREATE_EXPRESSION_TEMPLATE auto syntax_result = SyntaxAnalyzer(context).analyze(ast_template, expr_template.second); actions_on_literals = ExpressionAnalyzer(ast_template, syntax_result, context).getActions(false); @@ -73,6 +77,7 @@ void ConstantExpressionTemplate::parseExpression(ReadBuffer & istr, const Format skipWhitespaceIfAny(istr); assertString(tokens[cur_token++], istr); } + ++rows_count; } catch (DB::Exception & e) { @@ -90,6 +95,8 @@ ColumnPtr ConstantExpressionTemplate::evaluateAll() { Block evaluated = literals.cloneWithColumns(std::move(columns)); columns = literals.cloneEmptyColumns(); + if (!literals.columns()) + evaluated.insert({ColumnConst::create(ColumnUInt8::create(1, 0), rows_count), std::make_shared(), "_dummy"}); actions_on_literals->execute(evaluated); if (!evaluated || evaluated.rows() == 0) @@ -104,10 +111,9 @@ ColumnPtr ConstantExpressionTemplate::evaluateAll() } std::pair -ConstantExpressionTemplate::replaceLiteralsWithDummyIdentifiers(TokenIterator & begin, TokenIterator & end) +ConstantExpressionTemplate::replaceLiteralsWithDummyIdentifiers(TokenIterator & begin, TokenIterator & end, const IDataType & result_column_type) { NamesAndTypesList dummy_columns; - ParserLiteral parser; String result; size_t token_idx = 0; while (begin != end) @@ -116,21 +122,46 @@ ConstantExpressionTemplate::replaceLiteralsWithDummyIdentifiers(TokenIterator & if (t.isError()) throw DB::Exception("Error in tokens", ErrorCodes::CANNOT_CREATE_EXPRESSION_TEMPLATE); - // TODO don't convert constant string arguments of functions such as CAST(x, 'type') - // TODO process Array as one literal to make possible parsing constant arrays of different size - if (t.type == TokenType::Number || t.type == TokenType::StringLiteral) - { - Expected expected; - ASTPtr ast; - if (!parser.parse(begin, ast, expected)) - throw DB::Exception("Cannot determine literal type", ErrorCodes::CANNOT_CREATE_EXPRESSION_TEMPLATE); + Expected expected; + ASTPtr ast; + DataTypePtr type; - // TODO use nullable type if necessary (e.g. value is not NULL, but result_column_type is nullable and next rows may contain NULLs) - // TODO parse numbers more carefully: sign is a separate token before number + // TODO don't convert constant arguments of functions such as CAST(x, 'type') + + if (t.type == TokenType::Number) + { + ParserNumber parser; + if (!parser.parse(begin, ast, expected)) + throw DB::Exception("Cannot determine literal type: " + String(t.begin, t.size()), ErrorCodes::CANNOT_CREATE_EXPRESSION_TEMPLATE); Field & value = ast->as().value; - DataTypePtr type = DataTypeFactory::instance().get(value.getTypeName()); + // TODO parse numbers more carefully: distinguish unary and binary sign + type = DataTypeFactory::instance().get(value.getTypeName()); + } + else if (t.type == TokenType::StringLiteral) + { + type = std::make_shared(); + } + else if (t.type == TokenType::OpeningSquareBracket) + { + ParserArrayOfLiterals parser; + if (parser.parse(begin, ast, expected)) + { + Field & value = ast->as().value; + type = DataTypeFactory::instance().get(value.getTypeName()); + } + } + + if (type) + { + /// Allow literal to be NULL, if result column has nullable type + // TODO also allow NULL literals inside functions, which return not NULL for NULL arguments, + // even if result_column_type is not nullable + if (result_column_type.isNullable()) + type = makeNullable(type); + // TODO ensure dummy_col_name is unique (there was no _dummy_x identifier in expression) String dummy_col_name = "_dummy_" + std::to_string(dummy_columns.size()); + dummy_columns.push_back(NameAndTypePair(dummy_col_name, type)); token_after_literal_idx.push_back(token_idx); result.append(dummy_col_name); @@ -144,8 +175,6 @@ ConstantExpressionTemplate::replaceLiteralsWithDummyIdentifiers(TokenIterator & } result.append(" "); } - if (dummy_columns.empty()) // TODO - throw DB::Exception("not implemented yet", ErrorCodes::CANNOT_CREATE_EXPRESSION_TEMPLATE); return std::make_pair(result, dummy_columns); } diff --git a/dbms/src/Formats/ConstantExpressionTemplate.h b/dbms/src/Formats/ConstantExpressionTemplate.h index 37038244da5..4b448fb2b51 100644 --- a/dbms/src/Formats/ConstantExpressionTemplate.h +++ b/dbms/src/Formats/ConstantExpressionTemplate.h @@ -18,7 +18,7 @@ public: ColumnPtr evaluateAll(); private: - std::pair replaceLiteralsWithDummyIdentifiers(TokenIterator & begin, TokenIterator & end); + std::pair replaceLiteralsWithDummyIdentifiers(TokenIterator & begin, TokenIterator & end, const IDataType & result_column_type); static void addNodesToCastResult(const IDataType & result_column_type, ASTPtr & expr); @@ -30,6 +30,9 @@ private: Block literals; MutableColumns columns; + /// For expressions without literals (e.g. "now()") + size_t rows_count = 0; + }; }