Improvements

dbms/src/Formats/ConstantExpressionTemplate.cpp

@@ -1,4 +1,5 @@
 
+#include <Columns/ColumnConst.h>
 #include <DataTypes/DataTypeFactory.h>
 #include <Formats/BlockInputStreamFromRowInputStream.h>
 #include <Interpreters/ExpressionAnalyzer.h>
@@ -11,6 +12,8 @@
 #include <Parsers/ExpressionElementParsers.h>
 #include <Parsers/ExpressionListParsers.h>
 #include <DataTypes/DataTypesNumber.h>
+#include <DataTypes/DataTypeNullable.h>
+#include <DataTypes/DataTypeString.h>
 
 namespace DB
 {
@@ -26,7 +29,7 @@ namespace ErrorCodes
 ConstantExpressionTemplate::ConstantExpressionTemplate(const IDataType & result_column_type, TokenIterator begin, TokenIterator end,
                                                        const Context & context)
 {
-    std::pair<String, NamesAndTypesList> expr_template = replaceLiteralsWithDummyIdentifiers(begin, end);
+    std::pair<String, NamesAndTypesList> expr_template = replaceLiteralsWithDummyIdentifiers(begin, end, result_column_type);
     for (const auto & col : expr_template.second)
         literals.insert({nullptr, col.type, col.name});
     columns = literals.cloneEmptyColumns();
@@ -43,6 +46,7 @@ ConstantExpressionTemplate::ConstantExpressionTemplate(const IDataType & result_
     addNodesToCastResult(result_column_type, ast_template);
     result_column_name = ast_template->getColumnName();
 
+    // TODO convert SyntaxAnalyzer and ExpressionAnalyzer exceptions to CANNOT_CREATE_EXPRESSION_TEMPLATE
     auto syntax_result = SyntaxAnalyzer(context).analyze(ast_template, expr_template.second);
 
     actions_on_literals = ExpressionAnalyzer(ast_template, syntax_result, context).getActions(false);
@@ -73,6 +77,7 @@ void ConstantExpressionTemplate::parseExpression(ReadBuffer & istr, const Format
             skipWhitespaceIfAny(istr);
             assertString(tokens[cur_token++], istr);
         }
+        ++rows_count;
     }
     catch (DB::Exception & e)
     {
@@ -90,6 +95,8 @@ ColumnPtr ConstantExpressionTemplate::evaluateAll()
 {
     Block evaluated = literals.cloneWithColumns(std::move(columns));
     columns = literals.cloneEmptyColumns();
+    if (!literals.columns())
+        evaluated.insert({ColumnConst::create(ColumnUInt8::create(1, 0), rows_count), std::make_shared<DataTypeUInt8>(), "_dummy"});
     actions_on_literals->execute(evaluated);
 
     if (!evaluated || evaluated.rows() == 0)
@@ -104,10 +111,9 @@ ColumnPtr ConstantExpressionTemplate::evaluateAll()
 }
 
 std::pair<String, NamesAndTypesList>
-ConstantExpressionTemplate::replaceLiteralsWithDummyIdentifiers(TokenIterator & begin, TokenIterator & end)
+ConstantExpressionTemplate::replaceLiteralsWithDummyIdentifiers(TokenIterator & begin, TokenIterator & end, const IDataType & result_column_type)
 {
     NamesAndTypesList dummy_columns;
-    ParserLiteral parser;
     String result;
     size_t token_idx = 0;
     while (begin != end)
@@ -116,21 +122,46 @@ ConstantExpressionTemplate::replaceLiteralsWithDummyIdentifiers(TokenIterator &
         if (t.isError())
             throw DB::Exception("Error in tokens", ErrorCodes::CANNOT_CREATE_EXPRESSION_TEMPLATE);
 
-        // TODO don't convert constant string arguments of functions such as CAST(x, 'type')
-        // TODO process Array as one literal to make possible parsing constant arrays of different size
-        if (t.type == TokenType::Number || t.type == TokenType::StringLiteral)
-        {
         Expected expected;
         ASTPtr ast;
-            if (!parser.parse(begin, ast, expected))
+        DataTypePtr type;
-                throw DB::Exception("Cannot determine literal type", ErrorCodes::CANNOT_CREATE_EXPRESSION_TEMPLATE);
 
-            // TODO use nullable type if necessary (e.g. value is not NULL, but result_column_type is nullable and next rows may contain NULLs)
+        // TODO don't convert constant arguments of functions such as CAST(x, 'type')
-            // TODO parse numbers more carefully: sign is a separate token before number
+
+        if (t.type == TokenType::Number)
+        {
+            ParserNumber parser;
+            if (!parser.parse(begin, ast, expected))
+                throw DB::Exception("Cannot determine literal type: " + String(t.begin, t.size()), ErrorCodes::CANNOT_CREATE_EXPRESSION_TEMPLATE);
             Field & value = ast->as<ASTLiteral &>().value;
-            DataTypePtr type = DataTypeFactory::instance().get(value.getTypeName());
+            // TODO parse numbers more carefully: distinguish unary and binary sign
+            type = DataTypeFactory::instance().get(value.getTypeName());
+        }
+        else if (t.type == TokenType::StringLiteral)
+        {
+            type = std::make_shared<DataTypeString>();
+        }
+        else if (t.type == TokenType::OpeningSquareBracket)
+        {
+            ParserArrayOfLiterals parser;
+            if (parser.parse(begin, ast, expected))
+            {
+                Field & value = ast->as<ASTLiteral &>().value;
+                type = DataTypeFactory::instance().get(value.getTypeName());
+            }
+        }
+
+        if (type)
+        {
+            /// Allow literal to be NULL, if result column has nullable type
+            // TODO also allow NULL literals inside functions, which return not NULL for NULL arguments,
+            //  even if result_column_type is not nullable
+            if (result_column_type.isNullable())
+                type = makeNullable(type);
+
             // TODO ensure dummy_col_name is unique (there was no _dummy_x identifier in expression)
             String dummy_col_name = "_dummy_" + std::to_string(dummy_columns.size());
+
             dummy_columns.push_back(NameAndTypePair(dummy_col_name, type));
             token_after_literal_idx.push_back(token_idx);
             result.append(dummy_col_name);
@@ -144,8 +175,6 @@ ConstantExpressionTemplate::replaceLiteralsWithDummyIdentifiers(TokenIterator &
         }
         result.append(" ");
     }
-    if (dummy_columns.empty())  // TODO
-        throw DB::Exception("not implemented yet", ErrorCodes::CANNOT_CREATE_EXPRESSION_TEMPLATE);
     return std::make_pair(result, dummy_columns);
 }
 

dbms/src/Formats/ConstantExpressionTemplate.h

@@ -18,7 +18,7 @@ public:
     ColumnPtr evaluateAll();
 
 private:
-    std::pair<String, NamesAndTypesList> replaceLiteralsWithDummyIdentifiers(TokenIterator & begin, TokenIterator & end);
+    std::pair<String, NamesAndTypesList> replaceLiteralsWithDummyIdentifiers(TokenIterator & begin, TokenIterator & end, const IDataType & result_column_type);
 
     static void addNodesToCastResult(const IDataType & result_column_type, ASTPtr & expr);
 
@@ -30,6 +30,9 @@ private:
     Block literals;
     MutableColumns columns;
 
+    /// For expressions without literals (e.g. "now()")
+    size_t rows_count = 0;
+
 };
 
 }