Introduced HereDoc token

programs/client/Client.cpp

@@ -422,6 +422,7 @@ private:
                {TokenType::Semicolon, Replxx::Color::INTENSE},
                {TokenType::Dot, Replxx::Color::INTENSE},
                {TokenType::Asterisk, Replxx::Color::INTENSE},
+               {TokenType::HereDoc, Replxx::Color::BLUE},
                {TokenType::Plus, Replxx::Color::INTENSE},
                {TokenType::Minus, Replxx::Color::INTENSE},
                {TokenType::Slash, Replxx::Color::INTENSE},
@@ -447,8 +448,7 @@ private:
                {TokenType::ErrorDoubleQuoteIsNotClosed, Replxx::Color::RED},
                {TokenType::ErrorSinglePipeMark, Replxx::Color::RED},
                {TokenType::ErrorWrongNumber, Replxx::Color::RED},
-               { TokenType::ErrorMaxQuerySizeExceeded,
+               {TokenType::ErrorMaxQuerySizeExceeded, Replxx::Color::RED }};
-                 Replxx::Color::RED }};
 
         const Replxx::Color unknown_token_color = Replxx::Color::RED;
 

src/Parsers/ExpressionElementParsers.cpp

@@ -1555,26 +1555,36 @@ bool ParserUnsignedInteger::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
 
 bool ParserStringLiteral::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 {
-    if (pos->type != TokenType::StringLiteral)
+    if (pos->type != TokenType::StringLiteral && pos->type != TokenType::HereDoc)
         return false;
 
     String s;
-    ReadBufferFromMemory in(pos->begin, pos->size());
 
-    try
+    if (pos->type == TokenType::StringLiteral)
     {
-        readQuotedStringWithSQLStyle(s, in);
+        ReadBufferFromMemory in(pos->begin, pos->size());
-    }
-    catch (const Exception &)
-    {
-        expected.add(pos, "string literal");
-        return false;
-    }
 
-    if (in.count() != pos->size())
+        try
+        {
+            readQuotedStringWithSQLStyle(s, in);
+        }
+        catch (const Exception &)
+        {
+            expected.add(pos, "string literal");
+            return false;
+        }
+
+        if (in.count() != pos->size())
+        {
+            expected.add(pos, "string literal");
+            return false;
+        }
+    }
+    else if (pos->type == TokenType::HereDoc)
     {
-        expected.add(pos, "string literal");
+        std::string_view here_doc(pos->begin, pos->size());
-        return false;
+        size_t heredoc_size = here_doc.find('$', 1) + 1;
+        s = String(pos->begin + heredoc_size, pos->size() - heredoc_size * 2);
     }
 
     auto literal = std::make_shared<ASTLiteral>(s);

src/Parsers/ExpressionElementParsers.h

@@ -308,6 +308,7 @@ protected:
 
 
 /** String in single quotes.
+  * String in heredoc $here$txt$here$ equivalent to 'txt'.
   */
 class ParserStringLiteral : public IParserBase
 {

src/Parsers/Lexer.cpp

@@ -1,7 +1,7 @@
 #include <Parsers/Lexer.h>
 #include <Common/StringUtils/StringUtils.h>
 #include <common/find_symbols.h>
-
+#include <iostream>
 
 namespace DB
 {
@@ -338,10 +338,34 @@ Token Lexer::nextTokenImpl()
         }
 
         default:
-            if (*pos == '$' && ((pos + 1 < end && !isWordCharASCII(pos[1])) || pos + 1 == end))
+            if (*pos == '$')
             {
-                /// Capture standalone dollar sign
+                if (((pos + 1 < end && !isWordCharASCII(pos[1])) || pos + 1 == end))
-                return Token(TokenType::DollarSign, token_begin, ++pos);
+                {
+                    /// Capture standalone dollar sign
+                    return Token(TokenType::DollarSign, token_begin, ++pos);
+                }
+                else
+                {
+                    std::string_view token_stream(pos, end - pos);
+
+                    auto heredoc_name_end_position = token_stream.find('$', 1);
+                    if (heredoc_name_end_position != std::string::npos)
+                    {
+                        size_t heredoc_size = heredoc_name_end_position + 1;
+                        std::string_view heredoc = {token_stream.data(), heredoc_size};
+
+                        size_t heredoc_end_position = token_stream.find(heredoc, heredoc_size + 1);
+                        if (heredoc_end_position != std::string::npos)
+                        {
+
+                            pos += heredoc_end_position;
+                            pos += heredoc_size;
+
+                            return Token(TokenType::HereDoc, token_begin, pos);
+                        }
+                    }
+                }
             }
             if (isWordCharASCII(*pos) || *pos == '$')
             {

src/Parsers/Lexer.h

@@ -33,6 +33,8 @@ namespace DB
     \
     M(Asterisk)               /** Could be used as multiplication operator or on it's own: "SELECT *" */ \
     \
+    M(HereDoc) \
+    \
     M(DollarSign) \
     M(Plus) \
     M(Minus) \