Combining our lexer with replxx

base/base/ReplxxLineReader.cpp

@@ -25,13 +25,6 @@ void trim(String & s)
     s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { return !std::isspace(ch); }).base(), s.end());
 }
 
-/// Check if string ends with given character after skipping whitespaces.
-bool ends_with(const std::string_view & s, const std::string_view & p)
-{
-    auto ss = std::string_view(s.data(), s.rend() - std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { return !std::isspace(ch); }));
-    return ss.ends_with(p);
-}
-
 std::string getEditor()
 {
     const char * editor = std::getenv("EDITOR");
@@ -132,6 +125,7 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx)
 
 }
 
+bool replxx_last_is_delimiter = false;
 ReplxxLineReader::ReplxxLineReader(
     Suggest & suggest,
     const String & history_file_path_,
@@ -196,21 +190,10 @@ ReplxxLineReader::ReplxxLineReader(
 
     auto commit_action = [this](char32_t code)
     {
-        std::string_view str = rx.get_state().text();
-
-        /// Always commit line when we see extender at the end. It will start a new prompt.
-        for (const auto * extender : extenders)
-            if (ends_with(str, extender))
-                return rx.invoke(Replxx::ACTION::COMMIT_LINE, code);
-
-        /// If we see an delimiter at the end, commit right away.
-        for (const auto * delimiter : delimiters)
-            if (ends_with(str, delimiter))
-                return rx.invoke(Replxx::ACTION::COMMIT_LINE, code);
-
         /// If we allow multiline and there is already something in the input, start a newline.
-        if (multiline && !input.empty())
+        if (multiline && !replxx_last_is_delimiter)
             return rx.invoke(Replxx::ACTION::NEW_LINE, code);
+        replxx_last_is_delimiter = false;
         return rx.invoke(Replxx::ACTION::COMMIT_LINE, code);
     };
     /// bind C-j to ENTER action.

src/Client/ClientBaseHelpers.cpp

@@ -6,6 +6,9 @@
 #include <Parsers/Lexer.h>
 #include <Common/UTF8Helpers.h>
 
+#if USE_REPLXX
+extern bool replxx_last_is_delimiter;
+#endif
 
 namespace DB
 {
@@ -114,6 +117,7 @@ void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors
 
             {TokenType::Comma, replxx::color::bold(Replxx::Color::DEFAULT)},
             {TokenType::Semicolon, replxx::color::bold(Replxx::Color::DEFAULT)},
+            {TokenType::VerticalDelimiter, replxx::color::bold(Replxx::Color::DEFAULT)},
             {TokenType::Dot, replxx::color::bold(Replxx::Color::DEFAULT)},
             {TokenType::Asterisk, replxx::color::bold(Replxx::Color::DEFAULT)},
             {TokenType::HereDoc, Replxx::Color::CYAN},
@@ -151,6 +155,11 @@ void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors
 
     for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken())
     {
+        if (token.type == TokenType::Semicolon || token.type == TokenType::VerticalDelimiter)
+            replxx_last_is_delimiter = true;
+        else if (token.type != TokenType::Whitespace)
+            replxx_last_is_delimiter = false;
+
         size_t utf8_len = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(token.begin), token.size());
         for (size_t code_point_index = 0; code_point_index < utf8_len; ++code_point_index)
         {

src/Parsers/Lexer.cpp

@@ -335,6 +335,13 @@ Token Lexer::nextTokenImpl()
                 return Token(TokenType::DoubleAt, token_begin, ++pos);
             return Token(TokenType::At, token_begin, pos);
         }
+        case '\\':
+        {
+            ++pos;
+            if (pos < end && *pos == 'G')
+                return Token(TokenType::VerticalDelimiter, token_begin, ++pos);
+            return Token(TokenType::Error, token_begin, pos);
+        }
 
         default:
             if (*pos == '$')

src/Parsers/Lexer.h

@@ -28,6 +28,7 @@ namespace DB
     \
     M(Comma) \
     M(Semicolon) \
+    M(VerticalDelimiter)      /** Vertical delimiter \G */ \
     M(Dot)                    /** Compound identifiers, like a.b or tuple access operator a.1, (x, y).2. */ \
                               /** Need to be distinguished from floating point number with omitted integer part: .1 */ \
     \