Merge pull request #34114 from amosbird/replxxwithlexer

Combining our lexer with replxx for better multiple line editing

base/base/LineReader.cpp

@@ -81,7 +81,7 @@ replxx::Replxx::completions_t LineReader::Suggest::getCompletions(const String &
     std::lock_guard lock(mutex);
 
     /// Only perform case sensitive completion when the prefix string contains any uppercase characters
-    if (std::none_of(prefix.begin(), prefix.end(), [&](auto c) { return c >= 'A' && c <= 'Z'; }))
+    if (std::none_of(prefix.begin(), prefix.end(), [](char32_t x) { return iswupper(static_cast<wint_t>(x)); }))
         range = std::equal_range(
             words_no_case.begin(), words_no_case.end(), last_word, [prefix_length](std::string_view s, std::string_view prefix_searched)
             {

base/base/ReplxxLineReader.cpp

@@ -25,13 +25,6 @@ void trim(String & s)
     s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { return !std::isspace(ch); }).base(), s.end());
 }
 
-/// Check if string ends with given character after skipping whitespaces.
-bool ends_with(const std::string_view & s, const std::string_view & p)
-{
-    auto ss = std::string_view(s.data(), s.rend() - std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { return !std::isspace(ch); }));
-    return ss.ends_with(p);
-}
-
 std::string getEditor()
 {
     const char * editor = std::getenv("EDITOR");
@@ -132,6 +125,12 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx)
 
 }
 
+static bool replxx_last_is_delimiter = false;
+void ReplxxLineReader::setLastIsDelimiter(bool flag)
+{
+    replxx_last_is_delimiter = flag;
+}
+
 ReplxxLineReader::ReplxxLineReader(
     Suggest & suggest,
     const String & history_file_path_,
@@ -185,6 +184,7 @@ ReplxxLineReader::ReplxxLineReader(
     rx.set_completion_callback(callback);
     rx.set_complete_on_empty(false);
     rx.set_word_break_characters(word_break_characters);
+    rx.set_ignore_case(true);
 
     if (highlighter)
         rx.set_highlighter_callback(highlighter);
@@ -196,21 +196,11 @@ ReplxxLineReader::ReplxxLineReader(
 
     auto commit_action = [this](char32_t code)
     {
-        std::string_view str = rx.get_state().text();
-
-        /// Always commit line when we see extender at the end. It will start a new prompt.
-        for (const auto * extender : extenders)
-            if (ends_with(str, extender))
-                return rx.invoke(Replxx::ACTION::COMMIT_LINE, code);
-
-        /// If we see an delimiter at the end, commit right away.
-        for (const auto * delimiter : delimiters)
-            if (ends_with(str, delimiter))
-                return rx.invoke(Replxx::ACTION::COMMIT_LINE, code);
-
         /// If we allow multiline and there is already something in the input, start a newline.
-        if (multiline && !input.empty())
+        /// NOTE: Lexer is only available if we use highlighter.
+        if (highlighter && multiline && !replxx_last_is_delimiter)
             return rx.invoke(Replxx::ACTION::NEW_LINE, code);
+        replxx_last_is_delimiter = false;
         return rx.invoke(Replxx::ACTION::COMMIT_LINE, code);
     };
     /// bind C-j to ENTER action.

base/base/ReplxxLineReader.h

@@ -19,6 +19,9 @@ public:
 
     void enableBracketedPaste() override;
 
+    /// If highlight is on, we will set a flag to denote whether the last token is a delimiter.
+    /// This is useful to determine the behavior of <ENTER> key when multiline is enabled.
+    static void setLastIsDelimiter(bool flag);
 private:
     InputStatus readOneLine(const String & prompt) override;
     void addToHistory(const String & line) override;

contrib/replxx

@@ -1 +1 @@
-Subproject commit f019cba7ea1bcd1b4feb7826f28ed57fb581b04c
+Subproject commit c745b3fb012ee5ae762fbc8cd7a40c4dc3fe15df

src/Client/ClientBaseHelpers.cpp

@@ -6,7 +6,6 @@
 #include <Parsers/Lexer.h>
 #include <Common/UTF8Helpers.h>
 
-
 namespace DB
 {
 
@@ -114,6 +113,7 @@ void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors
 
             {TokenType::Comma, replxx::color::bold(Replxx::Color::DEFAULT)},
             {TokenType::Semicolon, replxx::color::bold(Replxx::Color::DEFAULT)},
+            {TokenType::VerticalDelimiter, replxx::color::bold(Replxx::Color::DEFAULT)},
             {TokenType::Dot, replxx::color::bold(Replxx::Color::DEFAULT)},
             {TokenType::Asterisk, replxx::color::bold(Replxx::Color::DEFAULT)},
             {TokenType::HereDoc, Replxx::Color::CYAN},
@@ -151,6 +151,11 @@ void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors
 
     for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken())
     {
+        if (token.type == TokenType::Semicolon || token.type == TokenType::VerticalDelimiter)
+            ReplxxLineReader::setLastIsDelimiter(true);
+        else if (token.type != TokenType::Whitespace)
+            ReplxxLineReader::setLastIsDelimiter(false);
+
         size_t utf8_len = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(token.begin), token.size());
         for (size_t code_point_index = 0; code_point_index < utf8_len; ++code_point_index)
         {

src/Parsers/Lexer.cpp

@@ -335,6 +335,13 @@ Token Lexer::nextTokenImpl()
                 return Token(TokenType::DoubleAt, token_begin, ++pos);
             return Token(TokenType::At, token_begin, pos);
         }
+        case '\\':
+        {
+            ++pos;
+            if (pos < end && *pos == 'G')
+                return Token(TokenType::VerticalDelimiter, token_begin, ++pos);
+            return Token(TokenType::Error, token_begin, pos);
+        }
 
         default:
             if (*pos == '$')

src/Parsers/Lexer.h

@@ -28,6 +28,7 @@ namespace DB
     \
     M(Comma) \
     M(Semicolon) \
+    M(VerticalDelimiter)      /** Vertical delimiter \G */ \
     M(Dot)                    /** Compound identifiers, like a.b or tuple access operator a.1, (x, y).2. */ \
                               /** Need to be distinguished from floating point number with omitted integer part: .1 */ \
     \

tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect

@@ -1,7 +1,7 @@
 #!/usr/bin/expect -f
 
 log_user 0
-set timeout 60
+set timeout 10
 match_max 100000
 
 expect_after {
@@ -11,6 +11,9 @@ expect_after {
     timeout { exit 1 }
 }
 
+# useful debugging configuration
+# exp_internal 1
+
 set basedir [file dirname $argv0]
 spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion"
 expect ":) "
@@ -41,7 +44,7 @@ expect ":) "
 send -- ""
 expect eof
 
-spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --multiline"
+spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --highlight 0 --multiline"
 expect ":) "
 
 send -- "SELECT 1;\r"