From 54517753d782273cafffd3702b802e8ee8733ab7 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 28 Jan 2022 23:14:54 +0800 Subject: [PATCH 1/6] Combining our lexer with replxx --- base/base/ReplxxLineReader.cpp | 23 +++-------------------- src/Client/ClientBaseHelpers.cpp | 9 +++++++++ src/Parsers/Lexer.cpp | 7 +++++++ src/Parsers/Lexer.h | 1 + 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/base/base/ReplxxLineReader.cpp b/base/base/ReplxxLineReader.cpp index 6ba63a00d01..66cbad99a54 100644 --- a/base/base/ReplxxLineReader.cpp +++ b/base/base/ReplxxLineReader.cpp @@ -25,13 +25,6 @@ void trim(String & s) s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { return !std::isspace(ch); }).base(), s.end()); } -/// Check if string ends with given character after skipping whitespaces. -bool ends_with(const std::string_view & s, const std::string_view & p) -{ - auto ss = std::string_view(s.data(), s.rend() - std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { return !std::isspace(ch); })); - return ss.ends_with(p); -} - std::string getEditor() { const char * editor = std::getenv("EDITOR"); @@ -132,6 +125,7 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx) } +bool replxx_last_is_delimiter = false; ReplxxLineReader::ReplxxLineReader( Suggest & suggest, const String & history_file_path_, @@ -196,21 +190,10 @@ ReplxxLineReader::ReplxxLineReader( auto commit_action = [this](char32_t code) { - std::string_view str = rx.get_state().text(); - - /// Always commit line when we see extender at the end. It will start a new prompt. - for (const auto * extender : extenders) - if (ends_with(str, extender)) - return rx.invoke(Replxx::ACTION::COMMIT_LINE, code); - - /// If we see an delimiter at the end, commit right away. - for (const auto * delimiter : delimiters) - if (ends_with(str, delimiter)) - return rx.invoke(Replxx::ACTION::COMMIT_LINE, code); - /// If we allow multiline and there is already something in the input, start a newline. - if (multiline && !input.empty()) + if (multiline && !replxx_last_is_delimiter) return rx.invoke(Replxx::ACTION::NEW_LINE, code); + replxx_last_is_delimiter = false; return rx.invoke(Replxx::ACTION::COMMIT_LINE, code); }; /// bind C-j to ENTER action. diff --git a/src/Client/ClientBaseHelpers.cpp b/src/Client/ClientBaseHelpers.cpp index 3a5d4f4cf33..1d558159139 100644 --- a/src/Client/ClientBaseHelpers.cpp +++ b/src/Client/ClientBaseHelpers.cpp @@ -6,6 +6,9 @@ #include #include +#if USE_REPLXX +extern bool replxx_last_is_delimiter; +#endif namespace DB { @@ -114,6 +117,7 @@ void highlight(const String & query, std::vector & colors {TokenType::Comma, replxx::color::bold(Replxx::Color::DEFAULT)}, {TokenType::Semicolon, replxx::color::bold(Replxx::Color::DEFAULT)}, + {TokenType::VerticalDelimiter, replxx::color::bold(Replxx::Color::DEFAULT)}, {TokenType::Dot, replxx::color::bold(Replxx::Color::DEFAULT)}, {TokenType::Asterisk, replxx::color::bold(Replxx::Color::DEFAULT)}, {TokenType::HereDoc, Replxx::Color::CYAN}, @@ -151,6 +155,11 @@ void highlight(const String & query, std::vector & colors for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken()) { + if (token.type == TokenType::Semicolon || token.type == TokenType::VerticalDelimiter) + replxx_last_is_delimiter = true; + else if (token.type != TokenType::Whitespace) + replxx_last_is_delimiter = false; + size_t utf8_len = UTF8::countCodePoints(reinterpret_cast(token.begin), token.size()); for (size_t code_point_index = 0; code_point_index < utf8_len; ++code_point_index) { diff --git a/src/Parsers/Lexer.cpp b/src/Parsers/Lexer.cpp index 3ac9d417b46..654174c18f7 100644 --- a/src/Parsers/Lexer.cpp +++ b/src/Parsers/Lexer.cpp @@ -335,6 +335,13 @@ Token Lexer::nextTokenImpl() return Token(TokenType::DoubleAt, token_begin, ++pos); return Token(TokenType::At, token_begin, pos); } + case '\\': + { + ++pos; + if (pos < end && *pos == 'G') + return Token(TokenType::VerticalDelimiter, token_begin, ++pos); + return Token(TokenType::Error, token_begin, pos); + } default: if (*pos == '$') diff --git a/src/Parsers/Lexer.h b/src/Parsers/Lexer.h index f41e05147e5..ec472fb1a36 100644 --- a/src/Parsers/Lexer.h +++ b/src/Parsers/Lexer.h @@ -28,6 +28,7 @@ namespace DB \ M(Comma) \ M(Semicolon) \ + M(VerticalDelimiter) /** Vertical delimiter \G */ \ M(Dot) /** Compound identifiers, like a.b or tuple access operator a.1, (x, y).2. */ \ /** Need to be distinguished from floating point number with omitted integer part: .1 */ \ \ From 666313b51ad15673228658963f8ad18320b473cf Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 28 Jan 2022 23:14:54 +0800 Subject: [PATCH 2/6] Bump replxx --- contrib/replxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/replxx b/contrib/replxx index f019cba7ea1..54a9b16a278 160000 --- a/contrib/replxx +++ b/contrib/replxx @@ -1 +1 @@ -Subproject commit f019cba7ea1bcd1b4feb7826f28ed57fb581b04c +Subproject commit 54a9b16a278b85db58094a5bd495dca3b136dffc From 556cce9f3c36dda73d52f76de0def159484b0afa Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sat, 29 Jan 2022 03:13:01 +0800 Subject: [PATCH 3/6] We need highlight to use Lexer --- base/base/ReplxxLineReader.cpp | 3 ++- .../01293_client_interactive_vertical_multiline.expect | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/base/base/ReplxxLineReader.cpp b/base/base/ReplxxLineReader.cpp index 66cbad99a54..a08acfcfb8a 100644 --- a/base/base/ReplxxLineReader.cpp +++ b/base/base/ReplxxLineReader.cpp @@ -191,7 +191,8 @@ ReplxxLineReader::ReplxxLineReader( auto commit_action = [this](char32_t code) { /// If we allow multiline and there is already something in the input, start a newline. - if (multiline && !replxx_last_is_delimiter) + /// NOTE: Lexer is only available if we use highlighter. + if (highlighter && multiline && !replxx_last_is_delimiter) return rx.invoke(Replxx::ACTION::NEW_LINE, code); replxx_last_is_delimiter = false; return rx.invoke(Replxx::ACTION::COMMIT_LINE, code); diff --git a/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect index e4442047c87..5057ec44e8a 100755 --- a/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect +++ b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect @@ -1,7 +1,7 @@ #!/usr/bin/expect -f log_user 0 -set timeout 60 +set timeout 10 match_max 100000 expect_after { @@ -11,6 +11,9 @@ expect_after { timeout { exit 1 } } +# useful debugging configuration +# exp_internal 1 + set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" expect ":) " @@ -41,7 +44,7 @@ expect ":) " send -- "" expect eof -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --multiline" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --highlight 0 --multiline" expect ":) " send -- "SELECT 1;\r" From 62e89a64459f62b60b974c071f81628af389ea43 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sat, 29 Jan 2022 05:05:14 +0800 Subject: [PATCH 4/6] Ignore case for history search and completion --- base/base/ReplxxLineReader.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/base/base/ReplxxLineReader.cpp b/base/base/ReplxxLineReader.cpp index a08acfcfb8a..1f75eed479b 100644 --- a/base/base/ReplxxLineReader.cpp +++ b/base/base/ReplxxLineReader.cpp @@ -179,6 +179,7 @@ ReplxxLineReader::ReplxxLineReader( rx.set_completion_callback(callback); rx.set_complete_on_empty(false); rx.set_word_break_characters(word_break_characters); + rx.set_ignore_case(true); if (highlighter) rx.set_highlighter_callback(highlighter); From bb34435928a80b61a1b9bdea41a32a8ac435149f Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sat, 29 Jan 2022 19:44:27 +0800 Subject: [PATCH 5/6] Better code --- base/base/ReplxxLineReader.cpp | 7 ++++++- base/base/ReplxxLineReader.h | 3 +++ src/Client/ClientBaseHelpers.cpp | 8 ++------ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/base/base/ReplxxLineReader.cpp b/base/base/ReplxxLineReader.cpp index 1f75eed479b..9ea53bb132b 100644 --- a/base/base/ReplxxLineReader.cpp +++ b/base/base/ReplxxLineReader.cpp @@ -125,7 +125,12 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx) } -bool replxx_last_is_delimiter = false; +static bool replxx_last_is_delimiter = false; +void ReplxxLineReader::setLastIsDelimiter(bool flag) +{ + replxx_last_is_delimiter = flag; +} + ReplxxLineReader::ReplxxLineReader( Suggest & suggest, const String & history_file_path_, diff --git a/base/base/ReplxxLineReader.h b/base/base/ReplxxLineReader.h index 4a975d2975d..b9ec214d02c 100644 --- a/base/base/ReplxxLineReader.h +++ b/base/base/ReplxxLineReader.h @@ -19,6 +19,9 @@ public: void enableBracketedPaste() override; + /// If highlight is on, we will set a flag to denote whether the last token is a delimiter. + /// This is useful to determine the behavior of key when multiline is enabled. + static void setLastIsDelimiter(bool flag); private: InputStatus readOneLine(const String & prompt) override; void addToHistory(const String & line) override; diff --git a/src/Client/ClientBaseHelpers.cpp b/src/Client/ClientBaseHelpers.cpp index 1d558159139..5ad34ba8e81 100644 --- a/src/Client/ClientBaseHelpers.cpp +++ b/src/Client/ClientBaseHelpers.cpp @@ -6,10 +6,6 @@ #include #include -#if USE_REPLXX -extern bool replxx_last_is_delimiter; -#endif - namespace DB { @@ -156,9 +152,9 @@ void highlight(const String & query, std::vector & colors for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken()) { if (token.type == TokenType::Semicolon || token.type == TokenType::VerticalDelimiter) - replxx_last_is_delimiter = true; + ReplxxLineReader::setLastIsDelimiter(true); else if (token.type != TokenType::Whitespace) - replxx_last_is_delimiter = false; + ReplxxLineReader::setLastIsDelimiter(false); size_t utf8_len = UTF8::countCodePoints(reinterpret_cast(token.begin), token.size()); for (size_t code_point_index = 0; code_point_index < utf8_len; ++code_point_index) From ae16b362d39b3ca699c8c2b07c53059b62553327 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sun, 30 Jan 2022 19:46:16 +0800 Subject: [PATCH 6/6] Better case-insensitive completion --- base/base/LineReader.cpp | 2 +- contrib/replxx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/base/LineReader.cpp b/base/base/LineReader.cpp index d028dace52a..686d70f247d 100644 --- a/base/base/LineReader.cpp +++ b/base/base/LineReader.cpp @@ -81,7 +81,7 @@ replxx::Replxx::completions_t LineReader::Suggest::getCompletions(const String & std::lock_guard lock(mutex); /// Only perform case sensitive completion when the prefix string contains any uppercase characters - if (std::none_of(prefix.begin(), prefix.end(), [&](auto c) { return c >= 'A' && c <= 'Z'; })) + if (std::none_of(prefix.begin(), prefix.end(), [](char32_t x) { return iswupper(static_cast(x)); })) range = std::equal_range( words_no_case.begin(), words_no_case.end(), last_word, [prefix_length](std::string_view s, std::string_view prefix_searched) { diff --git a/contrib/replxx b/contrib/replxx index 54a9b16a278..c745b3fb012 160000 --- a/contrib/replxx +++ b/contrib/replxx @@ -1 +1 @@ -Subproject commit 54a9b16a278b85db58094a5bd495dca3b136dffc +Subproject commit c745b3fb012ee5ae762fbc8cd7a40c4dc3fe15df