Merge pull request #9355 from amosbird/client2

Better cmdline client

.gitmodules

@@ -140,7 +140,7 @@
 	url = https://github.com/ClickHouse-Extras/libc-headers.git
 [submodule "contrib/replxx"]
 	path = contrib/replxx
-	url = https://github.com/AmokHuginnsson/replxx.git
+	url = https://github.com/ClickHouse-Extras/replxx.git
 [submodule "contrib/ryu"]
 	path = contrib/ryu
 	url = https://github.com/ClickHouse-Extras/ryu.git

base/common/CMakeLists.txt

@@ -24,6 +24,11 @@ if (ENABLE_REPLXX)
         ReplxxLineReader.cpp
         ReplxxLineReader.h
     )
+elseif (ENABLE_READLINE)
+    set (SRCS ${SRCS}
+        ReadlineLineReader.cpp
+        ReadlineLineReader.h
+    )
 endif ()
 
 if (USE_DEBUG_HELPERS)
@@ -57,6 +62,28 @@ endif()
 
 target_link_libraries(common PUBLIC replxx)
 
+# allow explicitly fallback to readline
+if (NOT ENABLE_REPLXX AND ENABLE_READLINE)
+    message (STATUS "Attempt to fallback to readline explicitly")
+    set (READLINE_PATHS "/usr/local/opt/readline/lib")
+    # First try find custom lib for macos users (default lib without history support)
+    find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS} NO_DEFAULT_PATH)
+    if (NOT READLINE_LIB)
+        find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS})
+    endif ()
+
+    set(READLINE_INCLUDE_PATHS "/usr/local/opt/readline/include")
+    find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS} NO_DEFAULT_PATH)
+    if (NOT READLINE_INCLUDE_DIR)
+        find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS})
+    endif ()
+    if (READLINE_INCLUDE_DIR AND READLINE_LIB)
+        target_link_libraries(common PUBLIC ${READLINE_LIB})
+        target_compile_definitions(common PUBLIC USE_READLINE=1)
+        message (STATUS "Using readline: ${READLINE_INCLUDE_DIR} : ${READLINE_LIB}")
+    endif ()
+endif ()
+
 target_link_libraries (common
     PUBLIC
         ${Poco_Util_LIBRARY}

base/common/LineReader.cpp

@@ -53,11 +53,18 @@ LineReader::Suggest::WordsRange LineReader::Suggest::getCompletions(const String
 
     /// last_word can be empty.
 
-    return std::equal_range(
-        words.begin(), words.end(), last_word, [prefix_length](std::string_view s, std::string_view prefix_searched)
-        {
-            return strncmp(s.data(), prefix_searched.data(), prefix_length) < 0;
-        });
+    if (case_insensitive)
+        return std::equal_range(
+                words.begin(), words.end(), last_word, [prefix_length](std::string_view s, std::string_view prefix_searched)
+                {
+                    return strncasecmp(s.data(), prefix_searched.data(), prefix_length) < 0;
+                });
+    else
+        return std::equal_range(
+                words.begin(), words.end(), last_word, [prefix_length](std::string_view s, std::string_view prefix_searched)
+                {
+                    return strncmp(s.data(), prefix_searched.data(), prefix_length) < 0;
+                });
 }
 
 LineReader::LineReader(const String & history_file_path_, char extender_, char delimiter_)

base/common/LineReader.h

@@ -8,18 +8,19 @@
 class LineReader
 {
 public:
-    class Suggest
+    struct Suggest
     {
-    protected:
         using Words = std::vector<std::string>;
         using WordsRange = std::pair<Words::const_iterator, Words::const_iterator>;
 
         Words words;
         std::atomic<bool> ready{false};
 
-    public:
         /// Get iterators for the matched range of words if any.
         WordsRange getCompletions(const String & prefix, size_t prefix_length) const;
+
+        /// case sensitive suggestion
+        bool case_insensitive = false;
     };
 
     LineReader(const String & history_file_path, char extender, char delimiter = 0);  /// if delimiter != 0, then it's multiline mode
@@ -31,6 +32,13 @@ public:
     /// Typical delimiter is ';' (semicolon) and typical extender is '\' (backslash).
     String readLine(const String & first_prompt, const String & second_prompt);
 
+    /// When bracketed paste mode is set, pasted text is bracketed with control sequences so
+    /// that the program can differentiate pasted text from typed-in text. This helps
+    /// clickhouse-client so that without -m flag, one can still paste multiline queries, and
+    /// possibly get better pasting performance. See https://cirw.in/blog/bracketed-paste for
+    /// more details.
+    virtual void enableBracketedPaste() {}
+
 protected:
     enum InputStatus
     {

base/common/ReadlineLineReader.cpp

@@ -0,0 +1,173 @@
+#include <common/ReadlineLineReader.h>
+#include <ext/scope_guard.h>
+
+#include <errno.h>
+#include <signal.h>
+#include <string.h>
+#include <unistd.h>
+
+namespace
+{
+
+/// Trim ending whitespace inplace
+void trim(String & s)
+{
+    s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end());
+}
+
+}
+
+static const LineReader::Suggest * suggest;
+
+/// Points to current word to suggest.
+static LineReader::Suggest::Words::const_iterator pos;
+/// Points after the last possible match.
+static LineReader::Suggest::Words::const_iterator end;
+
+/// Set iterators to the matched range of words if any.
+static void findRange(const char * prefix, size_t prefix_length)
+{
+    std::string prefix_str(prefix);
+    std::tie(pos, end) = suggest->getCompletions(prefix_str, prefix_length);
+}
+
+/// Iterates through matched range.
+static char * nextMatch()
+{
+    if (pos >= end)
+        return nullptr;
+
+    /// readline will free memory by itself.
+    char * word = strdup(pos->c_str());
+    ++pos;
+    return word;
+}
+
+static char * generate(const char * text, int state)
+{
+    if (!suggest->ready)
+        return nullptr;
+    if (state == 0)
+        findRange(text, strlen(text));
+
+    /// Do not append whitespace after word. For unknown reason, rl_completion_append_character = '\0' does not work.
+    rl_completion_suppress_append = 1;
+
+    return nextMatch();
+};
+
+ReadlineLineReader::ReadlineLineReader(const Suggest & suggest_, const String & history_file_path_, char extender_, char delimiter_)
+    : LineReader(history_file_path_, extender_, delimiter_)
+{
+    suggest = &suggest_;
+
+    if (!history_file_path.empty())
+    {
+        int res = read_history(history_file_path.c_str());
+        if (res)
+            std::cerr << "Cannot read history from file " + history_file_path + ": "+ strerror(errno) << std::endl;
+    }
+
+    /// Added '.' to the default list. Because it is used to separate database and table.
+    rl_basic_word_break_characters = word_break_characters;
+
+    /// Not append whitespace after single suggestion. Because whitespace after function name is meaningless.
+    rl_completion_append_character = '\0';
+
+    rl_completion_entry_function = generate;
+
+    /// Install Ctrl+C signal handler that will be used in interactive mode.
+
+    if (rl_initialize())
+        throw std::runtime_error("Cannot initialize readline");
+
+    auto clear_prompt_or_exit = [](int)
+    {
+        /// This is signal safe.
+        ssize_t res = write(STDOUT_FILENO, "\n", 1);
+
+        /// Allow to quit client while query is in progress by pressing Ctrl+C twice.
+        /// (First press to Ctrl+C will try to cancel query by InterruptListener).
+        if (res == 1 && rl_line_buffer[0] && !RL_ISSTATE(RL_STATE_DONE))
+        {
+            rl_replace_line("", 0);
+            if (rl_forced_update_display())
+                _exit(0);
+        }
+        else
+        {
+            /// A little dirty, but we struggle to find better way to correctly
+            /// force readline to exit after returning from the signal handler.
+            _exit(0);
+        }
+    };
+
+    if (signal(SIGINT, clear_prompt_or_exit) == SIG_ERR)
+        throw std::runtime_error(std::string("Cannot set signal handler for readline: ") + strerror(errno));
+}
+
+ReadlineLineReader::~ReadlineLineReader()
+{
+}
+
+LineReader::InputStatus ReadlineLineReader::readOneLine(const String & prompt)
+{
+    input.clear();
+
+    const char* cinput = readline(prompt.c_str());
+    if (cinput == nullptr)
+        return (errno != EAGAIN) ? ABORT : RESET_LINE;
+    input = cinput;
+
+    trim(input);
+    return INPUT_LINE;
+}
+
+void ReadlineLineReader::addToHistory(const String & line)
+{
+    add_history(line.c_str());
+}
+
+#if RL_VERSION_MAJOR >= 7
+
+#define BRACK_PASTE_PREF "\033[200~"
+#define BRACK_PASTE_SUFF "\033[201~"
+
+#define BRACK_PASTE_LAST '~'
+#define BRACK_PASTE_SLEN 6
+
+/// This handler bypasses some unused macro/event checkings and remove trailing newlines before insertion.
+static int clickhouse_rl_bracketed_paste_begin(int /* count */, int /* key */)
+{
+    std::string buf;
+    buf.reserve(128);
+
+    RL_SETSTATE(RL_STATE_MOREINPUT);
+    SCOPE_EXIT(RL_UNSETSTATE(RL_STATE_MOREINPUT));
+    int c;
+    while ((c = rl_read_key()) >= 0)
+    {
+        if (c == '\r')
+            c = '\n';
+        buf.push_back(c);
+        if (buf.size() >= BRACK_PASTE_SLEN && c == BRACK_PASTE_LAST && buf.substr(buf.size() - BRACK_PASTE_SLEN) == BRACK_PASTE_SUFF)
+        {
+            buf.resize(buf.size() - BRACK_PASTE_SLEN);
+            break;
+        }
+    }
+    trim(buf);
+    return static_cast<size_t>(rl_insert_text(buf.c_str())) == buf.size() ? 0 : 1;
+}
+
+#endif
+
+void ReadlineLineReader::enableBracketedPaste()
+{
+#if RL_VERSION_MAJOR >= 7
+    rl_variable_bind("enable-bracketed-paste", "on");
+
+    /// Use our bracketed paste handler to get better user experience. See comments above.
+    rl_bind_keyseq(BRACK_PASTE_PREF, clickhouse_rl_bracketed_paste_begin);
+#endif
+};

base/common/ReadlineLineReader.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "LineReader.h"
+
+#include <readline/readline.h>
+#include <readline/history.h>
+
+class ReadlineLineReader : public LineReader
+{
+public:
+    ReadlineLineReader(const Suggest & suggest, const String & history_file_path, char extender, char delimiter = 0);
+    ~ReadlineLineReader() override;
+
+    void enableBracketedPaste() override;
+
+private:
+    InputStatus readOneLine(const String & prompt) override;
+    void addToHistory(const String & line) override;
+};

base/common/ReplxxLineReader.cpp

@@ -55,3 +55,8 @@ void ReplxxLineReader::addToHistory(const String & line)
 {
     rx.history_add(line);
 }
+
+void ReplxxLineReader::enableBracketedPaste()
+{
+    rx.enable_bracketed_paste();
+};

base/common/ReplxxLineReader.h

@@ -10,6 +10,8 @@ public:
     ReplxxLineReader(const Suggest & suggest, const String & history_file_path, char extender, char delimiter = 0);
     ~ReplxxLineReader() override;
 
+    void enableBracketedPaste() override;
+
 private:
     InputStatus readOneLine(const String & prompt) override;
     void addToHistory(const String & line) override;

contrib/replxx

@@ -1 +1 @@
-Subproject commit 37582f0bb8c52513c6c6b76797c02d852d701dad
+Subproject commit 07cbfbec550133b88c91c4073fa5af2ae2ae6a9a

dbms/programs/client/Client.cpp

@@ -4,6 +4,8 @@
 
 #if USE_REPLXX
 #   include <common/ReplxxLineReader.h>
+#elif USE_READLINE
+#   include <common/ReadlineLineReader.h>
 #else
 #   include <common/LineReader.h>
 #endif
@@ -484,8 +486,12 @@ private:
                 throw Exception("time option could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS);
 
             if (server_revision >= Suggest::MIN_SERVER_REVISION && !config().getBool("disable_suggestion", false))
+            {
+                if (config().has("case_insensitive_suggestion"))
+                    Suggest::instance().setCaseInsensitive();
                 /// Load suggestion data from the server.
                 Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit"));
+            }
 
             /// Load command history if present.
             if (config().has("history_file"))
@@ -504,10 +510,18 @@ private:
 
 #if USE_REPLXX
             ReplxxLineReader lr(Suggest::instance(), history_file, '\\', config().has("multiline") ? ';' : 0);
+#elif USE_READLINE
+            ReadlineLineReader lr(Suggest::instance(), history_file, '\\', config().has("multiline") ? ';' : 0);
 #else
             LineReader lr(history_file, '\\', config().has("multiline") ? ';' : 0);
 #endif
 
+            /// Enable bracketed-paste-mode only when multiquery is enabled and multiline is
+            ///  disabled, so that we are able to paste and execute multiline queries in a whole
+            ///  instead of erroring out, while be less intrusive.
+            if (config().has("multiquery") && !config().has("multiline"))
+                lr.enableBracketedPaste();
+
             do
             {
                 auto input = lr.readLine(prompt(), ":-] ");
@@ -1678,6 +1692,7 @@ public:
             ("always_load_suggestion_data", "Load suggestion data even if clickhouse-client is run in non-interactive mode. Used for testing.")
             ("suggestion_limit", po::value<int>()->default_value(10000),
                 "Suggestion limit for how many databases, tables and columns to fetch.")
+            ("case_insensitive_suggestion", "Case sensitive suggestions.")
             ("multiline,m", "multiline")
             ("multiquery,n", "multiquery")
             ("format,f", po::value<std::string>(), "default output format")

dbms/programs/client/Suggest.cpp

@@ -31,7 +31,17 @@ void Suggest::load(const ConnectionParameters & connection_parameters, size_t su
 
         /// Note that keyword suggestions are available even if we cannot load data from server.
 
-        std::sort(words.begin(), words.end());
+        if (case_insensitive)
+            std::sort(words.begin(), words.end(), [](const std::string & str1, const std::string & str2)
+            {
+                return std::lexicographical_compare(begin(str1), end(str1), begin(str2), end(str2), [](const char char1, const char char2)
+                {
+                    return std::tolower(char1) < std::tolower(char2);
+                });
+            });
+        else
+            std::sort(words.begin(), words.end());
+
         ready = true;
     });
 }

dbms/programs/client/Suggest.h

@@ -24,6 +24,9 @@ public:
         return instance;
     }
 
+    /// Need to set before load
+    void setCaseInsensitive() { case_insensitive = true; }
+
     void load(const ConnectionParameters & connection_parameters, size_t suggestion_limit);
 
     /// Older server versions cannot execute the query above.