Updates for #11422

base/common/CMakeLists.txt

@@ -17,19 +17,12 @@ set (SRCS
     terminalColors.cpp
 )
 
-#if (ENABLE_REPLXX)
-#    list (APPEND SRCS ReplxxLineReader.cpp)
-#elseif (ENABLE_READLINE)
-#    list (APPEND SRCS ReadlineLineReader.cpp)
-#endif ()
-
 if (USE_DEBUG_HELPERS)
     set (INCLUDE_DEBUG_HELPERS "-include ${ClickHouse_SOURCE_DIR}/base/common/iostream_debug_helpers.h")
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${INCLUDE_DEBUG_HELPERS}")
 endif ()
 
 add_library (common ${SRCS})
-#add_library(common clickhouse_parsers)
 
 if (WITH_COVERAGE)
     target_compile_definitions(common PUBLIC WITH_COVERAGE=1)
@@ -47,28 +40,6 @@ if (NOT USE_INTERNAL_BOOST_LIBRARY)
     target_include_directories (common SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
 endif ()
 
-# Allow explicit fallback to readline
-if (NOT ENABLE_REPLXX AND ENABLE_READLINE)
-    message (STATUS "Attempt to fallback to readline explicitly")
-    set (READLINE_PATHS "/usr/local/opt/readline/lib")
-    # First try find custom lib for macos users (default lib without history support)
-    find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS} NO_DEFAULT_PATH)
-    if (NOT READLINE_LIB)
-        find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS})
-    endif ()
-
-    set(READLINE_INCLUDE_PATHS "/usr/local/opt/readline/include")
-    find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS} NO_DEFAULT_PATH)
-    if (NOT READLINE_INCLUDE_DIR)
-        find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS})
-    endif ()
-    if (READLINE_INCLUDE_DIR AND READLINE_LIB)
-        target_link_libraries(common PUBLIC ${READLINE_LIB})
-        target_compile_definitions(common PUBLIC USE_READLINE=1)
-        message (STATUS "Using readline: ${READLINE_INCLUDE_DIR} : ${READLINE_LIB}")
-    endif ()
-endif ()
-
 target_link_libraries (common
     PUBLIC
         ${CITYHASH_LIBRARIES}
@@ -78,7 +49,6 @@ target_link_libraries (common
         Poco::Net::SSL
         Poco::Util
         Poco::Foundation
-        replxx
         fmt
 
     PRIVATE

base/common/ya.make

@@ -39,7 +39,6 @@ SRCS(
     getMemoryAmount.cpp
     getThreadId.cpp
     JSON.cpp
-    LineReader.cpp
     mremap.cpp
     phdr_cache.cpp
     preciseExp10.cpp

src/CMakeLists.txt

@@ -235,6 +235,29 @@ if(RE2_ST_LIBRARY)
     target_link_libraries(clickhouse_common_io PUBLIC ${RE2_ST_LIBRARY})
 endif()
 
+# Allow explicit fallback to readline
+if (NOT ENABLE_REPLXX AND ENABLE_READLINE)
+    message (STATUS "Attempt to fallback to readline explicitly")
+    set (READLINE_PATHS "/usr/local/opt/readline/lib")
+    # First try find custom lib for macos users (default lib without history support)
+    find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS} NO_DEFAULT_PATH)
+    if (NOT READLINE_LIB)
+        find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS})
+    endif ()
+
+    set(READLINE_INCLUDE_PATHS "/usr/local/opt/readline/include")
+    find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS} NO_DEFAULT_PATH)
+    if (NOT READLINE_INCLUDE_DIR)
+        find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS})
+    endif ()
+
+    if (READLINE_INCLUDE_DIR AND READLINE_LIB)
+        target_link_libraries(clickhouse_common_io PUBLIC ${READLINE_LIB})
+        target_compile_definitions(clickhouse_common_io PUBLIC USE_READLINE=1)
+        message (STATUS "Using readline: ${READLINE_INCLUDE_DIR} : ${READLINE_LIB}")
+    endif ()
+endif ()
+
 target_link_libraries(clickhouse_common_io
         PRIVATE
             ${EXECINFO_LIBRARIES}
@@ -247,6 +270,7 @@ target_link_libraries(clickhouse_common_io
             pcg_random
             Poco::Foundation
             roaring
+            replxx
 )
 
 if (USE_RDKAFKA)
@@ -261,6 +285,7 @@ if(RE2_INCLUDE_DIR)
     target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${RE2_INCLUDE_DIR})
 endif()
 
+
 dbms_target_link_libraries (
     PRIVATE
         ${BTRIE_LIBRARIES}

src/Common/CMakeLists.txt

@@ -1,7 +1,4 @@
 add_subdirectory(StringUtils)
-# after common_io
-#add_subdirectory(ZooKeeper)
-#add_subdirectory(ConfigProcessor)
 
 if (ENABLE_TESTS)
     add_subdirectory (tests)

src/Common/ReadlineLineReader.cpp

@@ -1,3 +1,5 @@
+#if defined(USE_READLINE)
+
 #include <Common/ReadlineLineReader.h>
 #include <ext/scope_guard.h>
 
@@ -176,3 +178,5 @@ void ReadlineLineReader::enableBracketedPaste()
     rl_bind_keyseq(BRACK_PASTE_PREF, clickhouse_rl_bracketed_paste_begin);
 #endif
 };
+
+#endif

src/Common/ReplxxLineReader.cpp

@@ -15,8 +15,81 @@ void trim(String & s)
     s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end());
 }
 
+
+void highlightCallback(const String & query, std::vector<replxx::Replxx::Color> & colors)
+{
+    using namespace DB;
+    using namespace replxx;
+
+    static const std::unordered_map<TokenType, Replxx::Color> token_to_color =
+    {
+        { TokenType::Whitespace, Replxx::Color::DEFAULT },
+        { TokenType::Comment, Replxx::Color::GRAY },
+        { TokenType::BareWord, Replxx::Color::WHITE },
+        { TokenType::Number, Replxx::Color::BRIGHTGREEN },
+        { TokenType::StringLiteral, Replxx::Color::BRIGHTCYAN },
+        { TokenType::QuotedIdentifier, Replxx::Color::BRIGHTMAGENTA },
+        { TokenType::OpeningRoundBracket, Replxx::Color::LIGHTGRAY },
+        { TokenType::ClosingRoundBracket, Replxx::Color::LIGHTGRAY },
+        { TokenType::OpeningSquareBracket, Replxx::Color::BROWN },
+        { TokenType::ClosingSquareBracket, Replxx::Color::BROWN },
+        { TokenType::OpeningCurlyBrace, Replxx::Color::WHITE },
+        { TokenType::ClosingCurlyBrace, Replxx::Color::WHITE },
+
+        { TokenType::Comma, Replxx::Color::YELLOW },
+        { TokenType::Semicolon, Replxx::Color::YELLOW },
+        { TokenType::Dot, Replxx::Color::YELLOW },
+        { TokenType::Asterisk, Replxx::Color::YELLOW },
+        { TokenType::Plus, Replxx::Color::YELLOW },
+        { TokenType::Minus, Replxx::Color::YELLOW },
+        { TokenType::Slash, Replxx::Color::YELLOW },
+        { TokenType::Percent, Replxx::Color::YELLOW },
+        { TokenType::Arrow, Replxx::Color::YELLOW },
+        { TokenType::QuestionMark, Replxx::Color::YELLOW },
+        { TokenType::Colon, Replxx::Color::YELLOW },
+        { TokenType::Equals, Replxx::Color::YELLOW },
+        { TokenType::NotEquals, Replxx::Color::YELLOW },
+        { TokenType::Less, Replxx::Color::YELLOW },
+        { TokenType::Greater, Replxx::Color::YELLOW },
+        { TokenType::LessOrEquals, Replxx::Color::YELLOW },
+        { TokenType::GreaterOrEquals, Replxx::Color::YELLOW },
+        { TokenType::Concatenation, Replxx::Color::YELLOW },
+        { TokenType::At, Replxx::Color::YELLOW },
+
+        { TokenType::EndOfStream, Replxx::Color::DEFAULT },
+
+        { TokenType::Error, Replxx::Color::RED },
+        { TokenType::ErrorMultilineCommentIsNotClosed, Replxx::Color::RED },
+        { TokenType::ErrorSingleQuoteIsNotClosed, Replxx::Color::RED },
+        { TokenType::ErrorDoubleQuoteIsNotClosed, Replxx::Color::RED },
+        { TokenType::ErrorSinglePipeMark, Replxx::Color::RED },
+        { TokenType::ErrorWrongNumber, Replxx::Color::RED },
+        { TokenType::ErrorMaxQuerySizeExceeded, Replxx::Color::RED }
+    };
+
+    const Replxx::Color unknown_token_color = Replxx::Color::RED;
+
+    Lexer lexer(query.data(), query.data() + query.size());
+    size_t pos = 0;
+
+    for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken())
+    {
+        size_t utf8_len = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(token.begin), token.size());
+        for (size_t code_point_index = 0; code_point_index < utf8_len; ++code_point_index)
+        {
+            if (token_to_color.find(token.type) != token_to_color.end())
+                colors[pos + code_point_index] = token_to_color.at(token.type);
+            else
+                colors[pos + code_point_index] = unknown_token_color;
+        }
+
+        pos += utf8_len;
+    }
 }
 
+}
+
+
 ReplxxLineReader::ReplxxLineReader(
     const Suggest & suggest, const String & history_file_path_, bool multiline_, Patterns extenders_, Patterns delimiters_)
     : LineReader(history_file_path_, multiline_, std::move(extenders_), std::move(delimiters_))
@@ -34,27 +107,7 @@ ReplxxLineReader::ReplxxLineReader(
         return Replxx::completions_t(range.first, range.second);
     };
 
-
-    auto highlighter_callback = [this] (const String & query, std::vector<Replxx::Color> & colors)
-    {
-        Lexer lexer(query.data(), query.data() + query.size());
-        size_t pos = 0;
-
-        for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken())
-        {
-            size_t utf8_len = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(token.begin), token.size());
-            for (size_t code_point_index = 0; code_point_index < utf8_len; ++code_point_index)
-            {
-                if (this->token_to_color.find(token.type) != this->token_to_color.end())
-                    colors[pos + code_point_index] = this->token_to_color.at(token.type);
-                else
-                    colors[pos + code_point_index] = this->unknown_token_color;
-            }
-            pos += utf8_len;
-        }
-    };
-
-    rx.set_highlighter_callback(highlighter_callback);
+    rx.set_highlighter_callback(highlightCallback);
     rx.set_completion_callback(suggesting_callback);
     rx.set_complete_on_empty(false);
     rx.set_word_break_characters(word_break_characters);
@@ -63,6 +116,7 @@ ReplxxLineReader::ReplxxLineReader(
     /// bind C-p/C-n to history-previous/history-next like readline.
     rx.bind_key(Replxx::KEY::control('N'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_NEXT, code); });
     rx.bind_key(Replxx::KEY::control('P'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_PREVIOUS, code); });
+
     /// By default COMPLETE_NEXT/COMPLETE_PREV was binded to C-p/C-n, re-bind
     /// to M-P/M-N (that was used for HISTORY_COMMON_PREFIX_SEARCH before, but
     /// it also binded to M-p/M-n).

src/Common/ReplxxLineReader.h

@@ -7,6 +7,7 @@
 
 #include <replxx.hxx>
 
+
 class ReplxxLineReader : public LineReader
 {
 public:
@@ -19,50 +20,5 @@ private:
     InputStatus readOneLine(const String & prompt) override;
     void addToHistory(const String & line) override;
 
-    const std::unordered_map<DB::TokenType, replxx::Replxx::Color> token_to_color = {
-        { DB::TokenType::Whitespace, replxx::Replxx::Color::NORMAL },
-        { DB::TokenType::Comment, replxx::Replxx::Color::GRAY },
-        { DB::TokenType::BareWord, replxx::Replxx::Color::YELLOW },
-        { DB::TokenType::Number, replxx::Replxx::Color::BLUE },
-        { DB::TokenType::StringLiteral, replxx::Replxx::Color::BRIGHTCYAN },
-        { DB::TokenType::QuotedIdentifier, replxx::Replxx::Color::BRIGHTMAGENTA },
-        { DB::TokenType::OpeningRoundBracket, replxx::Replxx::Color::LIGHTGRAY },
-        { DB::TokenType::ClosingRoundBracket, replxx::Replxx::Color::LIGHTGRAY },
-        { DB::TokenType::OpeningSquareBracket, replxx::Replxx::Color::BROWN },
-        { DB::TokenType::ClosingSquareBracket, replxx::Replxx::Color::BROWN },
-        { DB::TokenType::OpeningCurlyBrace, replxx::Replxx::Color::WHITE },
-        { DB::TokenType::ClosingCurlyBrace, replxx::Replxx::Color::WHITE },
-        { DB::TokenType::Comma, replxx::Replxx::Color::GREEN },
-        { DB::TokenType::Semicolon, replxx::Replxx::Color::GREEN },
-        { DB::TokenType::Dot, replxx::Replxx::Color::GREEN },
-        { DB::TokenType::Asterisk, replxx::Replxx::Color::GREEN },
-        { DB::TokenType::Plus, replxx::Replxx::Color::GREEN },
-        { DB::TokenType::Minus, replxx::Replxx::Color::GREEN },
-        { DB::TokenType::Slash, replxx::Replxx::Color::GREEN },
-        { DB::TokenType::Percent, replxx::Replxx::Color::GREEN },
-        { DB::TokenType::Arrow, replxx::Replxx::Color::GREEN },
-        { DB::TokenType::QuestionMark, replxx::Replxx::Color::GREEN },
-        { DB::TokenType::Colon, replxx::Replxx::Color::CYAN },
-        { DB::TokenType::Equals, replxx::Replxx::Color::CYAN },
-        { DB::TokenType::NotEquals, replxx::Replxx::Color::CYAN },
-        { DB::TokenType::Less, replxx::Replxx::Color::CYAN },
-        { DB::TokenType::Greater, replxx::Replxx::Color::CYAN },
-        { DB::TokenType::LessOrEquals, replxx::Replxx::Color::CYAN },
-        { DB::TokenType::GreaterOrEquals, replxx::Replxx::Color::CYAN },
-        { DB::TokenType::Concatenation, replxx::Replxx::Color::YELLOW },
-        { DB::TokenType::At, replxx::Replxx::Color::YELLOW },
-        { DB::TokenType::EndOfStream, replxx::Replxx::Color::NORMAL },
-        { DB::TokenType::Error, replxx::Replxx::Color::RED },
-        { DB::TokenType::ErrorMultilineCommentIsNotClosed, replxx::Replxx::Color::RED },
-        { DB::TokenType::ErrorSingleQuoteIsNotClosed, replxx::Replxx::Color::RED },
-        { DB::TokenType::ErrorDoubleQuoteIsNotClosed, replxx::Replxx::Color::RED },
-       // { DB::TokenType::ErrorBackQuoteIsNotClosedErrorSingleExclamationMark, replxx::Replxx::Color::RED },
-        { DB::TokenType::ErrorSinglePipeMark, replxx::Replxx::Color::RED },
-        { DB::TokenType::ErrorWrongNumber, replxx::Replxx::Color::RED },
-        { DB::TokenType::ErrorMaxQuerySizeExceeded, replxx::Replxx::Color::RED }
-    };
-
-    const replxx::Replxx::Color unknown_token_color = replxx::Replxx::Color::RED;
-
     replxx::Replxx rx;
 };

src/Common/ya.make

@@ -52,6 +52,7 @@ SRCS(
     IntervalKind.cpp
     IPv6ToBinary.cpp
     isLocalAddress.cpp
+    LineReader.cpp
     Macros.cpp
     malloc.cpp
     MemoryStatisticsOS.cpp