Merge branch 'add_syntax_highlighting' of https://github.com/kuskarov/ClickHouse into kuskarov-add_syntax_highlighting

base/common/CMakeLists.txt

@@ -8,7 +8,6 @@ set (SRCS
     getMemoryAmount.cpp
     getThreadId.cpp
     JSON.cpp
-    LineReader.cpp
     mremap.cpp
     phdr_cache.cpp
     preciseExp10.cpp
@@ -18,11 +17,11 @@ set (SRCS
     terminalColors.cpp
 )
 
-if (ENABLE_REPLXX)
-    list (APPEND SRCS ReplxxLineReader.cpp)
-elseif (ENABLE_READLINE)
-    list (APPEND SRCS ReadlineLineReader.cpp)
-endif ()
+#if (ENABLE_REPLXX)
+#    list (APPEND SRCS ReplxxLineReader.cpp)
+#elseif (ENABLE_READLINE)
+#    list (APPEND SRCS ReadlineLineReader.cpp)
+#endif ()
 
 if (USE_DEBUG_HELPERS)
     set (INCLUDE_DEBUG_HELPERS "-include ${ClickHouse_SOURCE_DIR}/base/common/iostream_debug_helpers.h")
@@ -30,6 +29,7 @@ if (USE_DEBUG_HELPERS)
 endif ()
 
 add_library (common ${SRCS})
+#add_library(common clickhouse_parsers)
 
 if (WITH_COVERAGE)
     target_compile_definitions(common PUBLIC WITH_COVERAGE=1)

base/common/ReplxxLineReader.h

@@ -1,20 +0,0 @@
-#pragma once
-
-#include "LineReader.h"
-
-#include <replxx.hxx>
-
-class ReplxxLineReader : public LineReader
-{
-public:
-    ReplxxLineReader(const Suggest & suggest, const String & history_file_path, bool multiline, Patterns extenders_, Patterns delimiters_);
-    ~ReplxxLineReader() override;
-
-    void enableBracketedPaste() override;
-
-private:
-    InputStatus readOneLine(const String & prompt) override;
-    void addToHistory(const String & line) override;
-
-    replxx::Replxx rx;
-};

programs/client/Client.cpp

@@ -3,11 +3,11 @@
 #include "Suggest.h"
 
 #if USE_REPLXX
-#   include <common/ReplxxLineReader.h>
+#   include <Common/ReplxxLineReader.h>
 #elif defined(USE_READLINE) && USE_READLINE
-#   include <common/ReadlineLineReader.h>
+#   include <Common/ReadlineLineReader.h>
 #else
-#   include <common/LineReader.h>
+#   include <Common/LineReader.h>
 #endif
 
 #include <stdlib.h>
@@ -27,7 +27,7 @@
 #include <Poco/File.h>
 #include <Poco/Util/Application.h>
 #include <common/find_symbols.h>
-#include <common/LineReader.h>
+#include <Common/LineReader.h>
 #include <Common/ClickHouseRevision.h>
 #include <Common/Stopwatch.h>
 #include <Common/Exception.h>

programs/client/Suggest.h

@@ -4,7 +4,7 @@
 
 #include <Client/Connection.h>
 #include <IO/ConnectionTimeouts.h>
-#include <common/LineReader.h>
+#include <Common/LineReader.h>
 #include <thread>
 
 

src/Common/LineReader.cpp

@@ -1,4 +1,4 @@
-#include <common/LineReader.h>
+#include <Common/LineReader.h>
 
 #include <iostream>
 #include <string_view>

src/Common/ReadlineLineReader.cpp

@@ -1,10 +1,12 @@
-#include <common/ReadlineLineReader.h>
+#include <Common/ReadlineLineReader.h>
 #include <ext/scope_guard.h>
 
 #include <errno.h>
 #include <signal.h>
 #include <string.h>
 #include <unistd.h>
+#include <iostream>
+#include <ostream>
 
 namespace
 {

src/Common/ReplxxLineReader.cpp

@@ -1,4 +1,5 @@
-#include <common/ReplxxLineReader.h>
+#include <Common/ReplxxLineReader.h>
+#include <Common/UTF8Helpers.h>
 
 #include <errno.h>
 #include <string.h>
@@ -22,17 +23,39 @@ ReplxxLineReader::ReplxxLineReader(
 {
     using namespace std::placeholders;
     using Replxx = replxx::Replxx;
+    using namespace DB;
 
     if (!history_file_path.empty())
         rx.history_load(history_file_path);
 
-    auto callback = [&suggest] (const String & context, size_t context_size)
+    auto suggesting_callback = [&suggest] (const String & context, size_t context_size)
     {
         auto range = suggest.getCompletions(context, context_size);
         return Replxx::completions_t(range.first, range.second);
     };
 
-    rx.set_completion_callback(callback);
+
+    auto highlighter_callback = [this] (const String & query, std::vector<Replxx::Color> & colors)
+    {
+        Lexer lexer(query.data(), query.data() + query.size());
+        size_t pos = 0;
+
+        for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken())
+        {
+            size_t utf8_len = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(token.begin), token.size());
+            for (size_t code_point_index = 0; code_point_index < utf8_len; ++code_point_index)
+            {
+                if (this->token_to_color.find(token.type) != this->token_to_color.end())
+                    colors[pos + code_point_index] = this->token_to_color.at(token.type);
+                else
+                    colors[pos + code_point_index] = this->unknown_token_color;
+            }
+            pos += utf8_len;
+        }
+    };
+
+    rx.set_highlighter_callback(highlighter_callback);
+    rx.set_completion_callback(suggesting_callback);
     rx.set_complete_on_empty(false);
     rx.set_word_break_characters(word_break_characters);
 

src/Common/ReplxxLineReader.h

@@ -0,0 +1,68 @@
+#pragma once
+
+#include <unordered_map>
+
+#include "LineReader.h"
+#include <Parsers/Lexer.h>
+
+#include <replxx.hxx>
+
+class ReplxxLineReader : public LineReader
+{
+public:
+    ReplxxLineReader(const Suggest & suggest, const String & history_file_path, bool multiline, Patterns extenders_, Patterns delimiters_);
+    ~ReplxxLineReader() override;
+
+    void enableBracketedPaste() override;
+
+private:
+    InputStatus readOneLine(const String & prompt) override;
+    void addToHistory(const String & line) override;
+
+    const std::unordered_map<DB::TokenType, replxx::Replxx::Color> token_to_color = {
+        { DB::TokenType::Whitespace, replxx::Replxx::Color::NORMAL },
+        { DB::TokenType::Comment, replxx::Replxx::Color::GRAY },
+        { DB::TokenType::BareWord, replxx::Replxx::Color::YELLOW },
+        { DB::TokenType::Number, replxx::Replxx::Color::BLUE },
+        { DB::TokenType::StringLiteral, replxx::Replxx::Color::BRIGHTCYAN },
+        { DB::TokenType::QuotedIdentifier, replxx::Replxx::Color::BRIGHTMAGENTA },
+        { DB::TokenType::OpeningRoundBracket, replxx::Replxx::Color::LIGHTGRAY },
+        { DB::TokenType::ClosingRoundBracket, replxx::Replxx::Color::LIGHTGRAY },
+        { DB::TokenType::OpeningSquareBracket, replxx::Replxx::Color::BROWN },
+        { DB::TokenType::ClosingSquareBracket, replxx::Replxx::Color::BROWN },
+        { DB::TokenType::OpeningCurlyBrace, replxx::Replxx::Color::WHITE },
+        { DB::TokenType::ClosingCurlyBrace, replxx::Replxx::Color::WHITE },
+        { DB::TokenType::Comma, replxx::Replxx::Color::GREEN },
+        { DB::TokenType::Semicolon, replxx::Replxx::Color::GREEN },
+        { DB::TokenType::Dot, replxx::Replxx::Color::GREEN },
+        { DB::TokenType::Asterisk, replxx::Replxx::Color::GREEN },
+        { DB::TokenType::Plus, replxx::Replxx::Color::GREEN },
+        { DB::TokenType::Minus, replxx::Replxx::Color::GREEN },
+        { DB::TokenType::Slash, replxx::Replxx::Color::GREEN },
+        { DB::TokenType::Percent, replxx::Replxx::Color::GREEN },
+        { DB::TokenType::Arrow, replxx::Replxx::Color::GREEN },
+        { DB::TokenType::QuestionMark, replxx::Replxx::Color::GREEN },
+        { DB::TokenType::Colon, replxx::Replxx::Color::CYAN },
+        { DB::TokenType::Equals, replxx::Replxx::Color::CYAN },
+        { DB::TokenType::NotEquals, replxx::Replxx::Color::CYAN },
+        { DB::TokenType::Less, replxx::Replxx::Color::CYAN },
+        { DB::TokenType::Greater, replxx::Replxx::Color::CYAN },
+        { DB::TokenType::LessOrEquals, replxx::Replxx::Color::CYAN },
+        { DB::TokenType::GreaterOrEquals, replxx::Replxx::Color::CYAN },
+        { DB::TokenType::Concatenation, replxx::Replxx::Color::YELLOW },
+        { DB::TokenType::At, replxx::Replxx::Color::YELLOW },
+        { DB::TokenType::EndOfStream, replxx::Replxx::Color::NORMAL },
+        { DB::TokenType::Error, replxx::Replxx::Color::RED },
+        { DB::TokenType::ErrorMultilineCommentIsNotClosed, replxx::Replxx::Color::RED },
+        { DB::TokenType::ErrorSingleQuoteIsNotClosed, replxx::Replxx::Color::RED },
+        { DB::TokenType::ErrorDoubleQuoteIsNotClosed, replxx::Replxx::Color::RED },
+       // { DB::TokenType::ErrorBackQuoteIsNotClosedErrorSingleExclamationMark, replxx::Replxx::Color::RED },
+        { DB::TokenType::ErrorSinglePipeMark, replxx::Replxx::Color::RED },
+        { DB::TokenType::ErrorWrongNumber, replxx::Replxx::Color::RED },
+        { DB::TokenType::ErrorMaxQuerySizeExceeded, replxx::Replxx::Color::RED }
+    };
+
+    const replxx::Replxx::Color unknown_token_color = replxx::Replxx::Color::RED;
+
+    replxx::Replxx rx;
+};

utils/zookeeper-cli/zookeeper-cli.cpp

@@ -3,7 +3,7 @@
 #include <Poco/ConsoleChannel.h>
 #include <Common/ZooKeeper/KeeperException.h>
 #include <Common/ZooKeeper/ZooKeeper.h>
-#include <common/LineReader.h>
+#include <Common/LineReader.h>
 #include <common/logger_useful.h>
 
 #include <iostream>