diff --git a/.gitmodules b/.gitmodules index 8333e5544bc..f6f2c652004 100644 --- a/.gitmodules +++ b/.gitmodules @@ -134,6 +134,9 @@ [submodule "contrib/libc-headers"] path = contrib/libc-headers url = https://github.com/ClickHouse-Extras/libc-headers.git +[submodule "contrib/replxx"] + path = contrib/replxx + url = https://github.com/AmokHuginnsson/replxx.git [submodule "contrib/ryu"] path = contrib/ryu url = https://github.com/ClickHouse-Extras/ryu.git diff --git a/CMakeLists.txt b/CMakeLists.txt index cd32288ec9c..7c8ccb6e17c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -328,7 +328,6 @@ include (cmake/find/xxhash.cmake) include (cmake/find/sparsehash.cmake) include (cmake/find/rt.cmake) include (cmake/find/execinfo.cmake) -include (cmake/find/readline_edit.cmake) include (cmake/find/re2.cmake) include (cmake/find/libgsasl.cmake) include (cmake/find/rdkafka.cmake) diff --git a/cmake/find/readline_edit.cmake b/cmake/find/readline_edit.cmake deleted file mode 100644 index 96518a66887..00000000000 --- a/cmake/find/readline_edit.cmake +++ /dev/null @@ -1,60 +0,0 @@ -include (CMakePushCheckState) -cmake_push_check_state () - -option (ENABLE_READLINE "Enable readline" ${ENABLE_LIBRARIES}) -if (ENABLE_READLINE) - -set (READLINE_PATHS "/usr/local/opt/readline/lib") -# First try find custom lib for macos users (default lib without history support) -find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS} NO_DEFAULT_PATH) -if (NOT READLINE_LIB) - find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS}) -endif () - -list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .so.2) - -find_library (EDIT_LIB NAMES edit) - -set(READLINE_INCLUDE_PATHS "/usr/local/opt/readline/include") -if (READLINE_LIB AND TERMCAP_LIBRARY) - find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS} NO_DEFAULT_PATH) - if (NOT READLINE_INCLUDE_DIR) - find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS}) - endif () - if (READLINE_INCLUDE_DIR AND READLINE_LIB) - set (USE_READLINE 1) - set (LINE_EDITING_LIBS ${READLINE_LIB} ${TERMCAP_LIBRARY}) - message (STATUS "Using line editing libraries (readline): ${READLINE_INCLUDE_DIR} : ${LINE_EDITING_LIBS}") - endif () -elseif (EDIT_LIB AND TERMCAP_LIBRARY) - find_library (CURSES_LIB NAMES curses) - find_path (READLINE_INCLUDE_DIR NAMES editline/readline.h PATHS ${READLINE_INCLUDE_PATHS}) - if (CURSES_LIB AND READLINE_INCLUDE_DIR) - set (USE_LIBEDIT 1) - set (LINE_EDITING_LIBS ${EDIT_LIB} ${CURSES_LIB} ${TERMCAP_LIBRARY}) - message (STATUS "Using line editing libraries (edit): ${READLINE_INCLUDE_DIR} : ${LINE_EDITING_LIBS}") - endif () -endif () - -endif () - -if (LINE_EDITING_LIBS AND READLINE_INCLUDE_DIR) - include (CheckCXXSourceRuns) - - set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${LINE_EDITING_LIBS}) - set (CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES} ${READLINE_INCLUDE_DIR}) - check_cxx_source_runs (" - #include - #include - #include - int main() { - add_history(NULL); - append_history(1,NULL); - return 0; - } - " HAVE_READLINE_HISTORY) -else () - message (STATUS "Not using any library for line editing.") -endif () - -cmake_pop_check_state () diff --git a/cmake/target.cmake b/cmake/target.cmake index 9233af62dcf..1f40e28e76b 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -15,7 +15,6 @@ if (CMAKE_CROSSCOMPILING) set (USE_SNAPPY OFF CACHE INTERNAL "") set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") set (ENABLE_PARQUET OFF CACHE INTERNAL "") - set (ENABLE_READLINE OFF CACHE INTERNAL "") set (ENABLE_ICU OFF CACHE INTERNAL "") set (ENABLE_FASTOPS OFF CACHE INTERNAL "") elseif (OS_LINUX) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 53ad9a0c138..fe3e4e83f03 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -331,3 +331,5 @@ endif() if (USE_FASTOPS) add_subdirectory (fastops-cmake) endif() + +add_subdirectory (replxx-cmake) diff --git a/contrib/replxx b/contrib/replxx new file mode 160000 index 00000000000..37582f0bb8c --- /dev/null +++ b/contrib/replxx @@ -0,0 +1 @@ +Subproject commit 37582f0bb8c52513c6c6b76797c02d852d701dad diff --git a/contrib/replxx-cmake/CMakeLists.txt b/contrib/replxx-cmake/CMakeLists.txt new file mode 100644 index 00000000000..fed70558b07 --- /dev/null +++ b/contrib/replxx-cmake/CMakeLists.txt @@ -0,0 +1,53 @@ +option (ENABLE_READLINE "Enable readline support" ${ENABLE_LIBRARIES}) + +if (ENABLE_READLINE) + option (USE_INTERNAL_REPLXX "Use internal replxx library" ${NOT_UNBUNDLED}) + + if (USE_INTERNAL_REPLXX) + set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/replxx") + + set(SRCS + ${LIBRARY_DIR}/src/conversion.cxx + ${LIBRARY_DIR}/src/ConvertUTF.cpp + ${LIBRARY_DIR}/src/escape.cxx + ${LIBRARY_DIR}/src/history.cxx + ${LIBRARY_DIR}/src/io.cxx + ${LIBRARY_DIR}/src/prompt.cxx + ${LIBRARY_DIR}/src/replxx.cxx + ${LIBRARY_DIR}/src/replxx_impl.cxx + ${LIBRARY_DIR}/src/util.cxx + ${LIBRARY_DIR}/src/wcwidth.cpp + ${LIBRARY_DIR}/src/windows.cxx + ) + + add_library(replxx ${SRCS}) + target_include_directories(replxx PUBLIC ${LIBRARY_DIR}/include) + target_compile_options(replxx PUBLIC -Wno-documentation) + else () + find_library(LIBRARY_REPLXX NAMES replxx replxx-static) + find_path(INCLUDE_REPLXX replxx.hxx) + + add_library(replxx UNKNOWN IMPORTED) + set_property(TARGET replxx PROPERTY IMPORTED_LOCATION ${LIBRARY_REPLXX}) + target_include_directories(replxx PUBLIC ${INCLUDE_REPLXX}) + + set(CMAKE_REQUIRED_LIBRARIES replxx) + check_cxx_source_compiles( + " + #include + int main() { + replxx::Replxx rx; + } + " + EXTERNAL_REPLXX_WORKS + ) + + if (NOT EXTERNAL_REPLXX_WORKS) + message (FATAL_ERROR "replxx is unusable: ${LIBRARY_REPLXX} ${INCLUDE_REPLXX}") + endif () + endif () + + target_compile_definitions(replxx PUBLIC USE_REPLXX) + + message (STATUS "Using replxx") +endif () diff --git a/dbms/programs/client/CMakeLists.txt b/dbms/programs/client/CMakeLists.txt index dc5cf787adf..2cd31ab0ab6 100644 --- a/dbms/programs/client/CMakeLists.txt +++ b/dbms/programs/client/CMakeLists.txt @@ -1,15 +1,12 @@ set(CLICKHOUSE_CLIENT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Client.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ConnectionParameters.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Suggest.cpp ) set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io clickhouse_parsers string_utils ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY}) set(CLICKHOUSE_CLIENT_INCLUDE PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/include) -if (READLINE_INCLUDE_DIR) - set(CLICKHOUSE_CLIENT_INCLUDE ${CLICKHOUSE_CLIENT_INCLUDE} SYSTEM PRIVATE ${READLINE_INCLUDE_DIR}) -endif () - include(CheckSymbolExists) check_symbol_exists(readpassphrase readpassphrase.h HAVE_READPASSPHRASE) configure_file(config_client.h.in ${ConfigIncludePath}/config_client.h) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index b37acab77ea..1c4902c48f6 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -1,7 +1,7 @@ #include "TestHint.h" #include "ConnectionParameters.h" +#include "Suggest.h" -#include #include #include #include @@ -18,8 +18,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -69,10 +69,6 @@ #include #include -#if USE_READLINE -#include "Suggest.h" -#endif - #ifndef __clang__ #pragma GCC optimize("-fno-var-tracking-assignments") #endif @@ -89,39 +85,6 @@ #define DISABLE_LINE_WRAPPING "\033[?7l" #define ENABLE_LINE_WRAPPING "\033[?7h" -#if USE_READLINE && RL_VERSION_MAJOR >= 7 - -#define BRACK_PASTE_PREF "\033[200~" -#define BRACK_PASTE_SUFF "\033[201~" - -#define BRACK_PASTE_LAST '~' -#define BRACK_PASTE_SLEN 6 - -/// This handler bypasses some unused macro/event checkings. -static int clickhouse_rl_bracketed_paste_begin(int /* count */, int /* key */) -{ - std::string buf; - buf.reserve(128); - - RL_SETSTATE(RL_STATE_MOREINPUT); - SCOPE_EXIT(RL_UNSETSTATE(RL_STATE_MOREINPUT)); - int c; - while ((c = rl_read_key()) >= 0) - { - if (c == '\r') - c = '\n'; - buf.push_back(c); - if (buf.size() >= BRACK_PASTE_SLEN && c == BRACK_PASTE_LAST && buf.substr(buf.size() - BRACK_PASTE_SLEN) == BRACK_PASTE_SUFF) - { - buf.resize(buf.size() - BRACK_PASTE_SLEN); - break; - } - } - return static_cast(rl_insert_text(buf.c_str())) == buf.size() ? 0 : 1; -} - -#endif - namespace DB { @@ -514,26 +477,10 @@ private: if (print_time_to_stderr) throw Exception("time option could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS); -#if USE_READLINE - SCOPE_EXIT({ Suggest::instance().finalize(); }); - if (server_revision >= Suggest::MIN_SERVER_REVISION - && !config().getBool("disable_suggestion", false)) - { + if (server_revision >= Suggest::MIN_SERVER_REVISION && !config().getBool("disable_suggestion", false)) /// Load suggestion data from the server. - Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit")); + Suggest::instance()->load(connection_parameters, config().getInt("suggestion_limit")); - /// Added '.' to the default list. Because it is used to separate database and table. - rl_basic_word_break_characters = " \t\n\r\"\\'`@$><=;|&{(."; - - /// Not append whitespace after single suggestion. Because whitespace after function name is meaningless. - rl_completion_append_character = '\0'; - - rl_completion_entry_function = Suggest::generator; - } - else - /// Turn tab completion off. - rl_bind_key('\t', rl_insert); -#endif /// Load command history if present. if (config().has("history_file")) history_file = config().getString("history_file"); @@ -546,70 +493,45 @@ private: history_file = home_path + "/.clickhouse-client-history"; } - if (!history_file.empty()) + if (!history_file.empty() && !Poco::File(history_file).exists()) + Poco::File(history_file).createFile(); + + LineReader lr(Suggest::instance(), history_file, '\\', config().has("multiline") ? ';' : 0); + + do { - if (Poco::File(history_file).exists()) + auto input = lr.readLine(prompt(), ":-] "); + if (input.empty()) + break; + + try { -#if USE_READLINE - int res = read_history(history_file.c_str()); - if (res) - std::cerr << "Cannot read history from file " + history_file + ": "+ errnoToString(ErrorCodes::CANNOT_READ_HISTORY); -#endif + if (!process(input)) + break; + } + catch (const Exception & e) + { + actual_client_error = e.code(); + if (!actual_client_error || actual_client_error != expected_client_error) + { + std::cerr << std::endl + << "Exception on client:" << std::endl + << "Code: " << e.code() << ". " << e.displayText() << std::endl; + + if (config().getBool("stacktrace", false)) + std::cerr << "Stack trace:" << std::endl << e.getStackTraceString() << std::endl; + + std::cerr << std::endl; + + } + + /// Client-side exception during query execution can result in the loss of + /// sync in the connection protocol. + /// So we reconnect and allow to enter the next query. + connect(); } - else /// Create history file. - Poco::File(history_file).createFile(); } - -#if USE_READLINE - /// Install Ctrl+C signal handler that will be used in interactive mode. - - if (rl_initialize()) - throw Exception("Cannot initialize readline", ErrorCodes::CANNOT_READLINE); - -#if RL_VERSION_MAJOR >= 7 - /// Enable bracketed-paste-mode only when multiquery is enabled and multiline is - /// disabled, so that we are able to paste and execute multiline queries in a whole - /// instead of erroring out, while be less intrusive. - if (config().has("multiquery") && !config().has("multiline")) - { - /// When bracketed paste mode is set, pasted text is bracketed with control sequences so - /// that the program can differentiate pasted text from typed-in text. This helps - /// clickhouse-client so that without -m flag, one can still paste multiline queries, and - /// possibly get better pasting performance. See https://cirw.in/blog/bracketed-paste for - /// more details. - rl_variable_bind("enable-bracketed-paste", "on"); - - /// Use our bracketed paste handler to get better user experience. See comments above. - rl_bind_keyseq(BRACK_PASTE_PREF, clickhouse_rl_bracketed_paste_begin); - } -#endif - - auto clear_prompt_or_exit = [](int) - { - /// This is signal safe. - ssize_t res = write(STDOUT_FILENO, "\n", 1); - - /// Allow to quit client while query is in progress by pressing Ctrl+C twice. - /// (First press to Ctrl+C will try to cancel query by InterruptListener). - if (res == 1 && rl_line_buffer[0] && !RL_ISSTATE(RL_STATE_DONE)) - { - rl_replace_line("", 0); - if (rl_forced_update_display()) - _exit(0); - } - else - { - /// A little dirty, but we struggle to find better way to correctly - /// force readline to exit after returning from the signal handler. - _exit(0); - } - }; - - if (signal(SIGINT, clear_prompt_or_exit) == SIG_ERR) - throwFromErrno("Cannot set signal handler.", ErrorCodes::CANNOT_SET_SIGNAL_HANDLER); -#endif - - loop(); + while (true); if (isNewYearMode()) std::cout << "Happy new year." << std::endl; @@ -624,7 +546,7 @@ private: /// This is intended for testing purposes. if (config().getBool("always_load_suggestion_data", false)) { -#if USE_READLINE +#ifdef USE_REPLXX SCOPE_EXIT({ Suggest::instance().finalize(); }); Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit")); #else @@ -706,111 +628,11 @@ private: } - /// Check if multi-line query is inserted from the paste buffer. - /// Allows delaying the start of query execution until the entirety of query is inserted. - static bool hasDataInSTDIN() - { - timeval timeout = { 0, 0 }; - fd_set fds; - FD_ZERO(&fds); - FD_SET(STDIN_FILENO, &fds); - return select(1, &fds, nullptr, nullptr, &timeout) == 1; - } - inline const String prompt() const { return boost::replace_all_copy(prompt_by_server_display_name, "{database}", config().getString("database", "default")); } - void loop() - { - String input; - String prev_input; - - while (char * line_ = readline(input.empty() ? prompt().c_str() : ":-] ")) - { - String line = line_; - free(line_); - - size_t ws = line.size(); - while (ws > 0 && isWhitespaceASCII(line[ws - 1])) - --ws; - - if (ws == 0 || line.empty()) - continue; - - bool ends_with_semicolon = line[ws - 1] == ';'; - bool ends_with_backslash = line[ws - 1] == '\\'; - - has_vertical_output_suffix = (ws >= 2) && (line[ws - 2] == '\\') && (line[ws - 1] == 'G'); - - if (ends_with_backslash) - line = line.substr(0, ws - 1); - - input += line; - - if (!ends_with_backslash && (ends_with_semicolon || has_vertical_output_suffix || (!config().has("multiline") && !hasDataInSTDIN()))) - { - // TODO: should we do sensitive data masking on client too? History file can be source of secret leaks. - if (input != prev_input) - { - /// Replace line breaks with spaces to prevent the following problem. - /// Every line of multi-line query is saved to history file as a separate line. - /// If the user restarts the client then after pressing the "up" button - /// every line of the query will be displayed separately. - std::string logged_query = input; - if (config().has("multiline")) - std::replace(logged_query.begin(), logged_query.end(), '\n', ' '); - add_history(logged_query.c_str()); - -#if USE_READLINE && HAVE_READLINE_HISTORY - if (!history_file.empty() && append_history(1, history_file.c_str())) - std::cerr << "Cannot append history to file " + history_file + ": " + errnoToString(ErrorCodes::CANNOT_APPEND_HISTORY); -#endif - - prev_input = input; - } - - if (has_vertical_output_suffix) - input = input.substr(0, input.length() - 2); - - try - { - if (!process(input)) - break; - } - catch (const Exception & e) - { - actual_client_error = e.code(); - if (!actual_client_error || actual_client_error != expected_client_error) - { - std::cerr << std::endl - << "Exception on client:" << std::endl - << "Code: " << e.code() << ". " << e.displayText() << std::endl; - - if (config().getBool("stacktrace", false)) - std::cerr << "Stack trace:" << std::endl - << e.getStackTraceString() << std::endl; - - std::cerr << std::endl; - - } - - /// Client-side exception during query execution can result in the loss of - /// sync in the connection protocol. - /// So we reconnect and allow to enter the next query. - connect(); - } - - input = ""; - } - else - { - input += '\n'; - } - } - } - void nonInteractive() { diff --git a/dbms/programs/client/Suggest.cpp b/dbms/programs/client/Suggest.cpp new file mode 100644 index 00000000000..65487e07a93 --- /dev/null +++ b/dbms/programs/client/Suggest.cpp @@ -0,0 +1,144 @@ +#include "Suggest.h" + +#include +#include + +namespace DB +{ + +void Suggest::load(const ConnectionParameters & connection_parameters, size_t suggestion_limit) +{ + loading_thread = std::thread([connection_parameters, suggestion_limit, this] + { + try + { + Connection connection( + connection_parameters.host, + connection_parameters.port, + connection_parameters.default_database, + connection_parameters.user, + connection_parameters.password, + "client", + connection_parameters.compression, + connection_parameters.security); + + loadImpl(connection, connection_parameters.timeouts, suggestion_limit); + } + catch (...) + { + std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n"; + } + + /// Note that keyword suggestions are available even if we cannot load data from server. + + std::sort(words.begin(), words.end()); + ready = true; + }); +} + +Suggest::Suggest() +{ + /// Keywords may be not up to date with ClickHouse parser. + words = {"CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT", + "MATERIALIZED", "ALIAS", "ENGINE", "AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", "DROP", + "RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER", "COPY", "PROJECT", + "PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", "INTO", + "OUTFILE", "FORMAT", "TABLES", "DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", "ELSE", + "END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE", "INSERT", "VALUES", + "SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", "INNER", + "LEFT", "RIGHT", "FULL", "OUTER", "CROSS", "USING", "PREWHERE", "WHERE", "GROUP", "BY", + "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC", + "IN", "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE"}; +} + +void Suggest::loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit) +{ + std::stringstream query; + query << "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM (" + "SELECT name FROM system.functions" + " UNION ALL " + "SELECT name FROM system.table_engines" + " UNION ALL " + "SELECT name FROM system.formats" + " UNION ALL " + "SELECT name FROM system.table_functions" + " UNION ALL " + "SELECT name FROM system.data_type_families" + " UNION ALL " + "SELECT name FROM system.settings" + " UNION ALL " + "SELECT cluster FROM system.clusters" + " UNION ALL " + "SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate"; + + /// The user may disable loading of databases, tables, columns by setting suggestion_limit to zero. + if (suggestion_limit > 0) + { + String limit_str = toString(suggestion_limit); + query << + " UNION ALL " + "SELECT name FROM system.databases LIMIT " << limit_str + << " UNION ALL " + "SELECT DISTINCT name FROM system.tables LIMIT " << limit_str + << " UNION ALL " + "SELECT DISTINCT name FROM system.columns LIMIT " << limit_str; + } + + query << ") WHERE notEmpty(res)"; + + fetch(connection, timeouts, query.str()); +} + +void Suggest::fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query) +{ + connection.sendQuery(timeouts, query); + + while (true) + { + Packet packet = connection.receivePacket(); + switch (packet.type) + { + case Protocol::Server::Data: + fillWordsFromBlock(packet.block); + continue; + + case Protocol::Server::Progress: + continue; + case Protocol::Server::ProfileInfo: + continue; + case Protocol::Server::Totals: + continue; + case Protocol::Server::Extremes: + continue; + case Protocol::Server::Log: + continue; + + case Protocol::Server::Exception: + packet.exception->rethrow(); + return; + + case Protocol::Server::EndOfStream: + return; + + default: + throw Exception("Unknown packet from server", ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); + } + } +} + +void Suggest::fillWordsFromBlock(const Block & block) +{ + if (!block) + return; + + if (block.columns() != 1) + throw Exception("Wrong number of columns received for query to read words for suggestion", ErrorCodes::LOGICAL_ERROR); + + const ColumnString & column = typeid_cast(*block.getByPosition(0).column); + + size_t rows = block.rows(); + for (size_t i = 0; i < rows; ++i) + words.emplace_back(column.getDataAt(i).toString()); +} + +} diff --git a/dbms/programs/client/Suggest.h b/dbms/programs/client/Suggest.h index e37bd54f69d..2fea534a986 100644 --- a/dbms/programs/client/Suggest.h +++ b/dbms/programs/client/Suggest.h @@ -2,18 +2,9 @@ #include "ConnectionParameters.h" -#include -#include -#include -#include -#include - -#include - -#include -#include #include #include +#include namespace DB @@ -24,208 +15,34 @@ namespace ErrorCodes extern const int UNKNOWN_PACKET_FROM_SERVER; } -class Suggest : private boost::noncopyable +class Suggest : public LineReader::Suggest, boost::noncopyable { -private: - /// The vector will be filled with completion words from the server and sorted. - using Words = std::vector; - - /// Keywords may be not up to date with ClickHouse parser. - Words words - { - "CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT", "MATERIALIZED", "ALIAS", "ENGINE", - "AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", "DROP", "RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER", - "COPY", "PROJECT", "PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", "INTO", "OUTFILE", "FORMAT", "TABLES", - "DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", "ELSE", "END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE", - "INSERT", "VALUES", "SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", "INNER", "LEFT", "RIGHT", "FULL", "OUTER", - "CROSS", "USING", "PREWHERE", "WHERE", "GROUP", "BY", "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC", "IN", - "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE" - }; - - /// Words are fetched asynchronously. - std::thread loading_thread; - std::atomic ready{false}; - - /// Points to current word to suggest. - Words::const_iterator pos; - /// Points after the last possible match. - Words::const_iterator end; - - /// Set iterators to the matched range of words if any. - void findRange(const char * prefix, size_t prefix_length) - { - std::string prefix_str(prefix); - std::tie(pos, end) = std::equal_range(words.begin(), words.end(), prefix_str, - [prefix_length](const std::string & s, const std::string & prefix_searched) { return strncmp(s.c_str(), prefix_searched.c_str(), prefix_length) < 0; }); - } - - /// Iterates through matched range. - char * nextMatch() - { - if (pos >= end) - return nullptr; - - /// readline will free memory by itself. - char * word = strdup(pos->c_str()); - ++pos; - return word; - } - - void loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit) - { - std::stringstream query; - query << "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM (" - "SELECT name FROM system.functions" - " UNION ALL " - "SELECT name FROM system.table_engines" - " UNION ALL " - "SELECT name FROM system.formats" - " UNION ALL " - "SELECT name FROM system.table_functions" - " UNION ALL " - "SELECT name FROM system.data_type_families" - " UNION ALL " - "SELECT name FROM system.settings" - " UNION ALL " - "SELECT cluster FROM system.clusters" - " UNION ALL " - "SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate"; - - /// The user may disable loading of databases, tables, columns by setting suggestion_limit to zero. - if (suggestion_limit > 0) - { - String limit_str = toString(suggestion_limit); - query << - " UNION ALL " - "SELECT name FROM system.databases LIMIT " << limit_str - << " UNION ALL " - "SELECT DISTINCT name FROM system.tables LIMIT " << limit_str - << " UNION ALL " - "SELECT DISTINCT name FROM system.columns LIMIT " << limit_str; - } - - query << ") WHERE notEmpty(res)"; - - fetch(connection, timeouts, query.str()); - } - - void fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query) - { - connection.sendQuery(timeouts, query); - - while (true) - { - Packet packet = connection.receivePacket(); - switch (packet.type) - { - case Protocol::Server::Data: - fillWordsFromBlock(packet.block); - continue; - - case Protocol::Server::Progress: - continue; - case Protocol::Server::ProfileInfo: - continue; - case Protocol::Server::Totals: - continue; - case Protocol::Server::Extremes: - continue; - case Protocol::Server::Log: - continue; - - case Protocol::Server::Exception: - packet.exception->rethrow(); - return; - - case Protocol::Server::EndOfStream: - return; - - default: - throw Exception("Unknown packet from server", ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); - } - } - } - - void fillWordsFromBlock(const Block & block) - { - if (!block) - return; - - if (block.columns() != 1) - throw Exception("Wrong number of columns received for query to read words for suggestion", ErrorCodes::LOGICAL_ERROR); - - const ColumnString & column = typeid_cast(*block.getByPosition(0).column); - - size_t rows = block.rows(); - for (size_t i = 0; i < rows; ++i) - words.emplace_back(column.getDataAt(i).toString()); - } - public: - static Suggest & instance() + static Suggest * instance() { static Suggest instance; - return instance; + return &instance; } - /// More old server versions cannot execute the query above. + void load(const ConnectionParameters & connection_parameters, size_t suggestion_limit); + + /// Older server versions cannot execute the query above. static constexpr int MIN_SERVER_REVISION = 54406; - void load(const ConnectionParameters & connection_parameters, size_t suggestion_limit) - { - loading_thread = std::thread([connection_parameters, suggestion_limit, this] - { - try - { - Connection connection( - connection_parameters.host, - connection_parameters.port, - connection_parameters.default_database, - connection_parameters.user, - connection_parameters.password, - "client", - connection_parameters.compression, - connection_parameters.security); - - loadImpl(connection, connection_parameters.timeouts, suggestion_limit); - } - catch (...) - { - std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n"; - } - - /// Note that keyword suggestions are available even if we cannot load data from server. - - std::sort(words.begin(), words.end()); - ready = true; - }); - } - - void finalize() +private: + Suggest(); + ~Suggest() { if (loading_thread.joinable()) loading_thread.join(); } - /// A function for readline. - static char * generator(const char * text, int state) - { - Suggest & suggest = Suggest::instance(); - if (!suggest.ready) - return nullptr; - if (state == 0) - suggest.findRange(text, strlen(text)); + void loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit); + void fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query); + void fillWordsFromBlock(const Block & block); - /// Do not append whitespace after word. For unknown reason, rl_completion_append_character = '\0' does not work. - rl_completion_suppress_append = 1; - - return suggest.nextMatch(); - } - - ~Suggest() - { - finalize(); - } + /// Words are fetched asynchronously. + std::thread loading_thread; }; } diff --git a/dbms/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/dbms/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index 25e7086c1a6..65c4f19b7cb 100644 --- a/dbms/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/dbms/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -36,7 +36,6 @@ const char * auto_config_build[] "USE_INTERNAL_MEMCPY", "@USE_INTERNAL_MEMCPY@", "USE_GLIBC_COMPATIBILITY", "@GLIBC_COMPATIBILITY@", "USE_JEMALLOC", "@USE_JEMALLOC@", - "USE_TCMALLOC", "@USE_TCMALLOC@", "USE_MIMALLOC", "@USE_MIMALLOC@", "USE_UNWIND", "@USE_UNWIND@", "USE_ICU", "@USE_ICU@", diff --git a/dbms/tests/queries/0_stateless/00981_no_virtual_columns.reference b/dbms/tests/queries/0_stateless/00981_no_virtual_columns.reference index c1df99e5f94..a7ec77dc030 100644 --- a/dbms/tests/queries/0_stateless/00981_no_virtual_columns.reference +++ b/dbms/tests/queries/0_stateless/00981_no_virtual_columns.reference @@ -1,2 +1 @@ default merge_ab x UInt8 0 0 0 0 0 0 0 -default as_kafka x UInt8 0 0 0 0 0 0 0 diff --git a/dbms/tests/queries/0_stateless/00981_no_virtual_columns.sql b/dbms/tests/queries/0_stateless/00981_no_virtual_columns.sql index 43c08b71b97..476377b4ddf 100644 --- a/dbms/tests/queries/0_stateless/00981_no_virtual_columns.sql +++ b/dbms/tests/queries/0_stateless/00981_no_virtual_columns.sql @@ -1,26 +1,13 @@ DROP TABLE IF EXISTS merge_a; DROP TABLE IF EXISTS merge_b; DROP TABLE IF EXISTS merge_ab; -DROP TABLE IF EXISTS kafka; -DROP TABLE IF EXISTS as_kafka; CREATE TABLE merge_a (x UInt8) ENGINE = StripeLog; CREATE TABLE merge_b (x UInt8) ENGINE = StripeLog; CREATE TABLE merge_ab AS merge(currentDatabase(), '^merge_[ab]$'); -CREATE TABLE kafka (x UInt8) - ENGINE = Kafka - SETTINGS kafka_broker_list = 'kafka', - kafka_topic_list = 'topic', - kafka_group_name = 'group', - kafka_format = 'CSV'; -CREATE TABLE as_kafka AS kafka ENGINE = Memory; - SELECT * FROM system.columns WHERE database = currentDatabase() AND table = 'merge_ab'; -SELECT * FROM system.columns WHERE database = currentDatabase() AND table = 'as_kafka'; DROP TABLE merge_a; DROP TABLE merge_b; DROP TABLE merge_ab; -DROP TABLE kafka; -DROP TABLE as_kafka; diff --git a/docker/packager/packager b/docker/packager/packager index 5e8ffbf1cb9..62767cae8f0 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -176,7 +176,8 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ result.append("ALIEN_PKGS='" + ' '.join(['--' + pkg for pkg in alien_pkgs]) + "'") if unbundled: - cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0') + # TODO: fix build with ENABLE_RDKAFKA + cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0 -DENABLE_READLINE=0 -DENABLE_RDKAFKA=0') if split_binary: cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1') diff --git a/libs/libcommon/CMakeLists.txt b/libs/libcommon/CMakeLists.txt index 878fcf5585d..3267bbe6ce1 100644 --- a/libs/libcommon/CMakeLists.txt +++ b/libs/libcommon/CMakeLists.txt @@ -11,46 +11,48 @@ if (DEFINED APPLE_HAVE_CLOCK_GETTIME) endif () add_library (common + src/argsToConfig.cpp + src/coverage.cpp src/DateLUT.cpp src/DateLUTImpl.cpp - src/preciseExp10.c - src/shift10.cpp - src/mremap.cpp - src/JSON.cpp - src/getMemoryAmount.cpp src/demangle.cpp - src/setTerminalEcho.cpp + src/getMemoryAmount.cpp src/getThreadNumber.cpp - src/sleep.cpp - src/argsToConfig.cpp + src/JSON.cpp + src/LineReader.cpp + src/mremap.cpp src/phdr_cache.cpp - src/coverage.cpp + src/preciseExp10.c + src/setTerminalEcho.cpp + src/shift10.cpp + src/sleep.cpp - include/common/SimpleCache.h - include/common/Types.h - include/common/DayNum.h + include/common/constexpr_helpers.h + include/common/coverage.h include/common/DateLUT.h include/common/DateLUTImpl.h + include/common/DayNum.h + include/common/demangle.h + include/common/ErrorHandlers.h + include/common/find_symbols.h + include/common/getMemoryAmount.h + include/common/getThreadNumber.h + include/common/JSON.h + include/common/likely.h + include/common/LineReader.h include/common/LocalDate.h include/common/LocalDateTime.h - include/common/ErrorHandlers.h - include/common/preciseExp10.h - include/common/shift10.h - include/common/mremap.h - include/common/likely.h include/common/logger_useful.h - include/common/strong_typedef.h - include/common/JSON.h - include/common/getMemoryAmount.h - include/common/demangle.h - include/common/setTerminalEcho.h - include/common/find_symbols.h - include/common/constexpr_helpers.h - include/common/getThreadNumber.h - include/common/sleep.h - include/common/SimpleCache.h + include/common/mremap.h include/common/phdr_cache.h - include/common/coverage.h + include/common/preciseExp10.h + include/common/setTerminalEcho.h + include/common/shift10.h + include/common/SimpleCache.h + include/common/SimpleCache.h + include/common/sleep.h + include/common/strong_typedef.h + include/common/Types.h include/ext/bit_cast.h include/ext/chrono_io.h @@ -90,6 +92,10 @@ if(CCTZ_LIBRARY) target_link_libraries(common PRIVATE ${CCTZ_LIBRARY}) endif() +if (ENABLE_READLINE) + target_link_libraries(common PRIVATE replxx) +endif () + target_link_libraries (common PUBLIC ${Poco_Util_LIBRARY} diff --git a/libs/libcommon/include/common/LineReader.h b/libs/libcommon/include/common/LineReader.h new file mode 100644 index 00000000000..120ff76dac6 --- /dev/null +++ b/libs/libcommon/include/common/LineReader.h @@ -0,0 +1,57 @@ +#pragma once + +#include + +#include +#include + +class LineReader +{ +public: + class Suggest + { + protected: + using Words = std::vector; + using WordsRange = std::pair; + + Words words; + std::atomic ready{false}; + + public: + /// Get iterators for the matched range of words if any. + WordsRange getCompletions(const String & prefix, size_t prefix_length) const; + }; + + LineReader(const Suggest * suggest, const String & history_file_path, char extender, char delimiter = 0); /// if delimiter != 0, then it's multiline mode + ~LineReader(); + + /// Reads the whole line until delimiter (in multiline mode) or until the last line without extender. + /// If resulting line is empty, it means the user interrupted the input. + /// Non-empty line is appended to history - without duplication. + /// Typical delimiter is ';' (semicolon) and typical extender is '\' (backslash). + String readLine(const String & first_prompt, const String & second_prompt); + +private: + enum InputStatus + { + ABORT = 0, + RESET_LINE, + INPUT_LINE, + }; + + String input; + String prev_line; + const String history_file_path; + const char extender; + const char delimiter; + + InputStatus readOneLine(const String & prompt); + void addToHistory(const String & line); + + /// Since CMake doesn't impose restrictions on includes between unrelated targets + /// it's possible that we include this file without USE_REPLXX defined. +#ifdef __clang__ + [[maybe_unused]] +#endif + void * impl; +}; diff --git a/libs/libcommon/include/common/Types.h b/libs/libcommon/include/common/Types.h index 5d933f218c1..f499fbad012 100644 --- a/libs/libcommon/include/common/Types.h +++ b/libs/libcommon/include/common/Types.h @@ -1,8 +1,10 @@ #pragma once + +#include #include #include +#include #include -#include using Int8 = int8_t; using Int16 = int16_t; @@ -14,6 +16,8 @@ using UInt16 = uint16_t; using UInt32 = uint32_t; using UInt64 = uint64_t; +using String = std::string; + /// The standard library type traits, such as std::is_arithmetic, with one exception /// (std::common_type), are "set in stone". Attempting to specialize them causes undefined behavior. /// So instead of using the std type_traits, we use our own version which allows extension. @@ -52,4 +56,3 @@ struct is_arithmetic template inline constexpr bool is_arithmetic_v = is_arithmetic::value; - diff --git a/libs/libcommon/include/common/config_common.h.in b/libs/libcommon/include/common/config_common.h.in index 810cf0b87f9..41999bb5cde 100644 --- a/libs/libcommon/include/common/config_common.h.in +++ b/libs/libcommon/include/common/config_common.h.in @@ -2,10 +2,6 @@ // .h autogenerated by cmake ! -#cmakedefine01 USE_TCMALLOC #cmakedefine01 USE_JEMALLOC -#cmakedefine01 USE_READLINE -#cmakedefine01 USE_LIBEDIT -#cmakedefine01 HAVE_READLINE_HISTORY #cmakedefine01 UNBUNDLED #cmakedefine01 WITH_COVERAGE diff --git a/libs/libcommon/include/common/readline_use.h b/libs/libcommon/include/common/readline_use.h deleted file mode 100644 index 2d9c6d154c1..00000000000 --- a/libs/libcommon/include/common/readline_use.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#if __has_include() -#include -#endif - -/// Different line editing libraries can be used depending on the environment. -#if USE_READLINE - #include - #include -#elif USE_LIBEDIT - #include -#else - #include - #include - #include - inline char * readline(const char * prompt) - { - std::string s; - std::cout << prompt; - std::getline(std::cin, s); - - if (!std::cin.good()) - return nullptr; - return strdup(s.data()); - } - #define add_history(...) do {} while (0) - #define rl_bind_key(...) do {} while (0) -#endif diff --git a/libs/libcommon/src/LineReader.cpp b/libs/libcommon/src/LineReader.cpp new file mode 100644 index 00000000000..5e4c853b185 --- /dev/null +++ b/libs/libcommon/src/LineReader.cpp @@ -0,0 +1,153 @@ +#include + +#ifdef USE_REPLXX +# include +#endif + +#include + +#include +#include + +namespace +{ + +/// Trim ending whitespace inplace +void trim(String & s) +{ + s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end()); +} + +/// Check if multi-line query is inserted from the paste buffer. +/// Allows delaying the start of query execution until the entirety of query is inserted. +bool hasInputData() +{ + timeval timeout = {0, 0}; + fd_set fds; + FD_ZERO(&fds); + FD_SET(STDIN_FILENO, &fds); + return select(1, &fds, nullptr, nullptr, &timeout) == 1; +} + +} + +LineReader::Suggest::WordsRange LineReader::Suggest::getCompletions(const String & prefix, size_t prefix_length) const +{ + if (!ready) + return std::make_pair(words.end(), words.end()); + + return std::equal_range( + words.begin(), words.end(), prefix, [prefix_length](const std::string & s, const std::string & prefix_searched) + { + return strncmp(s.c_str(), prefix_searched.c_str(), prefix_length) < 0; + }); +} + +LineReader::LineReader(const Suggest * suggest, const String & history_file_path_, char extender_, char delimiter_) + : history_file_path(history_file_path_), extender(extender_), delimiter(delimiter_) +{ +#ifdef USE_REPLXX + impl = new replxx::Replxx; + auto & rx = *(replxx::Replxx*)(impl); + + if (!history_file_path.empty()) + rx.history_load(history_file_path); + + auto callback = [suggest] (const String & context, size_t context_size) + { + auto range = suggest->getCompletions(context, context_size); + return replxx::Replxx::completions_t(range.first, range.second); + }; + + if (suggest) + { + rx.set_completion_callback(callback); + rx.set_complete_on_empty(false); + rx.set_word_break_characters(" \t\n\r\"\\'`@$><=;|&{(."); + } +#endif + /// FIXME: check extender != delimiter +} + +LineReader::~LineReader() +{ +#ifdef USE_REPLXX + auto & rx = *(replxx::Replxx*)(impl); + if (!history_file_path.empty()) + rx.history_save(history_file_path); + delete (replxx::Replxx *)impl; +#endif +} + +String LineReader::readLine(const String & first_prompt, const String & second_prompt) +{ + String line; + bool is_multiline = false; + + while (auto status = readOneLine(is_multiline ? second_prompt : first_prompt)) + { + if (status == RESET_LINE) + { + line.clear(); + is_multiline = false; + continue; + } + + if (input.empty()) + continue; + + is_multiline = (input.back() == extender) || (delimiter && input.back() != delimiter) || hasInputData(); + + if (input.back() == extender) + { + input = input.substr(0, input.size() - 1); + trim(input); + if (input.empty()) + continue; + } + + line += (line.empty() ? "" : " ") + input; + + if (!is_multiline) + { + if (line != prev_line) + { + addToHistory(line); + prev_line = line; + } + + return line; + } + } + + return {}; +} + +LineReader::InputStatus LineReader::readOneLine(const String & prompt) +{ + input.clear(); + +#ifdef USE_REPLXX + auto & rx = *(replxx::Replxx*)(impl); + const char* cinput = rx.input(prompt); + if (cinput == nullptr) + return (errno != EAGAIN) ? ABORT : RESET_LINE; + input = cinput; +#else + std::cout << prompt; + std::getline(std::cin, input); + if (!std::cin.good()) + return ABORT; +#endif + + trim(input); + return INPUT_LINE; +} + +void LineReader::addToHistory(const String & line) +{ +#ifdef USE_REPLXX + auto & rx = *(replxx::Replxx*)(impl); + rx.history_add(line); +#endif +} diff --git a/utils/zookeeper-cli/CMakeLists.txt b/utils/zookeeper-cli/CMakeLists.txt index 550d0e855d8..7c14ed605fb 100644 --- a/utils/zookeeper-cli/CMakeLists.txt +++ b/utils/zookeeper-cli/CMakeLists.txt @@ -1,6 +1,3 @@ add_executable(clickhouse-zookeeper-cli zookeeper-cli.cpp) target_link_libraries(clickhouse-zookeeper-cli PRIVATE clickhouse_common_zookeeper ${Poco_Foundation_LIBRARY} ${LINE_EDITING_LIBS}) -if (READLINE_INCLUDE_DIR) - target_include_directories (clickhouse-zookeeper-cli SYSTEM PRIVATE ${READLINE_INCLUDE_DIR}) -endif () INSTALL(TARGETS clickhouse-zookeeper-cli RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse-utils) diff --git a/utils/zookeeper-cli/zookeeper-cli.cpp b/utils/zookeeper-cli/zookeeper-cli.cpp index 6655358f105..5e36ffecdaa 100644 --- a/utils/zookeeper-cli/zookeeper-cli.cpp +++ b/utils/zookeeper-cli/zookeeper-cli.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include @@ -69,12 +69,13 @@ int main(int argc, char ** argv) Logger::root().setLevel("trace"); zkutil::ZooKeeper zk(argv[1]); + LineReader lr(nullptr, {}, '\\'); - while (char * line_ = readline(":3 ")) + do { - add_history(line_); - std::string line(line_); - free(line_); + const auto & line = lr.readLine(":3 ", ":3 "); + if (line.empty()) + break; try { @@ -211,6 +212,7 @@ int main(int argc, char ** argv) std::cerr << "KeeperException: " << e.displayText() << std::endl; } } + while (true); } catch (const Coordination::Exception & e) {