diff --git a/cmake/target.cmake b/cmake/target.cmake index 3c6aa225af9..f1b18786d1d 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -16,7 +16,6 @@ if (CMAKE_CROSSCOMPILING) set (ENABLE_SSL OFF CACHE INTERNAL "") set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") set (ENABLE_PARQUET OFF CACHE INTERNAL "") - set (ENABLE_READLINE OFF CACHE INTERNAL "") set (ENABLE_ICU OFF CACHE INTERNAL "") set (ENABLE_FASTOPS OFF CACHE INTERNAL "") elseif (OS_LINUX) diff --git a/contrib/replxx-cmake/CMakeLists.txt b/contrib/replxx-cmake/CMakeLists.txt index ae49752e376..fed70558b07 100644 --- a/contrib/replxx-cmake/CMakeLists.txt +++ b/contrib/replxx-cmake/CMakeLists.txt @@ -22,6 +22,7 @@ if (ENABLE_READLINE) add_library(replxx ${SRCS}) target_include_directories(replxx PUBLIC ${LIBRARY_DIR}/include) + target_compile_options(replxx PUBLIC -Wno-documentation) else () find_library(LIBRARY_REPLXX NAMES replxx replxx-static) find_path(INCLUDE_REPLXX replxx.hxx) diff --git a/dbms/programs/client/CMakeLists.txt b/dbms/programs/client/CMakeLists.txt index dc5cf787adf..8016ba63b5e 100644 --- a/dbms/programs/client/CMakeLists.txt +++ b/dbms/programs/client/CMakeLists.txt @@ -6,10 +6,6 @@ set(CLICKHOUSE_CLIENT_SOURCES set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io clickhouse_parsers string_utils ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY}) set(CLICKHOUSE_CLIENT_INCLUDE PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/include) -if (READLINE_INCLUDE_DIR) - set(CLICKHOUSE_CLIENT_INCLUDE ${CLICKHOUSE_CLIENT_INCLUDE} SYSTEM PRIVATE ${READLINE_INCLUDE_DIR}) -endif () - include(CheckSymbolExists) check_symbol_exists(readpassphrase readpassphrase.h HAVE_READPASSPHRASE) configure_file(config_client.h.in ${ConfigIncludePath}/config_client.h) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 4b9cee29ff6..98db1cd913b 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -18,8 +18,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -69,10 +69,6 @@ #include #include -#if USE_READLINE -#include "Suggest.h" -#endif - #ifndef __clang__ #pragma GCC optimize("-fno-var-tracking-assignments") #endif @@ -89,39 +85,6 @@ #define DISABLE_LINE_WRAPPING "\033[?7l" #define ENABLE_LINE_WRAPPING "\033[?7h" -#if USE_READLINE && RL_VERSION_MAJOR >= 7 - -#define BRACK_PASTE_PREF "\033[200~" -#define BRACK_PASTE_SUFF "\033[201~" - -#define BRACK_PASTE_LAST '~' -#define BRACK_PASTE_SLEN 6 - -/// This handler bypasses some unused macro/event checkings. -static int clickhouse_rl_bracketed_paste_begin(int /* count */, int /* key */) -{ - std::string buf; - buf.reserve(128); - - RL_SETSTATE(RL_STATE_MOREINPUT); - SCOPE_EXIT(RL_UNSETSTATE(RL_STATE_MOREINPUT)); - int c; - while ((c = rl_read_key()) >= 0) - { - if (c == '\r') - c = '\n'; - buf.push_back(c); - if (buf.size() >= BRACK_PASTE_SLEN && c == BRACK_PASTE_LAST && buf.substr(buf.size() - BRACK_PASTE_SLEN) == BRACK_PASTE_SUFF) - { - buf.resize(buf.size() - BRACK_PASTE_SLEN); - break; - } - } - return static_cast(rl_insert_text(buf.c_str())) == buf.size() ? 0 : 1; -} - -#endif - namespace DB { @@ -443,26 +406,6 @@ private: if (print_time_to_stderr) throw Exception("time option could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS); -#if USE_READLINE - SCOPE_EXIT({ Suggest::instance().finalize(); }); - if (server_revision >= Suggest::MIN_SERVER_REVISION - && !config().getBool("disable_suggestion", false)) - { - /// Load suggestion data from the server. - Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit")); - - /// Added '.' to the default list. Because it is used to separate database and table. - rl_basic_word_break_characters = " \t\n\r\"\\'`@$><=;|&{(."; - - /// Not append whitespace after single suggestion. Because whitespace after function name is meaningless. - rl_completion_append_character = '\0'; - - rl_completion_entry_function = Suggest::generator; - } - else - /// Turn tab completion off. - rl_bind_key('\t', rl_insert); -#endif /// Load command history if present. if (config().has("history_file")) history_file = config().getString("history_file"); @@ -475,70 +418,46 @@ private: history_file = home_path + "/.clickhouse-client-history"; } - if (!history_file.empty()) + if (!history_file.empty() && !Poco::File(history_file).exists()) + Poco::File(history_file).createFile(); + + LineReader lr(history_file, '\\', config().has("multiline") ? ';' : 0); + + do { - if (Poco::File(history_file).exists()) + auto input = lr.readLine(prompt(), ":-] "); + if (input.empty()) + break; + + try { -#if USE_READLINE - int res = read_history(history_file.c_str()); - if (res) - std::cerr << "Cannot read history from file " + history_file + ": "+ errnoToString(ErrorCodes::CANNOT_READ_HISTORY); -#endif + if (!process(input)) + break; + } + catch (const Exception & e) + { + actual_client_error = e.code(); + if (!actual_client_error || actual_client_error != expected_client_error) + { + std::cerr << std::endl + << "Exception on client:" << std::endl + << "Code: " << e.code() << ". " << e.displayText() << std::endl; + + if (config().getBool("stacktrace", false)) + std::cerr << "Stack trace:" << std::endl + << e.getStackTrace().toString() << std::endl; + + std::cerr << std::endl; + + } + + /// Client-side exception during query execution can result in the loss of + /// sync in the connection protocol. + /// So we reconnect and allow to enter the next query. + connect(); } - else /// Create history file. - Poco::File(history_file).createFile(); } - -#if USE_READLINE - /// Install Ctrl+C signal handler that will be used in interactive mode. - - if (rl_initialize()) - throw Exception("Cannot initialize readline", ErrorCodes::CANNOT_READLINE); - -#if RL_VERSION_MAJOR >= 7 - /// Enable bracketed-paste-mode only when multiquery is enabled and multiline is - /// disabled, so that we are able to paste and execute multiline queries in a whole - /// instead of erroring out, while be less intrusive. - if (config().has("multiquery") && !config().has("multiline")) - { - /// When bracketed paste mode is set, pasted text is bracketed with control sequences so - /// that the program can differentiate pasted text from typed-in text. This helps - /// clickhouse-client so that without -m flag, one can still paste multiline queries, and - /// possibly get better pasting performance. See https://cirw.in/blog/bracketed-paste for - /// more details. - rl_variable_bind("enable-bracketed-paste", "on"); - - /// Use our bracketed paste handler to get better user experience. See comments above. - rl_bind_keyseq(BRACK_PASTE_PREF, clickhouse_rl_bracketed_paste_begin); - } -#endif - - auto clear_prompt_or_exit = [](int) - { - /// This is signal safe. - ssize_t res = write(STDOUT_FILENO, "\n", 1); - - /// Allow to quit client while query is in progress by pressing Ctrl+C twice. - /// (First press to Ctrl+C will try to cancel query by InterruptListener). - if (res == 1 && rl_line_buffer[0] && !RL_ISSTATE(RL_STATE_DONE)) - { - rl_replace_line("", 0); - if (rl_forced_update_display()) - _exit(0); - } - else - { - /// A little dirty, but we struggle to find better way to correctly - /// force readline to exit after returning from the signal handler. - _exit(0); - } - }; - - if (signal(SIGINT, clear_prompt_or_exit) == SIG_ERR) - throwFromErrno("Cannot set signal handler.", ErrorCodes::CANNOT_SET_SIGNAL_HANDLER); -#endif - - loop(); + while (true); std::cout << (isNewYearMode() ? "Happy new year." : "Bye.") << std::endl; return 0; @@ -548,12 +467,7 @@ private: /// This is intended for testing purposes. if (config().getBool("always_load_suggestion_data", false)) { -#if USE_READLINE - SCOPE_EXIT({ Suggest::instance().finalize(); }); - Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit")); -#else throw Exception("Command line suggestions cannot work without readline", ErrorCodes::BAD_ARGUMENTS); -#endif } query_id = config().getString("query_id", ""); @@ -648,91 +562,7 @@ private: void loop() { - String input; - String prev_input; - while (char * line_ = readline(input.empty() ? prompt().c_str() : ":-] ")) - { - String line = line_; - free(line_); - - size_t ws = line.size(); - while (ws > 0 && isWhitespaceASCII(line[ws - 1])) - --ws; - - if (ws == 0 || line.empty()) - continue; - - bool ends_with_semicolon = line[ws - 1] == ';'; - bool ends_with_backslash = line[ws - 1] == '\\'; - - has_vertical_output_suffix = (ws >= 2) && (line[ws - 2] == '\\') && (line[ws - 1] == 'G'); - - if (ends_with_backslash) - line = line.substr(0, ws - 1); - - input += line; - - if (!ends_with_backslash && (ends_with_semicolon || has_vertical_output_suffix || (!config().has("multiline") && !hasDataInSTDIN()))) - { - // TODO: should we do sensitive data masking on client too? History file can be source of secret leaks. - if (input != prev_input) - { - /// Replace line breaks with spaces to prevent the following problem. - /// Every line of multi-line query is saved to history file as a separate line. - /// If the user restarts the client then after pressing the "up" button - /// every line of the query will be displayed separately. - std::string logged_query = input; - if (config().has("multiline")) - std::replace(logged_query.begin(), logged_query.end(), '\n', ' '); - add_history(logged_query.c_str()); - -#if USE_READLINE && HAVE_READLINE_HISTORY - if (!history_file.empty() && append_history(1, history_file.c_str())) - std::cerr << "Cannot append history to file " + history_file + ": " + errnoToString(ErrorCodes::CANNOT_APPEND_HISTORY); -#endif - - prev_input = input; - } - - if (has_vertical_output_suffix) - input = input.substr(0, input.length() - 2); - - try - { - if (!process(input)) - break; - } - catch (const Exception & e) - { - actual_client_error = e.code(); - if (!actual_client_error || actual_client_error != expected_client_error) - { - std::cerr << std::endl - << "Exception on client:" << std::endl - << "Code: " << e.code() << ". " << e.displayText() << std::endl; - - if (config().getBool("stacktrace", false)) - std::cerr << "Stack trace:" << std::endl - << e.getStackTrace().toString() << std::endl; - - std::cerr << std::endl; - - } - - /// Client-side exception during query execution can result in the loss of - /// sync in the connection protocol. - /// So we reconnect and allow to enter the next query. - connect(); - } - - input = ""; - } - else - { - input += '\n'; - } - } } diff --git a/dbms/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/dbms/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index 25e7086c1a6..65c4f19b7cb 100644 --- a/dbms/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/dbms/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -36,7 +36,6 @@ const char * auto_config_build[] "USE_INTERNAL_MEMCPY", "@USE_INTERNAL_MEMCPY@", "USE_GLIBC_COMPATIBILITY", "@GLIBC_COMPATIBILITY@", "USE_JEMALLOC", "@USE_JEMALLOC@", - "USE_TCMALLOC", "@USE_TCMALLOC@", "USE_MIMALLOC", "@USE_MIMALLOC@", "USE_UNWIND", "@USE_UNWIND@", "USE_ICU", "@USE_ICU@", diff --git a/libs/libcommon/CMakeLists.txt b/libs/libcommon/CMakeLists.txt index 878fcf5585d..3267bbe6ce1 100644 --- a/libs/libcommon/CMakeLists.txt +++ b/libs/libcommon/CMakeLists.txt @@ -11,46 +11,48 @@ if (DEFINED APPLE_HAVE_CLOCK_GETTIME) endif () add_library (common + src/argsToConfig.cpp + src/coverage.cpp src/DateLUT.cpp src/DateLUTImpl.cpp - src/preciseExp10.c - src/shift10.cpp - src/mremap.cpp - src/JSON.cpp - src/getMemoryAmount.cpp src/demangle.cpp - src/setTerminalEcho.cpp + src/getMemoryAmount.cpp src/getThreadNumber.cpp - src/sleep.cpp - src/argsToConfig.cpp + src/JSON.cpp + src/LineReader.cpp + src/mremap.cpp src/phdr_cache.cpp - src/coverage.cpp + src/preciseExp10.c + src/setTerminalEcho.cpp + src/shift10.cpp + src/sleep.cpp - include/common/SimpleCache.h - include/common/Types.h - include/common/DayNum.h + include/common/constexpr_helpers.h + include/common/coverage.h include/common/DateLUT.h include/common/DateLUTImpl.h + include/common/DayNum.h + include/common/demangle.h + include/common/ErrorHandlers.h + include/common/find_symbols.h + include/common/getMemoryAmount.h + include/common/getThreadNumber.h + include/common/JSON.h + include/common/likely.h + include/common/LineReader.h include/common/LocalDate.h include/common/LocalDateTime.h - include/common/ErrorHandlers.h - include/common/preciseExp10.h - include/common/shift10.h - include/common/mremap.h - include/common/likely.h include/common/logger_useful.h - include/common/strong_typedef.h - include/common/JSON.h - include/common/getMemoryAmount.h - include/common/demangle.h - include/common/setTerminalEcho.h - include/common/find_symbols.h - include/common/constexpr_helpers.h - include/common/getThreadNumber.h - include/common/sleep.h - include/common/SimpleCache.h + include/common/mremap.h include/common/phdr_cache.h - include/common/coverage.h + include/common/preciseExp10.h + include/common/setTerminalEcho.h + include/common/shift10.h + include/common/SimpleCache.h + include/common/SimpleCache.h + include/common/sleep.h + include/common/strong_typedef.h + include/common/Types.h include/ext/bit_cast.h include/ext/chrono_io.h @@ -90,6 +92,10 @@ if(CCTZ_LIBRARY) target_link_libraries(common PRIVATE ${CCTZ_LIBRARY}) endif() +if (ENABLE_READLINE) + target_link_libraries(common PRIVATE replxx) +endif () + target_link_libraries (common PUBLIC ${Poco_Util_LIBRARY} diff --git a/libs/libcommon/include/common/LineReader.h b/libs/libcommon/include/common/LineReader.h new file mode 100644 index 00000000000..a5bc692efc4 --- /dev/null +++ b/libs/libcommon/include/common/LineReader.h @@ -0,0 +1,34 @@ +#pragma once + +#include + +#ifdef USE_REPLXX +# include +#endif + +class LineReader +{ +public: + LineReader(const String & history_file_path, char extender, char delimiter = 0); /// if delimiter != 0, then it's multiline mode + ~LineReader(); + + /// Reads the whole line until delimiter (in multiline mode) or until the last line without extender. + /// If resulting line is empty, it means the user interrupted the input. + /// Non-empty line is appended to history - without duplication. + /// Typical delimiter is ';' (semicolon) and typical extender is '\' (backslash). + String readLine(const String & first_prompt, const String & second_prompt); + +private: + String input; + String prev_line; + const String history_file_path; + const char extender; + const char delimiter; + + bool readOneLine(const String & prompt); + void addToHistory(const String & line); + +#ifdef USE_REPLXX + replxx::Replxx rx; +#endif +}; diff --git a/libs/libcommon/include/common/Types.h b/libs/libcommon/include/common/Types.h index 5d933f218c1..f499fbad012 100644 --- a/libs/libcommon/include/common/Types.h +++ b/libs/libcommon/include/common/Types.h @@ -1,8 +1,10 @@ #pragma once + +#include #include #include +#include #include -#include using Int8 = int8_t; using Int16 = int16_t; @@ -14,6 +16,8 @@ using UInt16 = uint16_t; using UInt32 = uint32_t; using UInt64 = uint64_t; +using String = std::string; + /// The standard library type traits, such as std::is_arithmetic, with one exception /// (std::common_type), are "set in stone". Attempting to specialize them causes undefined behavior. /// So instead of using the std type_traits, we use our own version which allows extension. @@ -52,4 +56,3 @@ struct is_arithmetic template inline constexpr bool is_arithmetic_v = is_arithmetic::value; - diff --git a/libs/libcommon/include/common/config_common.h.in b/libs/libcommon/include/common/config_common.h.in index 810cf0b87f9..41999bb5cde 100644 --- a/libs/libcommon/include/common/config_common.h.in +++ b/libs/libcommon/include/common/config_common.h.in @@ -2,10 +2,6 @@ // .h autogenerated by cmake ! -#cmakedefine01 USE_TCMALLOC #cmakedefine01 USE_JEMALLOC -#cmakedefine01 USE_READLINE -#cmakedefine01 USE_LIBEDIT -#cmakedefine01 HAVE_READLINE_HISTORY #cmakedefine01 UNBUNDLED #cmakedefine01 WITH_COVERAGE diff --git a/libs/libcommon/include/common/readline_use.h b/libs/libcommon/include/common/readline_use.h deleted file mode 100644 index 2d9c6d154c1..00000000000 --- a/libs/libcommon/include/common/readline_use.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#if __has_include() -#include -#endif - -/// Different line editing libraries can be used depending on the environment. -#if USE_READLINE - #include - #include -#elif USE_LIBEDIT - #include -#else - #include - #include - #include - inline char * readline(const char * prompt) - { - std::string s; - std::cout << prompt; - std::getline(std::cin, s); - - if (!std::cin.good()) - return nullptr; - return strdup(s.data()); - } - #define add_history(...) do {} while (0) - #define rl_bind_key(...) do {} while (0) -#endif diff --git a/libs/libcommon/src/LineReader.cpp b/libs/libcommon/src/LineReader.cpp new file mode 100644 index 00000000000..a9d713357ba --- /dev/null +++ b/libs/libcommon/src/LineReader.cpp @@ -0,0 +1,95 @@ +#include + +#include + +namespace +{ + +/// Trim ending whitespace inplace +void trim(String & s) +{ + s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end()); +} + +} + +LineReader::LineReader(const String & history_file_path_, char extender_, char delimiter_) + : history_file_path(history_file_path_), extender(extender_), delimiter(delimiter_) +{ +#ifdef USE_REPLXX + if (!history_file_path.empty()) + rx.history_load(history_file_path); +#endif + /// FIXME: check extender != delimiter +} + +LineReader::~LineReader() +{ +#ifdef USE_REPLXX + if (!history_file_path.empty()) + rx.history_save(history_file_path); +#endif +} + +String LineReader::readLine(const String & first_prompt, const String & second_prompt) +{ + String line; + bool is_multiline = false; + + while (readOneLine(is_multiline ? second_prompt : first_prompt)) + { + if (input.empty()) + continue; + + is_multiline = (input.back() == extender) || (delimiter && input.back() != delimiter); + + if (input.back() == extender) + { + input = input.substr(0, input.size() - 1); + trim(input); + if (input.empty()) + continue; + } + + line += (line.empty() ? "" : " ") + input; + + if (!is_multiline) + { + if (line != prev_line) + { + addToHistory(line); + prev_line = line; + } + + return line; + } + } + + return {}; +} + +bool LineReader::readOneLine(const String & prompt) +{ +#ifdef USE_REPLXX + const char* cinput = rx.input(prompt); + if (cinput == nullptr && errno != EAGAIN) + return false; + input = cinput; +#else + std::cout << prompt; + std::getline(std::cin, input); + if (!std::cin.good()) + return false; +#endif + trim(input); + + return true; +} + +void LineReader::addToHistory(const String & line) +{ +#ifdef USE_REPLXX + rx.history_add(line); + /// TODO: implement this. +#endif +} diff --git a/utils/zookeeper-cli/CMakeLists.txt b/utils/zookeeper-cli/CMakeLists.txt index 550d0e855d8..7c14ed605fb 100644 --- a/utils/zookeeper-cli/CMakeLists.txt +++ b/utils/zookeeper-cli/CMakeLists.txt @@ -1,6 +1,3 @@ add_executable(clickhouse-zookeeper-cli zookeeper-cli.cpp) target_link_libraries(clickhouse-zookeeper-cli PRIVATE clickhouse_common_zookeeper ${Poco_Foundation_LIBRARY} ${LINE_EDITING_LIBS}) -if (READLINE_INCLUDE_DIR) - target_include_directories (clickhouse-zookeeper-cli SYSTEM PRIVATE ${READLINE_INCLUDE_DIR}) -endif () INSTALL(TARGETS clickhouse-zookeeper-cli RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse-utils) diff --git a/utils/zookeeper-cli/zookeeper-cli.cpp b/utils/zookeeper-cli/zookeeper-cli.cpp index 6655358f105..4a76d5c7a81 100644 --- a/utils/zookeeper-cli/zookeeper-cli.cpp +++ b/utils/zookeeper-cli/zookeeper-cli.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include @@ -69,12 +69,13 @@ int main(int argc, char ** argv) Logger::root().setLevel("trace"); zkutil::ZooKeeper zk(argv[1]); + LineReader lr({}, '\\'); - while (char * line_ = readline(":3 ")) + do { - add_history(line_); - std::string line(line_); - free(line_); + const auto & line = lr.readLine(":3 ", ":3 "); + if (line.empty()) + break; try { @@ -211,6 +212,7 @@ int main(int argc, char ** argv) std::cerr << "KeeperException: " << e.displayText() << std::endl; } } + while (true); } catch (const Coordination::Exception & e) {