thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-24 00:22:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactoring of replxx (#8748)

Browse Source
This commit is contained in:
Ivan 2020-01-23 11:18:19 +03:00 committed by GitHub
parent 2e849490bc
commit b80e3dcd46
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 191 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -352,7 +352,6 @@ include (cmake/find/simdjson.cmake)
include (cmake/find/rapidjson.cmake)
include (cmake/find/fastops.cmake)
include (cmake/find/orc.cmake)
include (cmake/find/replxx.cmake)
find_contrib_lib(cityhash)
find_contrib_lib(farmhash)

40
cmake/find/replxx.cmake
View File

@ -1,40 +0,0 @@
option (ENABLE_REPLXX "Enable replxx support" ${NOT_UNBUNDLED})
if (ENABLE_REPLXX)
option (USE_INTERNAL_REPLXX "Use internal replxx library" ${NOT_UNBUNDLED})
if (USE_INTERNAL_REPLXX AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/replxx/README.md")
message (WARNING "submodule contrib/replxx is missing. to fix try run: \n git submodule update --init --recursive")
set (USE_INTERNAL_REPLXX 0)
endif ()
if (NOT USE_INTERNAL_REPLXX)
find_library(LIBRARY_REPLXX NAMES replxx replxx-static)
find_path(INCLUDE_REPLXX replxx.hxx)
add_library(replxx UNKNOWN IMPORTED)
set_property(TARGET replxx PROPERTY IMPORTED_LOCATION ${LIBRARY_REPLXX})
target_include_directories(replxx PUBLIC ${INCLUDE_REPLXX})
set(CMAKE_REQUIRED_LIBRARIES replxx)
check_cxx_source_compiles(
"
#include <replxx.hxx>
int main() {
replxx::Replxx rx;
}
"
EXTERNAL_REPLXX_WORKS
)
if (NOT EXTERNAL_REPLXX_WORKS)
message (FATAL_ERROR "replxx is unusable: ${LIBRARY_REPLXX} ${INCLUDE_REPLXX}")
endif ()
endif ()
set(USE_REPLXX 1)
message (STATUS "Using replxx")
else ()
set(USE_REPLXX 0)
endif ()

4
contrib/CMakeLists.txt vendored
View File

@ -332,6 +332,4 @@ if (USE_FASTOPS)
add_subdirectory (fastops-cmake)
endif()
if (USE_INTERNAL_REPLXX)
add_subdirectory (replxx-cmake)
endif()
add_subdirectory(replxx-cmake)

71
contrib/replxx-cmake/CMakeLists.txt
View File

@ -1,18 +1,57 @@
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/replxx")
option (ENABLE_REPLXX "Enable replxx support" ${ENABLE_LIBRARIES})
set(SRCS
${LIBRARY_DIR}/src/conversion.cxx
${LIBRARY_DIR}/src/escape.cxx
${LIBRARY_DIR}/src/history.cxx
${LIBRARY_DIR}/src/io.cxx
${LIBRARY_DIR}/src/prompt.cxx
${LIBRARY_DIR}/src/replxx.cxx
${LIBRARY_DIR}/src/replxx_impl.cxx
${LIBRARY_DIR}/src/util.cxx
${LIBRARY_DIR}/src/wcwidth.cpp
${LIBRARY_DIR}/src/ConvertUTF.cpp
)
if (ENABLE_REPLXX)
option (USE_INTERNAL_REPLXX "Use internal replxx library" ${NOT_UNBUNDLED})
add_library(replxx ${SRCS})
target_include_directories(replxx PUBLIC ${LIBRARY_DIR}/include)
target_compile_options(replxx PUBLIC -Wno-documentation)
if (USE_INTERNAL_REPLXX)
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/replxx")
set(SRCS
${LIBRARY_DIR}/src/conversion.cxx
${LIBRARY_DIR}/src/ConvertUTF.cpp
${LIBRARY_DIR}/src/escape.cxx
${LIBRARY_DIR}/src/history.cxx
${LIBRARY_DIR}/src/io.cxx
${LIBRARY_DIR}/src/prompt.cxx
${LIBRARY_DIR}/src/replxx_impl.cxx
${LIBRARY_DIR}/src/replxx.cxx
${LIBRARY_DIR}/src/util.cxx
${LIBRARY_DIR}/src/wcwidth.cpp
)
add_library (replxx ${SRCS})
target_include_directories(replxx PUBLIC ${LIBRARY_DIR}/include)
else ()
find_library(LIBRARY_REPLXX NAMES replxx replxx-static)
find_path(INCLUDE_REPLXX replxx.hxx)
add_library(replxx UNKNOWN IMPORTED)
set_property(TARGET replxx PROPERTY IMPORTED_LOCATION ${LIBRARY_REPLXX})
target_include_directories(replxx PUBLIC ${INCLUDE_REPLXX})
set(CMAKE_REQUIRED_LIBRARIES replxx)
check_cxx_source_compiles(
"
#include <replxx.hxx>
int main() {
replxx::Replxx rx;
}
"
EXTERNAL_REPLXX_WORKS
)
if (NOT EXTERNAL_REPLXX_WORKS)
message (FATAL_ERROR "replxx is unusable: ${LIBRARY_REPLXX} ${INCLUDE_REPLXX}")
endif ()
endif ()
target_compile_options(replxx PUBLIC -Wno-documentation)
target_compile_definitions(replxx PUBLIC USE_REPLXX=1)
message (STATUS "Using replxx")
else ()
add_library(replxx INTERFACE)
target_compile_definitions(replxx INTERFACE USE_REPLXX=0)
message (STATUS "Not using replxx (Beware! Runtime fallback to readline is possible!)")
endif ()

2
dbms/programs/client/CMakeLists.txt
View File

@ -4,7 +4,7 @@ set(CLICKHOUSE_CLIENT_SOURCES
${CMAKE_CURRENT_SOURCE_DIR}/Suggest.cpp
)
set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io clickhouse_parsers string_utils ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io clickhouse_parsers string_utils ${Boost_PROGRAM_OPTIONS_LIBRARY})
include(CheckSymbolExists)
check_symbol_exists(readpassphrase readpassphrase.h HAVE_READPASSPHRASE)

13
dbms/programs/client/Client.cpp
View File

@ -2,6 +2,12 @@
#include "ConnectionParameters.h"
#include "Suggest.h"
#if USE_REPLXX
# include <common/ReplxxLineReader.h>
#else
# include <common/LineReader.h>
#endif
#include <stdlib.h>
#include <fcntl.h>
#include <signal.h>
@ -19,7 +25,6 @@
#include <Poco/File.h>
#include <Poco/Util/Application.h>
#include <common/find_symbols.h>
#include <common/config_common.h>
#include <common/LineReader.h>
#include <Common/ClickHouseRevision.h>
#include <Common/Stopwatch.h>
@ -496,7 +501,11 @@ private:
if (!history_file.empty() && !Poco::File(history_file).exists())
Poco::File(history_file).createFile();
LineReader lr(&Suggest::instance(), history_file, '\\', config().has("multiline") ? ';' : 0);
#if USE_REPLXX
ReplxxLineReader lr(Suggest::instance(), history_file, '\\', config().has("multiline") ? ';' : 0);
#else
LineReader lr(history_file, '\\', config().has("multiline") ? ';' : 0);
#endif
do
{

1
dbms/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in
View File

@ -61,7 +61,6 @@ const char * auto_config_build[]
"USE_HYPERSCAN", "@USE_HYPERSCAN@",
"USE_SIMDJSON", "@USE_SIMDJSON@",
"USE_POCO_REDIS", "@USE_POCO_REDIS@",
"USE_REPLXX", "@USE_REPLXX@",
nullptr, nullptr
};

2
docker/packager/packager
View File

@ -177,7 +177,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
if unbundled:
# TODO: fix build with ENABLE_RDKAFKA
cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0 -DENABLE_READLINE=0 -DENABLE_RDKAFKA=0')
cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_POCO_ODBC=0 -DENABLE_ODBC=0 -DENABLE_REPLXX=0 -DENABLE_RDKAFKA=0')
if split_binary:
cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1')

18
libs/libcommon/CMakeLists.txt
View File

@ -10,7 +10,7 @@ if (DEFINED APPLE_HAVE_CLOCK_GETTIME)
target_compile_definitions(apple_rt PUBLIC -DAPPLE_HAVE_CLOCK_GETTIME=${APPLE_HAVE_CLOCK_GETTIME})
endif ()
add_library (common
set (COMMON_SRCS
src/argsToConfig.cpp
src/coverage.cpp
src/DateLUT.cpp
@ -65,7 +65,19 @@ add_library (common
include/ext/scope_guard.h
include/ext/size.h
include/ext/unlock_guard.h
)
if (ENABLE_REPLXX)
set (COMMON_SRCS
src/ReplxxLineReader.cpp
include/common/ReplxxLineReader.h
${COMMON_SRCS}
)
endif ()
add_library (common
${COMMON_SRCS}
${CONFIG_COMMON})
if (USE_INTERNAL_MEMCPY)
@ -92,8 +104,8 @@ if(CCTZ_LIBRARY)
target_link_libraries(common PRIVATE ${CCTZ_LIBRARY})
endif()
if (USE_REPLXX)
target_link_libraries(common PRIVATE replxx)
if (ENABLE_REPLXX)
target_link_libraries(common PUBLIC replxx)
endif ()
target_link_libraries (common

24
libs/libcommon/include/common/LineReader.h
View File

@ -22,8 +22,8 @@ public:
WordsRange getCompletions(const String & prefix, size_t prefix_length) const;
};
LineReader(const Suggest * suggest, const String & history_file_path, char extender, char delimiter = 0); /// if delimiter != 0, then it's multiline mode
~LineReader();
LineReader(const String & history_file_path, char extender, char delimiter = 0); /// if delimiter != 0, then it's multiline mode
virtual ~LineReader() {}
/// Reads the whole line until delimiter (in multiline mode) or until the last line without extender.
/// If resulting line is empty, it means the user interrupted the input.
@ -31,7 +31,7 @@ public:
/// Typical delimiter is ';' (semicolon) and typical extender is '\' (backslash).
String readLine(const String & first_prompt, const String & second_prompt);
private:
protected:
enum InputStatus
{
ABORT = 0,
@ -39,19 +39,17 @@ private:
INPUT_LINE,
};
String input;
String prev_line;
const String history_file_path;
static constexpr char word_break_characters[] = " \t\n\r\"\\'`@$><=;|&{(.";
String input;
private:
const char extender;
const char delimiter;
InputStatus readOneLine(const String & prompt);
void addToHistory(const String & line);
String prev_line;
/// Since CMake doesn't impose restrictions on includes between unrelated targets
/// it's possible that we include this file without USE_REPLXX defined.
#ifdef __clang__
[[maybe_unused]]
#endif
void * impl;
virtual InputStatus readOneLine(const String & prompt);
virtual void addToHistory(const String &) {}
};

18
libs/libcommon/include/common/ReplxxLineReader.h Normal file
View File

@ -0,0 +1,18 @@
#pragma once
#include "LineReader.h"
#include <replxx.hxx>
class ReplxxLineReader : public LineReader
{
public:
ReplxxLineReader(const Suggest & suggest, const String & history_file_path, char extender, char delimiter = 0);
~ReplxxLineReader() override;
private:
InputStatus readOneLine(const String & prompt) override;
void addToHistory(const String & line) override;
replxx::Replxx rx;
};

1
libs/libcommon/include/common/config_common.h.in
View File

@ -3,6 +3,5 @@
// .h autogenerated by cmake !
#cmakedefine01 USE_JEMALLOC
#cmakedefine01 USE_REPLXX
#cmakedefine01 UNBUNDLED
#cmakedefine01 WITH_COVERAGE

74
libs/libcommon/src/LineReader.cpp
View File

@ -1,26 +1,20 @@
#include <common/config_common.h>
#include <common/LineReader.h>
#if USE_REPLXX
#include <replxx.hxx>
#else
/// We can detect if code is linked with one or another readline variants or open the library dynamically.
#include <dlfcn.h>
extern "C"
{
char * readline(const char *) __attribute__((__weak__));
char * (*readline_ptr)(const char *) = readline;
}
#endif
#include <iostream>
#include <string_view>
#include <port/unistd.h>
#include <string.h>
#ifdef OS_LINUX
/// We can detect if code is linked with one or another readline variants or open the library dynamically.
# include <dlfcn.h>
extern "C"
{
char * readline(const char *) __attribute__((__weak__));
char * (*readline_ptr)(const char *) = readline;
}
#endif
namespace
{
@ -42,8 +36,6 @@ bool hasInputData()
return select(1, &fds, nullptr, nullptr, &timeout) == 1;
}
constexpr char word_break_characters[] = " \t\n\r\"\\'`@$><=;|&{(.";
}
LineReader::Suggest::WordsRange LineReader::Suggest::getCompletions(const String & prefix, size_t prefix_length) const
@ -68,39 +60,12 @@ LineReader::Suggest::WordsRange LineReader::Suggest::getCompletions(const String
});
}
LineReader::LineReader(const Suggest * suggest, const String & history_file_path_, char extender_, char delimiter_)
LineReader::LineReader(const String & history_file_path_, char extender_, char delimiter_)
: history_file_path(history_file_path_), extender(extender_), delimiter(delimiter_)
{
#if USE_REPLXX
impl = new replxx::Replxx;
auto & rx = *(replxx::Replxx*)(impl);
if (!history_file_path.empty())
rx.history_load(history_file_path);
auto callback = [suggest] (const String & context, size_t context_size)
{
auto range = suggest->getCompletions(context, context_size);
return replxx::Replxx::completions_t(range.first, range.second);
};
rx.set_completion_callback(callback);
rx.set_complete_on_empty(false);
rx.set_word_break_characters(word_break_characters);
#endif
/// FIXME: check extender != delimiter
}
LineReader::~LineReader()
{
#if USE_REPLXX
auto & rx = *(replxx::Replxx*)(impl);
if (!history_file_path.empty())
rx.history_save(history_file_path);
delete (replxx::Replxx *)impl;
#endif
}
String LineReader::readLine(const String & first_prompt, const String & second_prompt)
{
String line;
@ -149,14 +114,7 @@ LineReader::InputStatus LineReader::readOneLine(const String & prompt)
{
input.clear();
#if USE_REPLXX
auto & rx = *(replxx::Replxx*)(impl);
const char* cinput = rx.input(prompt);
if (cinput == nullptr)
return (errno != EAGAIN) ? ABORT : RESET_LINE;
input = cinput;
#else
#ifdef OS_LINUX
if (!readline_ptr)
{
for (auto name : {"libreadline.so", "libreadline.so.0", "libeditline.so", "libeditline.so.0"})
@ -182,22 +140,14 @@ LineReader::InputStatus LineReader::readOneLine(const String & prompt)
input = line_read;
}
else
#endif
{
std::cout << prompt;
std::getline(std::cin, input);
if (!std::cin.good())
return ABORT;
}
#endif
trim(input);
return INPUT_LINE;
}
void LineReader::addToHistory(const String & line)
{
#if USE_REPLXX
auto & rx = *(replxx::Replxx*)(impl);
rx.history_add(line);
#endif
}

57
libs/libcommon/src/ReplxxLineReader.cpp Normal file
View File

@ -0,0 +1,57 @@
#include <common/ReplxxLineReader.h>
#include <errno.h>
#include <port/unistd.h>
#include <string.h>
namespace
{
/// Trim ending whitespace inplace
void trim(String & s)
{
s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end());
}
}
ReplxxLineReader::ReplxxLineReader(const Suggest & suggest, const String & history_file_path_, char extender_, char delimiter_)
: LineReader(history_file_path_, extender_, delimiter_)
{
if (!history_file_path.empty())
rx.history_load(history_file_path);
auto callback = [&suggest] (const String & context, size_t context_size)
{
auto range = suggest.getCompletions(context, context_size);
return replxx::Replxx::completions_t(range.first, range.second);
};
rx.set_completion_callback(callback);
rx.set_complete_on_empty(false);
rx.set_word_break_characters(word_break_characters);
}
ReplxxLineReader::~ReplxxLineReader()
{
if (!history_file_path.empty())
rx.history_save(history_file_path);
}
LineReader::InputStatus ReplxxLineReader::readOneLine(const String & prompt)
{
input.clear();
const char* cinput = rx.input(prompt);
if (cinput == nullptr)
return (errno != EAGAIN) ? ABORT : RESET_LINE;
input = cinput;
trim(input);
return INPUT_LINE;
}
void ReplxxLineReader::addToHistory(const String & line)
{
rx.history_add(line);
}

2
utils/zookeeper-cli/CMakeLists.txt
View File

@ -1,3 +1,3 @@
add_executable(clickhouse-zookeeper-cli zookeeper-cli.cpp)
target_link_libraries(clickhouse-zookeeper-cli PRIVATE clickhouse_common_zookeeper ${Poco_Foundation_LIBRARY} ${LINE_EDITING_LIBS})
target_link_libraries(clickhouse-zookeeper-cli PRIVATE clickhouse_common_zookeeper ${Poco_Foundation_LIBRARY})
INSTALL(TARGETS clickhouse-zookeeper-cli RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse-utils)

15
utils/zookeeper-cli/zookeeper-cli.cpp
View File

@ -1,12 +1,13 @@
#include <Common/ZooKeeper/ZooKeeper.h>
#include <IO/ReadBufferFromString.h>
#include <IO/ReadHelpers.h>
#include <Poco/ConsoleChannel.h>
#include <Common/ZooKeeper/KeeperException.h>
#include <Common/ZooKeeper/ZooKeeper.h>
#include <common/LineReader.h>
#include <common/logger_useful.h>
#include <iostream>
#include <sstream>
#include <Poco/ConsoleChannel.h>
#include <common/logger_useful.h>
#include <common/LineReader.h>
#include <IO/ReadHelpers.h>
#include <IO/ReadBufferFromString.h>
void printStat(const Coordination::Stat & s)
@ -69,7 +70,7 @@ int main(int argc, char ** argv)
Logger::root().setLevel("trace");
zkutil::ZooKeeper zk(argv[1]);
LineReader lr(nullptr, {}, '\\');
LineReader lr({}, '\\');
do
{