From 1ec5773dc6225b68c1d4a8972f53779a29e99749 Mon Sep 17 00:00:00 2001 From: Aleksei Golub Date: Thu, 20 Apr 2023 20:07:44 +0300 Subject: [PATCH] Added dashed-settings and support for unicode dash in cli arguments --- base/base/argsToConfig.cpp | 26 +++++-- base/base/argsToConfig.h | 7 +- src/Client/ClientBase.cpp | 69 ++++++++++++++++++- ...02718_cli_dashed_options_parsing.reference | 12 ++++ .../02718_cli_dashed_options_parsing.sh | 44 ++++++++++++ 5 files changed, 150 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/02718_cli_dashed_options_parsing.reference create mode 100755 tests/queries/0_stateless/02718_cli_dashed_options_parsing.sh diff --git a/base/base/argsToConfig.cpp b/base/base/argsToConfig.cpp index d7983779d2d..faa1462218d 100644 --- a/base/base/argsToConfig.cpp +++ b/base/base/argsToConfig.cpp @@ -3,13 +3,29 @@ #include #include - -void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::LayeredConfiguration & config, int priority) +void argsToConfig(const Poco::Util::Application::ArgVec & argv, + Poco::Util::LayeredConfiguration & config, + int priority, + const std::unordered_set* alias_names) { /// Parsing all args and converting to config layer /// Test: -- --1=1 --1=2 --3 5 7 8 -9 10 -11=12 14= 15== --16==17 --=18 --19= --20 21 22 --23 --24 25 --26 -27 28 ---29=30 -- ----31 32 --33 3-4 Poco::AutoPtr map_config = new Poco::Util::MapConfiguration; std::string key; + + auto add_arg = [&map_config, &alias_names](const std::string & k, const std::string & v) + { + map_config->setString(k, v); + + if (alias_names && !alias_names->contains(k)) + { + std::string alias_key = k; + std::replace(alias_key.begin(), alias_key.end(), '-', '_'); + if (alias_names->contains(alias_key)) + map_config->setString(alias_key, v); + } + }; + for (const auto & arg : argv) { auto key_start = arg.find_first_not_of('-'); @@ -19,7 +35,7 @@ void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::Laye // old saved '--key', will set to some true value "1" if (!key.empty() && pos_minus != std::string::npos && pos_minus < key_start) { - map_config->setString(key, "1"); + add_arg(key, "1"); key = ""; } @@ -29,7 +45,7 @@ void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::Laye { if (pos_minus == std::string::npos || pos_minus > key_start) { - map_config->setString(key, arg); + add_arg(key, arg); } key = ""; } @@ -55,7 +71,7 @@ void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::Laye if (arg.size() > pos_eq) value = arg.substr(pos_eq + 1); - map_config->setString(key, value); + add_arg(key, value); key = ""; } diff --git a/base/base/argsToConfig.h b/base/base/argsToConfig.h index 9b7b44b7b7f..ef34a8a2145 100644 --- a/base/base/argsToConfig.h +++ b/base/base/argsToConfig.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include namespace Poco::Util { @@ -8,4 +10,7 @@ class LayeredConfiguration; // NOLINT(cppcoreguidelines-virtual-class-destructor } /// Import extra command line arguments to configuration. These are command line arguments after --. -void argsToConfig(const Poco::Util::Application::ArgVec & argv, Poco::Util::LayeredConfiguration & config, int priority); +void argsToConfig(const Poco::Util::Application::ArgVec & argv, + Poco::Util::LayeredConfiguration & config, + int priority, + const std::unordered_set* registered_alias_names = nullptr); diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 120d273aa62..dadd29d4d09 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -68,6 +68,7 @@ #include #include #include +#include #include #include "config_version.h" @@ -120,6 +121,7 @@ namespace ProfileEvents namespace { constexpr UInt64 THREAD_GROUP_ID = 0; +const std::string UNICODE_DASH = "—"; } namespace DB @@ -2395,6 +2397,54 @@ struct TransparentStringHash } }; +/* + * This functor is used to parse command line arguments and replace dashes with underscores, + * allowing options to be specified using either dashes or underscores. + */ +class OptionsAliasParser +{ +public: + explicit OptionsAliasParser(const boost::program_options::options_description& options) + { + options_names.reserve(options.options().size()); + for (const auto& option : options.options()) + options_names.insert(option->long_name()); + } + + /* + * Parses arguments by replacing dashes with underscores, and matches the resulting name with known options + * Implements boost::program_options::ext_parser logic + */ + std::pair operator()(const std::string& token) const + { + if (token.find("--") != 0) + return {}; + std::string arg = token.substr(2); + + // divide token by '=' to separate key and value if options style=long_allow_adjacent + auto pos_eq = arg.find('='); + std::string key = arg.substr(0, pos_eq); + + if (options_names.contains(key)) + // option does not require any changes, because it is already correct + return {}; + + std::replace(key.begin(), key.end(), '-', '_'); + if (!options_names.contains(key)) + // after replacing '-' with '_' argument is still unknown + return {}; + + std::string value; + if (pos_eq != std::string::npos && pos_eq < arg.size()) + value = arg.substr(pos_eq + 1); + + return {key, value}; + } + +private: + std::unordered_set options_names; +}; + } @@ -2445,7 +2495,10 @@ void ClientBase::parseAndCheckOptions(OptionsDescription & options_description, } /// Parse main commandline options. - auto parser = po::command_line_parser(arguments).options(options_description.main_description.value()).allow_unregistered(); + auto parser = po::command_line_parser(arguments) + .options(options_description.main_description.value()) + .extra_parser(OptionsAliasParser(options_description.main_description.value())) + .allow_unregistered(); po::parsed_options parsed = parser.run(); /// Check unrecognized options without positional options. @@ -2487,6 +2540,11 @@ void ClientBase::init(int argc, char ** argv) readArguments(argc, argv, common_arguments, external_tables_arguments, hosts_and_ports_arguments); + /// Support for Unicode dashes + /// Interpret Unicode dash as default double-dash + for (auto & arg : common_arguments) + arg = std::regex_replace(arg, std::regex(UNICODE_DASH), "--"); + po::variables_map options; OptionsDescription options_description; options_description.main_description.emplace(createOptionsDescription("Main options", terminal_width)); @@ -2660,7 +2718,14 @@ void ClientBase::init(int argc, char ** argv) profile_events.delay_ms = options["profile-events-delay-ms"].as(); processOptions(options_description, options, external_tables_arguments, hosts_and_ports_arguments); - argsToConfig(common_arguments, config(), 100); + { + std::unordered_set alias_names; + alias_names.reserve(options_description.main_description->options().size()); + for (const auto& option : options_description.main_description->options()) + alias_names.insert(option->long_name()); + argsToConfig(common_arguments, config(), 100, &alias_names); + } + clearPasswordFromCommandLine(argc, argv); /// Limit on total memory usage diff --git a/tests/queries/0_stateless/02718_cli_dashed_options_parsing.reference b/tests/queries/0_stateless/02718_cli_dashed_options_parsing.reference new file mode 100644 index 00000000000..e294a6c75b6 --- /dev/null +++ b/tests/queries/0_stateless/02718_cli_dashed_options_parsing.reference @@ -0,0 +1,12 @@ +Test 1: Check that you can specify options with a dashes, not an underscores +Test 1.1: Check option from config - server_logs_file +1 +0 +1 +0 +1 +0 +Test 1.1: Check some option from Settings.h - allow_deprecated_syntax_for_merge_tree +0 +Test 2: check that unicode dash processed correctly +1 diff --git a/tests/queries/0_stateless/02718_cli_dashed_options_parsing.sh b/tests/queries/0_stateless/02718_cli_dashed_options_parsing.sh new file mode 100755 index 00000000000..f9548909a88 --- /dev/null +++ b/tests/queries/0_stateless/02718_cli_dashed_options_parsing.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +file_name=${CLICKHOUSE_TEST_UNIQUE_NAME} +file_name_1=${file_name}_1 +file_name_2=${file_name}_2 +file_name_3=${file_name}_3 + +################# +echo "Test 1: Check that you can specify options with a dashes, not an underscores" + +[[ -e $file_name_1 ]] && rm $file_name_1 +[[ -e $file_name_2 ]] && rm $file_name_2 +[[ -e $file_name_3 ]] && rm $file_name_3 + +echo "Test 1.1: Check option from config - server_logs_file" + +$CLICKHOUSE_LOCAL --log-level=debug --server-logs-file=$file_name_1 -q "SELECT 1;" 2> /dev/null +ls | grep -q $file_name_1 +echo $? +$CLICKHOUSE_LOCAL --log-level=debug --server-logs-file $file_name_2 -q "SELECT 1;" 2> /dev/null +ls | grep -q $file_name_2 +echo $? +$CLICKHOUSE_LOCAL --log-level=debug --server_logs_file $file_name_3 -q "SELECT 1;" 2> /dev/null +ls | grep -q $file_name_3 +echo $? + +echo "Test 1.1: Check some option from Settings.h - allow_deprecated_syntax_for_merge_tree" + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS test"; +$CLICKHOUSE_CLIENT --allow-deprecated-syntax-for-merge-tree=1 --query="CREATE TABLE test (d Date, s String) ENGINE = MergeTree(d, s, 8192)"; +$CLICKHOUSE_CLIENT --query="DROP TABLE test"; +echo $? + +################# +echo "Test 2: check that unicode dash processed correctly" +$CLICKHOUSE_LOCAL —query "SELECT 1"; + +rm $file_name_1 +rm $file_name_2 +rm $file_name_3