From f5d1a9e59a1ce9054a9dc96a74bb9bee61fb9ccc Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 14 Oct 2020 11:50:36 +0300 Subject: [PATCH 1/3] Clickhouse client and local added queries-file parameter --- docs/en/interfaces/cli.md | 3 ++- .../operations/utilities/clickhouse-local.md | 3 ++- programs/client/Client.cpp | 25 +++++++++++++++--- programs/local/LocalServer.cpp | 25 ++++++++++++++++-- ...ent_local_queries_file_parameter.reference | 8 ++++++ ...523_client_local_queries_file_parameter.sh | 26 +++++++++++++++++++ 6 files changed, 83 insertions(+), 7 deletions(-) create mode 100644 tests/queries/0_stateless/01523_client_local_queries_file_parameter.reference create mode 100755 tests/queries/0_stateless/01523_client_local_queries_file_parameter.sh diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md index 42416383860..a2ea6edf24a 100644 --- a/docs/en/interfaces/cli.md +++ b/docs/en/interfaces/cli.md @@ -113,7 +113,8 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va - `--port` – The port to connect to. Default value: 9000. Note that the HTTP interface and the native interface use different ports. - `--user, -u` – The username. Default value: default. - `--password` – The password. Default value: empty string. -- `--query, -q` – The query to process when using non-interactive mode. +- `--query, -q` – The query to process when using non-interactive mode. You must specify either `query` or `queries-file` option. +- `--queries-file, -qf` - file path with queries to execute. You must specify either `query` or `queries-file` option. - `--database, -d` – Select the current default database. Default value: the current database from the server settings (‘default’ by default). - `--multiline, -m` – If specified, allow multiline queries (do not send the query on Enter). - `--multiquery, -n` – If specified, allow processing multiple queries separated by semicolons. diff --git a/docs/en/operations/utilities/clickhouse-local.md b/docs/en/operations/utilities/clickhouse-local.md index af3d06898fd..f93ba139cae 100644 --- a/docs/en/operations/utilities/clickhouse-local.md +++ b/docs/en/operations/utilities/clickhouse-local.md @@ -32,7 +32,8 @@ Arguments: - `-S`, `--structure` — table structure for input data. - `-if`, `--input-format` — input format, `TSV` by default. - `-f`, `--file` — path to data, `stdin` by default. -- `-q` `--query` — queries to execute with `;` as delimeter. +- `-q` `--query` — queries to execute with `;` as delimeter. You must specify either `query` or `queries-file` option. +- `-qf` `--queries-file` - file path with queries to execute. You must specify either `query` or `queries-file` option. - `-N`, `--table` — table name where to put output data, `table` by default. - `-of`, `--format`, `--output-format` — output format, `TSV` by default. - `--stacktrace` — whether to dump debug output in case of exception. diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index e4858eeda8b..dcd519bc6dd 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -475,9 +476,17 @@ private: /// The value of the option is used as the text of query (or of multiple queries). /// If stdin is not a terminal, INSERT data for the first query is read from it. /// - stdin is not a terminal. In this case queries are read from it. - if (!stdin_is_a_tty || config().has("query")) + /// - -qf (--queries-file) command line option is present. + /// The value of the option is used as file with query (or of multiple queries) to execute. + if (!stdin_is_a_tty || config().has("query") || config().has("queries-file")) is_interactive = false; + if (config().has("query") && config().has("queries-file")) + { + std::cerr << "Specify either `query` or `queries-file` option" << std::endl; + return Application::EXIT_OK; + } + std::cout << std::fixed << std::setprecision(3); std::cerr << std::fixed << std::setprecision(3); @@ -786,8 +795,15 @@ private: { String text; - if (config().has("query")) - text = config().getRawString("query"); /// Poco configuration should not process substitutions in form of ${...} inside query. + if (config().has("queries-file")) + { + ReadBufferFromFile in(config().getRawString("queries-file")); + readStringUntilEOF(text, in); + processMultiQuery(text); + return; + } + else if (config().has("query")) + text = config().getRawString("query"); /// Poco configuration should not process substitutions in form of ${...} inside query. else { /// If 'query' parameter is not set, read a query from stdin. @@ -2320,6 +2336,7 @@ public: "Suggestion limit for how many databases, tables and columns to fetch.") ("multiline,m", "multiline") ("multiquery,n", "multiquery") + ("queries-file,qf", po::value(), "file path with queries to execute") ("format,f", po::value(), "default output format") ("testmode,T", "enable test hints in comments") ("ignore-error", "do not stop processing in multiquery mode") @@ -2448,6 +2465,8 @@ public: config().setString("query_id", options["query_id"].as()); if (options.count("query")) config().setString("query", options["query"].as()); + if (options.count("queries-file")) + config().setString("queries-file", options["queries-file"].as()); if (options.count("database")) config().setString("database", options["database"].as()); if (options.count("pager")) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 15e71198eb1..60e6f470692 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -20,9 +20,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -195,7 +197,7 @@ try ThreadStatus thread_status; UseSSL use_ssl; - if (!config().has("query") && !config().has("table-structure")) /// Nothing to process + if (!config().has("query") && !config().has("table-structure") && !config().has("queries-file")) /// Nothing to process { if (config().hasOption("verbose")) std::cerr << "There are no queries to process." << '\n'; @@ -203,6 +205,12 @@ try return Application::EXIT_OK; } + if (config().has("query") && config().has("queries-file")) + { + std::cerr << "Specify either `query` or `queries-file` option" << std::endl; + return Application::EXIT_OK; + } + shared_context = Context::createShared(); global_context = std::make_unique(Context::createGlobal(shared_context.get())); global_context->makeGlobalContext(); @@ -340,7 +348,17 @@ std::string LocalServer::getInitialCreateTableQuery() void LocalServer::processQueries() { String initial_create_query = getInitialCreateTableQuery(); - String queries_str = initial_create_query + config().getRawString("query"); + String queries_str = initial_create_query; + + if (config().has("query")) + queries_str += config().getRawString("query"); + else + { + String queries_from_file; + ReadBufferFromFile in(config().getRawString("queries-file")); + readStringUntilEOF(queries_from_file, in); + queries_str += queries_from_file; + } const auto & settings = global_context->getSettingsRef(); @@ -505,6 +523,7 @@ void LocalServer::init(int argc, char ** argv) ("help", "produce help message") ("config-file,c", po::value(), "config-file path") ("query,q", po::value(), "query") + ("queries-file, qf", po::value(), "file path with queries to execute") ("database,d", po::value(), "database") ("table,N", po::value(), "name of the initial table") @@ -552,6 +571,8 @@ void LocalServer::init(int argc, char ** argv) config().setString("config-file", options["config-file"].as()); if (options.count("query")) config().setString("query", options["query"].as()); + if (options.count("queries-file")) + config().setString("queries-file", options["queries-file"].as()); if (options.count("database")) config().setString("default_database", options["database"].as()); diff --git a/tests/queries/0_stateless/01523_client_local_queries_file_parameter.reference b/tests/queries/0_stateless/01523_client_local_queries_file_parameter.reference new file mode 100644 index 00000000000..72465b2fb81 --- /dev/null +++ b/tests/queries/0_stateless/01523_client_local_queries_file_parameter.reference @@ -0,0 +1,8 @@ +1 +1 +2 +3 +Specify either `query` or `queries-file` option +1 2 +3 4 +Specify either `query` or `queries-file` option diff --git a/tests/queries/0_stateless/01523_client_local_queries_file_parameter.sh b/tests/queries/0_stateless/01523_client_local_queries_file_parameter.sh new file mode 100755 index 00000000000..a667615f1c4 --- /dev/null +++ b/tests/queries/0_stateless/01523_client_local_queries_file_parameter.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CURDIR"/../shell_config.sh + +echo "SELECT 1;" > 01523_client_local_queries_file_parameter_tmp.sql +$CLICKHOUSE_CLIENT --queries-file=01523_client_local_queries_file_parameter_tmp.sql 2>&1 + +echo "CREATE TABLE 01523_test(value Int32) ENGINE=Log; +INSERT INTO 01523_test + VALUES (1), (2), (3); +SELECT * FROM 01523_test; +DROP TABLE 01523_test;" > 01523_client_local_queries_file_parameter_tmp.sql +$CLICKHOUSE_CLIENT --queries-file=01523_client_local_queries_file_parameter_tmp.sql 2>&1 + +$CLICKHOUSE_CLIENT --queries='t' --query='t' 2>&1 + +echo "CREATE TABLE table (a Int64, b Int64) ENGINE = File(CSV, stdin); +SELECT a, b FROM table; +DROP TABLE table;" > 01523_client_local_queries_file_parameter_tmp.sql + +echo -e "1,2\n3,4" | $CLICKHOUSE_LOCAL --queries-file=01523_client_local_queries_file_parameter_tmp.sql 2>&1 + +$CLICKHOUSE_LOCAL --queries='t' --query='t' 2>&1 + +rm 01523_client_local_queries_file_parameter_tmp.sql From 963ee5211bbc56c07f8a0cf4b53db5c6b43afdab Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sat, 31 Oct 2020 22:41:19 +0300 Subject: [PATCH 2/3] Added queries-file parameter. Fixed code review issues --- programs/client/Client.cpp | 5 ++--- programs/local/LocalServer.cpp | 5 ++--- ...523_client_local_queries_file_parameter.reference | 12 ++++++------ .../01523_client_local_queries_file_parameter.sh | 2 -- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index dcd519bc6dd..92ae37123fa 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -483,8 +483,7 @@ private: if (config().has("query") && config().has("queries-file")) { - std::cerr << "Specify either `query` or `queries-file` option" << std::endl; - return Application::EXIT_OK; + throw Exception("Specify either `query` or `queries-file` option", ErrorCodes::BAD_ARGUMENTS); } std::cout << std::fixed << std::setprecision(3); @@ -797,7 +796,7 @@ private: if (config().has("queries-file")) { - ReadBufferFromFile in(config().getRawString("queries-file")); + ReadBufferFromFile in(config().getString("queries-file")); readStringUntilEOF(text, in); processMultiQuery(text); return; diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 60e6f470692..dbf153eeb81 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -207,8 +207,7 @@ try if (config().has("query") && config().has("queries-file")) { - std::cerr << "Specify either `query` or `queries-file` option" << std::endl; - return Application::EXIT_OK; + throw Exception("Specify either `query` or `queries-file` option", ErrorCodes::BAD_ARGUMENTS); } shared_context = Context::createShared(); @@ -355,7 +354,7 @@ void LocalServer::processQueries() else { String queries_from_file; - ReadBufferFromFile in(config().getRawString("queries-file")); + ReadBufferFromFile in(config().getString("queries-file")); readStringUntilEOF(queries_from_file, in); queries_str += queries_from_file; } diff --git a/tests/queries/0_stateless/01523_client_local_queries_file_parameter.reference b/tests/queries/0_stateless/01523_client_local_queries_file_parameter.reference index 72465b2fb81..2f534957919 100644 --- a/tests/queries/0_stateless/01523_client_local_queries_file_parameter.reference +++ b/tests/queries/0_stateless/01523_client_local_queries_file_parameter.reference @@ -1,8 +1,8 @@ -1 -1 -2 -3 -Specify either `query` or `queries-file` option +Code: 81. DB::Exception: Received from localhost:9000. DB::Exception: Database `test` doesn't exist. + +Code: 81. DB::Exception: Received from localhost:9000. DB::Exception: Database `test` doesn't exist. + +Code: 36. DB::Exception: Specify either `query` or `queries-file` option + 1 2 3 4 -Specify either `query` or `queries-file` option diff --git a/tests/queries/0_stateless/01523_client_local_queries_file_parameter.sh b/tests/queries/0_stateless/01523_client_local_queries_file_parameter.sh index a667615f1c4..bd33700277a 100755 --- a/tests/queries/0_stateless/01523_client_local_queries_file_parameter.sh +++ b/tests/queries/0_stateless/01523_client_local_queries_file_parameter.sh @@ -21,6 +21,4 @@ DROP TABLE table;" > 01523_client_local_queries_file_parameter_tmp.sql echo -e "1,2\n3,4" | $CLICKHOUSE_LOCAL --queries-file=01523_client_local_queries_file_parameter_tmp.sql 2>&1 -$CLICKHOUSE_LOCAL --queries='t' --query='t' 2>&1 - rm 01523_client_local_queries_file_parameter_tmp.sql From ae562236112d2a3141c845e1aca2248b84160ea4 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 6 Nov 2020 22:00:26 +0300 Subject: [PATCH 3/3] Added queries-file parameter. Updated tests. --- ...01523_client_local_queries_file_parameter.reference | 10 ++++------ .../01523_client_local_queries_file_parameter.sh | 8 +++----- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/queries/0_stateless/01523_client_local_queries_file_parameter.reference b/tests/queries/0_stateless/01523_client_local_queries_file_parameter.reference index 2f534957919..2a3eb745751 100644 --- a/tests/queries/0_stateless/01523_client_local_queries_file_parameter.reference +++ b/tests/queries/0_stateless/01523_client_local_queries_file_parameter.reference @@ -1,8 +1,6 @@ -Code: 81. DB::Exception: Received from localhost:9000. DB::Exception: Database `test` doesn't exist. - -Code: 81. DB::Exception: Received from localhost:9000. DB::Exception: Database `test` doesn't exist. - -Code: 36. DB::Exception: Specify either `query` or `queries-file` option - +1 +1 +2 +3 1 2 3 4 diff --git a/tests/queries/0_stateless/01523_client_local_queries_file_parameter.sh b/tests/queries/0_stateless/01523_client_local_queries_file_parameter.sh index bd33700277a..bd8cbc03095 100755 --- a/tests/queries/0_stateless/01523_client_local_queries_file_parameter.sh +++ b/tests/queries/0_stateless/01523_client_local_queries_file_parameter.sh @@ -13,11 +13,9 @@ SELECT * FROM 01523_test; DROP TABLE 01523_test;" > 01523_client_local_queries_file_parameter_tmp.sql $CLICKHOUSE_CLIENT --queries-file=01523_client_local_queries_file_parameter_tmp.sql 2>&1 -$CLICKHOUSE_CLIENT --queries='t' --query='t' 2>&1 - -echo "CREATE TABLE table (a Int64, b Int64) ENGINE = File(CSV, stdin); -SELECT a, b FROM table; -DROP TABLE table;" > 01523_client_local_queries_file_parameter_tmp.sql +echo "CREATE TABLE 01523_test (a Int64, b Int64) ENGINE = File(CSV, stdin); +SELECT a, b FROM 01523_test; +DROP TABLE 01523_test;" > 01523_client_local_queries_file_parameter_tmp.sql echo -e "1,2\n3,4" | $CLICKHOUSE_LOCAL --queries-file=01523_client_local_queries_file_parameter_tmp.sql 2>&1