Merge pull request #15930 from kitaisreal/clickhouse-client-and-local-added-queries-file-parameter

Clickhouse client and clickhouse local added queries-file parameter

docs/en/interfaces/cli.md

@@ -113,7 +113,8 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va
 -   `--port` – The port to connect to. Default value: 9000. Note that the HTTP interface and the native interface use different ports.
 -   `--user, -u` – The username. Default value: default.
 -   `--password` – The password. Default value: empty string.
--   `--query, -q` – The query to process when using non-interactive mode.
+-   `--query, -q` – The query to process when using non-interactive mode. You must specify either `query` or `queries-file` option.
+-   `--queries-file, -qf` - file path with queries to execute. You must specify either `query` or `queries-file` option.
 -   `--database, -d` – Select the current default database. Default value: the current database from the server settings (‘default’ by default).
 -   `--multiline, -m` – If specified, allow multiline queries (do not send the query on Enter).
 -   `--multiquery, -n` – If specified, allow processing multiple queries separated by semicolons.

docs/en/operations/utilities/clickhouse-local.md

@@ -32,7 +32,8 @@ Arguments:
 -   `-S`, `--structure` — table structure for input data.
 -   `-if`, `--input-format` — input format, `TSV` by default.
 -   `-f`, `--file` — path to data, `stdin` by default.
--   `-q` `--query` — queries to execute with `;` as delimeter.
+-   `-q` `--query` — queries to execute with `;` as delimeter. You must specify either `query` or `queries-file` option.
+-   `-qf` `--queries-file` - file path with queries to execute. You must specify either `query` or `queries-file` option.
 -   `-N`, `--table` — table name where to put output data, `table` by default.
 -   `-of`, `--format`, `--output-format` — output format, `TSV` by default.
 -   `--stacktrace` — whether to dump debug output in case of exception.

programs/client/Client.cpp

@@ -45,6 +45,7 @@
 #include <Core/Types.h>
 #include <Core/QueryProcessingStage.h>
 #include <Core/ExternalTable.h>
+#include <IO/ReadBufferFromFile.h>
 #include <IO/ReadBufferFromFileDescriptor.h>
 #include <IO/WriteBufferFromFileDescriptor.h>
 #include <IO/WriteBufferFromFile.h>
@@ -475,9 +476,16 @@ private:
         ///   The value of the option is used as the text of query (or of multiple queries).
         ///   If stdin is not a terminal, INSERT data for the first query is read from it.
         /// - stdin is not a terminal. In this case queries are read from it.
-        if (!stdin_is_a_tty || config().has("query"))
+        /// - -qf (--queries-file) command line option is present.
+        ///   The value of the option is used as file with query (or of multiple queries) to execute.
+        if (!stdin_is_a_tty || config().has("query") || config().has("queries-file"))
             is_interactive = false;
 
+        if (config().has("query") && config().has("queries-file"))
+        {
+            throw Exception("Specify either `query` or `queries-file` option", ErrorCodes::BAD_ARGUMENTS);
+        }
+
         std::cout << std::fixed << std::setprecision(3);
         std::cerr << std::fixed << std::setprecision(3);
 
@@ -786,8 +794,15 @@ private:
     {
         String text;
 
-        if (config().has("query"))
-            text = config().getRawString("query");  /// Poco configuration should not process substitutions in form of ${...} inside query.
+        if (config().has("queries-file"))
+        {
+            ReadBufferFromFile in(config().getString("queries-file"));
+            readStringUntilEOF(text, in);
+            processMultiQuery(text);
+            return;
+        }
+        else if (config().has("query"))
+            text = config().getRawString("query"); /// Poco configuration should not process substitutions in form of ${...} inside query.
         else
         {
             /// If 'query' parameter is not set, read a query from stdin.
@@ -2320,6 +2335,7 @@ public:
                 "Suggestion limit for how many databases, tables and columns to fetch.")
             ("multiline,m", "multiline")
             ("multiquery,n", "multiquery")
+            ("queries-file,qf", po::value<std::string>(), "file path with queries to execute")
             ("format,f", po::value<std::string>(), "default output format")
             ("testmode,T", "enable test hints in comments")
             ("ignore-error", "do not stop processing in multiquery mode")
@@ -2448,6 +2464,8 @@ public:
             config().setString("query_id", options["query_id"].as<std::string>());
         if (options.count("query"))
             config().setString("query", options["query"].as<std::string>());
+        if (options.count("queries-file"))
+            config().setString("queries-file", options["queries-file"].as<std::string>());
         if (options.count("database"))
             config().setString("database", options["database"].as<std::string>());
         if (options.count("pager"))

programs/local/LocalServer.cpp

@@ -20,9 +20,11 @@
 #include <Common/ThreadStatus.h>
 #include <Common/config_version.h>
 #include <Common/quoteString.h>
+#include <IO/ReadBufferFromFile.h>
 #include <IO/ReadBufferFromString.h>
 #include <IO/WriteBufferFromFileDescriptor.h>
 #include <IO/UseSSL.h>
+#include <IO/ReadHelpers.h>
 #include <Parsers/parseQuery.h>
 #include <Parsers/IAST.h>
 #include <common/ErrorHandlers.h>
@@ -195,7 +197,7 @@ try
     ThreadStatus thread_status;
     UseSSL use_ssl;
 
-    if (!config().has("query") && !config().has("table-structure")) /// Nothing to process
+    if (!config().has("query") && !config().has("table-structure") && !config().has("queries-file")) /// Nothing to process
     {
         if (config().hasOption("verbose"))
             std::cerr << "There are no queries to process." << '\n';
@@ -203,6 +205,11 @@ try
         return Application::EXIT_OK;
     }
 
+    if (config().has("query") && config().has("queries-file"))
+    {
+        throw Exception("Specify either `query` or `queries-file` option", ErrorCodes::BAD_ARGUMENTS);
+    }
+
     shared_context = Context::createShared();
     global_context = std::make_unique<Context>(Context::createGlobal(shared_context.get()));
     global_context->makeGlobalContext();
@@ -340,7 +347,17 @@ std::string LocalServer::getInitialCreateTableQuery()
 void LocalServer::processQueries()
 {
     String initial_create_query = getInitialCreateTableQuery();
-    String queries_str = initial_create_query + config().getRawString("query");
+    String queries_str = initial_create_query;
+
+    if (config().has("query"))
+        queries_str += config().getRawString("query");
+    else
+    {
+        String queries_from_file;
+        ReadBufferFromFile in(config().getString("queries-file"));
+        readStringUntilEOF(queries_from_file, in);
+        queries_str += queries_from_file;
+    }
 
     const auto & settings = global_context->getSettingsRef();
 
@@ -505,6 +522,7 @@ void LocalServer::init(int argc, char ** argv)
         ("help", "produce help message")
         ("config-file,c", po::value<std::string>(), "config-file path")
         ("query,q", po::value<std::string>(), "query")
+        ("queries-file, qf", po::value<std::string>(), "file path with queries to execute")
         ("database,d", po::value<std::string>(), "database")
 
         ("table,N", po::value<std::string>(), "name of the initial table")
@@ -552,6 +570,8 @@ void LocalServer::init(int argc, char ** argv)
         config().setString("config-file", options["config-file"].as<std::string>());
     if (options.count("query"))
         config().setString("query", options["query"].as<std::string>());
+    if (options.count("queries-file"))
+        config().setString("queries-file", options["queries-file"].as<std::string>());
     if (options.count("database"))
         config().setString("default_database", options["database"].as<std::string>());
 

tests/queries/0_stateless/01523_client_local_queries_file_parameter.reference

@@ -0,0 +1,6 @@
+1
+1
+2
+3
+1	2
+3	4

tests/queries/0_stateless/01523_client_local_queries_file_parameter.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+. "$CURDIR"/../shell_config.sh
+
+echo "SELECT 1;" > 01523_client_local_queries_file_parameter_tmp.sql
+$CLICKHOUSE_CLIENT --queries-file=01523_client_local_queries_file_parameter_tmp.sql 2>&1
+
+echo "CREATE TABLE 01523_test(value Int32) ENGINE=Log;
+INSERT INTO 01523_test 
+    VALUES (1), (2), (3);
+SELECT * FROM 01523_test;
+DROP TABLE 01523_test;" > 01523_client_local_queries_file_parameter_tmp.sql
+$CLICKHOUSE_CLIENT --queries-file=01523_client_local_queries_file_parameter_tmp.sql 2>&1
+
+echo "CREATE TABLE 01523_test (a Int64, b Int64) ENGINE = File(CSV, stdin);
+SELECT a, b FROM 01523_test;
+DROP TABLE 01523_test;" > 01523_client_local_queries_file_parameter_tmp.sql
+
+echo -e "1,2\n3,4" | $CLICKHOUSE_LOCAL --queries-file=01523_client_local_queries_file_parameter_tmp.sql 2>&1
+
+rm 01523_client_local_queries_file_parameter_tmp.sql