2018-08-20 03:34:10 +00:00
# include "TestHint.h"
2018-08-25 13:55:18 +00:00
# include "ConnectionParameters.h"
2020-06-26 05:44:42 +00:00
# include "QueryFuzzer.h"
2020-01-01 19:22:57 +00:00
# include "Suggest.h"
2018-08-20 03:34:10 +00:00
2020-01-23 08:18:19 +00:00
# if USE_REPLXX
2020-06-04 22:34:49 +00:00
# include <common / ReplxxLineReader.h>
2020-03-21 19:49:26 +00:00
# elif defined(USE_READLINE) && USE_READLINE
2020-06-04 22:34:49 +00:00
# include <common / ReadlineLineReader.h>
2020-01-23 08:18:19 +00:00
# else
2020-06-04 22:34:49 +00:00
# include <common / LineReader.h>
2020-01-23 08:18:19 +00:00
# endif
2012-03-25 03:47:13 +00:00
# include <stdlib.h>
2012-03-26 02:48:08 +00:00
# include <fcntl.h>
2012-05-09 08:16:09 +00:00
# include <signal.h>
2018-02-26 13:24:06 +00:00
# include <map>
2012-03-25 03:47:13 +00:00
# include <iostream>
2012-03-26 02:48:08 +00:00
# include <fstream>
# include <iomanip>
2014-01-08 16:33:28 +00:00
# include <unordered_set>
2014-12-05 13:10:55 +00:00
# include <algorithm>
2017-11-20 04:15:43 +00:00
# include <optional>
2019-02-08 18:15:54 +00:00
# include <ext/scope_guard.h>
2014-03-05 13:48:45 +00:00
# include <boost/program_options.hpp>
2018-02-26 13:24:06 +00:00
# include <boost/algorithm/string/replace.hpp>
2018-08-25 13:15:30 +00:00
# include <Poco/String.h>
2012-03-25 03:47:13 +00:00
# include <Poco/File.h>
# include <Poco/Util/Application.h>
2018-11-25 00:08:50 +00:00
# include <common/find_symbols.h>
2020-06-04 22:34:49 +00:00
# include <common/LineReader.h>
2017-04-01 09:19:00 +00:00
# include <Common/ClickHouseRevision.h>
# include <Common/Stopwatch.h>
# include <Common/Exception.h>
# include <Common/ShellCommand.h>
2017-07-15 03:48:36 +00:00
# include <Common/UnicodeBar.h>
# include <Common/formatReadable.h>
# include <Common/NetException.h>
2017-12-08 15:02:17 +00:00
# include <Common/Throttler.h>
2018-01-21 21:59:03 +00:00
# include <Common/StringUtils/StringUtils.h>
2017-07-15 03:48:36 +00:00
# include <Common/typeid_cast.h>
2020-06-14 19:23:05 +00:00
# include <Common/clearPasswordFromCommandLine.h>
2018-03-14 19:38:48 +00:00
# include <Common/Config/ConfigProcessor.h>
2020-06-30 09:25:23 +00:00
# include <Common/PODArray.h>
2017-04-01 09:19:00 +00:00
# include <Core/Types.h>
# include <Core/QueryProcessingStage.h>
2018-08-20 02:34:00 +00:00
# include <Core/ExternalTable.h>
2017-04-01 09:19:00 +00:00
# include <IO/ReadBufferFromFileDescriptor.h>
# include <IO/WriteBufferFromFileDescriptor.h>
# include <IO/WriteBufferFromFile.h>
# include <IO/ReadBufferFromMemory.h>
2018-03-23 16:55:35 +00:00
# include <IO/ReadBufferFromString.h>
2017-04-01 09:19:00 +00:00
# include <IO/ReadHelpers.h>
# include <IO/WriteHelpers.h>
2019-03-26 14:36:56 +00:00
# include <IO/Operators.h>
2018-09-06 18:05:33 +00:00
# include <IO/UseSSL.h>
2017-04-01 09:19:00 +00:00
# include <DataStreams/AsynchronousBlockInputStream.h>
2018-07-04 17:02:47 +00:00
# include <DataStreams/AddingDefaultsBlockInputStream.h>
2018-06-15 17:32:35 +00:00
# include <DataStreams/InternalTextLogsRowOutputStream.h>
2017-04-01 09:19:00 +00:00
# include <Parsers/ParserQuery.h>
# include <Parsers/ASTSetQuery.h>
# include <Parsers/ASTUseQuery.h>
# include <Parsers/ASTInsertQuery.h>
2018-02-25 06:34:20 +00:00
# include <Parsers/ASTSelectWithUnionQuery.h>
2017-04-01 09:19:00 +00:00
# include <Parsers/ASTQueryWithOutput.h>
# include <Parsers/ASTLiteral.h>
# include <Parsers/ASTIdentifier.h>
# include <Parsers/formatAST.h>
# include <Parsers/parseQuery.h>
# include <Interpreters/Context.h>
2019-01-23 19:23:37 +00:00
# include <Interpreters/InterpreterSetQuery.h>
2019-05-18 21:07:23 +00:00
# include <Interpreters/ReplaceQueryParameterVisitor.h>
2017-04-01 09:19:00 +00:00
# include <Client/Connection.h>
2018-06-05 20:09:51 +00:00
# include <Common/InterruptListener.h>
2017-07-13 00:04:06 +00:00
# include <Functions/registerFunctions.h>
# include <AggregateFunctions/registerAggregateFunctions.h>
2018-11-28 14:41:51 +00:00
# include <Common/Config/configReadClient.h>
2018-12-04 20:03:04 +00:00
# include <Storages/ColumnsDescription.h>
2019-06-14 14:00:37 +00:00
# include <common/argsToConfig.h>
2019-08-23 15:47:27 +00:00
# include <Common/TerminalSize.h>
2020-06-04 22:45:04 +00:00
# include <Common/UTF8Helpers.h>
2018-08-25 13:55:18 +00:00
2020-05-27 21:58:46 +00:00
# if !defined(ARCADIA_BUILD)
# include <Common / config_version.h>
# endif
2018-09-01 21:03:06 +00:00
# ifndef __clang__
# pragma GCC optimize("-fno-var-tracking-assignments")
# endif
2016-05-17 00:59:24 +00:00
2014-08-15 00:08:15 +00:00
/// http://en.wikipedia.org/wiki/ANSI_escape_code
# define CLEAR_TO_END_OF_LINE "\033[K"
2017-04-07 17:56:03 +00:00
2014-08-15 00:08:15 +00:00
2012-03-25 03:47:13 +00:00
namespace DB
{
2016-01-12 02:21:15 +00:00
namespace ErrorCodes
{
2017-04-01 07:20:54 +00:00
extern const int NETWORK_ERROR ;
extern const int NO_DATA_TO_INSERT ;
extern const int BAD_ARGUMENTS ;
extern const int UNKNOWN_PACKET_FROM_SERVER ;
extern const int UNEXPECTED_PACKET_FROM_SERVER ;
extern const int CLIENT_OUTPUT_FORMAT_SPECIFIED ;
2019-09-05 13:17:01 +00:00
extern const int INVALID_USAGE_OF_INPUT ;
2020-03-05 13:12:00 +00:00
extern const int DEADLOCK_AVOIDED ;
2020-06-24 16:34:49 +00:00
extern const int UNRECOGNIZED_ARGUMENTS ;
2016-01-12 02:21:15 +00:00
}
2012-03-25 03:47:13 +00:00
class Client : public Poco : : Util : : Application
{
public :
2020-03-09 03:41:03 +00:00
Client ( ) = default ;
2014-01-08 16:33:28 +00:00
2012-03-25 03:47:13 +00:00
private :
2017-04-01 07:20:54 +00:00
using StringSet = std : : unordered_set < String > ;
2018-01-10 00:04:08 +00:00
StringSet exit_strings
{
2017-04-01 07:20:54 +00:00
" exit " , " quit " , " logout " ,
" учше " , " йгше " , " дщпщге " ,
" exit; " , " quit; " , " logout; " ,
" учшеж " , " йгшеж " , " дщпщгеж " ,
" q " , " й " , " \\ q " , " \\ Q " , " \\ й " , " \\ Й " , " :q " , " Жй "
} ;
2020-01-18 15:44:08 +00:00
bool is_interactive = true ; /// Use either interactive line editing interface or batch mode.
2017-04-01 07:20:54 +00:00
bool need_render_progress = true ; /// Render query execution progress.
2020-06-22 19:16:48 +00:00
bool has_received_logs = false ; /// We have received some logs, do not use previous cursor position, to avoid overlaps with logs
2017-07-12 00:49:30 +00:00
bool echo_queries = false ; /// Print queries before execution in batch mode.
2018-08-08 02:11:16 +00:00
bool ignore_error = false ; /// In case of errors, don't print error message, continue to next query. Only applicable for non-interactive mode.
2017-07-12 00:49:30 +00:00
bool print_time_to_stderr = false ; /// Output execution time to stderr in batch mode.
2020-02-21 20:01:38 +00:00
bool stdin_is_a_tty = false ; /// stdin is a terminal.
bool stdout_is_a_tty = false ; /// stdout is a terminal.
2012-05-08 05:42:05 +00:00
2017-04-01 07:20:54 +00:00
std : : unique_ptr < Connection > connection ; /// Connection to DB.
2018-03-24 23:50:30 +00:00
String query_id ; /// Current query_id.
2020-06-26 04:37:18 +00:00
String full_query ; /// Current query as it was given to the client.
// Current query as it will be sent to the server. It may differ from the
// full query for INSERT queries, for which the data that follows the query
// is stripped and sent separately.
String query_to_send ;
2012-03-25 03:47:13 +00:00
2017-07-12 00:49:30 +00:00
String format ; /// Query results output format.
bool is_default_format = true ; /// false, if format is set in the config or command line.
2017-04-01 07:20:54 +00:00
size_t format_max_block_size = 0 ; /// Max block size for console output.
String insert_format ; /// Format of INSERT data that is read from stdin in batch mode.
size_t insert_format_max_block_size = 0 ; /// Max block size when reading INSERT data.
2017-12-13 20:53:53 +00:00
size_t max_client_network_bandwidth = 0 ; /// The maximum speed of data exchange over the network for the client in bytes per second.
2012-03-25 03:47:13 +00:00
2017-04-01 07:20:54 +00:00
bool has_vertical_output_suffix = false ; /// Is \G present at the end of the query string?
2014-12-19 23:02:18 +00:00
2020-04-17 09:47:40 +00:00
SharedContextHolder shared_context = Context : : createShared ( ) ;
Context context = Context : : createGlobal ( shared_context . get ( ) ) ;
2012-03-25 03:47:13 +00:00
2017-04-01 07:20:54 +00:00
/// Buffer that reads from stdin in batch mode.
ReadBufferFromFileDescriptor std_in { STDIN_FILENO } ;
2012-05-08 11:19:00 +00:00
2017-04-01 07:20:54 +00:00
/// Console output.
WriteBufferFromFileDescriptor std_out { STDOUT_FILENO } ;
std : : unique_ptr < ShellCommand > pager_cmd ;
/// The user can specify to redirect query output to a file.
2017-11-20 04:15:43 +00:00
std : : optional < WriteBufferFromFile > out_file_buf ;
2017-04-01 07:20:54 +00:00
BlockOutputStreamPtr block_out_stream ;
2017-03-13 17:35:17 +00:00
2018-06-07 18:46:53 +00:00
/// The user could specify special file for server logs (stderr by default)
std : : unique_ptr < WriteBuffer > out_logs_buf ;
String server_logs_file ;
2018-06-06 20:57:07 +00:00
BlockOutputStreamPtr logs_out_stream ;
2017-03-13 17:35:17 +00:00
2017-04-01 07:20:54 +00:00
String home_path ;
2017-03-13 17:35:17 +00:00
2017-04-01 07:20:54 +00:00
String current_profile ;
2017-03-13 17:35:17 +00:00
2018-02-26 13:48:07 +00:00
String prompt_by_server_display_name ;
2018-02-26 13:24:06 +00:00
2017-04-01 07:20:54 +00:00
/// Path to a file containing command history.
String history_file ;
2017-03-13 17:35:17 +00:00
2017-04-01 07:20:54 +00:00
/// How many rows have been read or written.
size_t processed_rows = 0 ;
2017-03-13 17:35:17 +00:00
2017-04-01 07:20:54 +00:00
/// Parsed query. Is used to determine some settings (e.g. format, output file).
ASTPtr parsed_query ;
2017-03-13 17:35:17 +00:00
2017-04-01 07:20:54 +00:00
/// The last exception that was received from the server. Is used for the return code in batch mode.
2020-06-26 01:34:06 +00:00
std : : unique_ptr < Exception > last_exception_received_from_server ;
2017-03-13 17:35:17 +00:00
2017-04-01 07:20:54 +00:00
/// If the last query resulted in exception.
2020-06-26 01:34:06 +00:00
bool received_exception_from_server = false ;
2018-07-27 17:19:22 +00:00
int expected_server_error = 0 ;
int expected_client_error = 0 ;
int actual_server_error = 0 ;
int actual_client_error = 0 ;
2018-08-25 13:41:50 +00:00
UInt64 server_revision = 0 ;
2017-11-02 14:45:23 +00:00
String server_version ;
2018-02-26 06:49:17 +00:00
String server_display_name ;
2017-03-13 17:35:17 +00:00
2017-04-01 07:20:54 +00:00
Stopwatch watch ;
2017-03-13 17:35:17 +00:00
2017-04-01 07:20:54 +00:00
/// The server periodically sends information about how much data was read since last time.
Progress progress ;
bool show_progress_bar = false ;
2017-03-13 17:35:17 +00:00
2017-04-01 07:20:54 +00:00
size_t written_progress_chars = 0 ;
bool written_first_block = false ;
2017-03-13 17:35:17 +00:00
2017-04-01 07:20:54 +00:00
/// External tables info.
std : : list < ExternalTable > external_tables ;
2017-03-13 17:35:17 +00:00
2019-05-18 21:07:23 +00:00
/// Dictionary with query parameters for prepared statements.
2019-06-15 17:52:53 +00:00
NameToNameMap query_parameters ;
2019-05-18 21:07:23 +00:00
2018-03-23 18:20:54 +00:00
ConnectionParameters connection_parameters ;
2020-06-26 05:44:42 +00:00
QueryFuzzer fuzzer ;
int query_fuzzer_runs ;
2020-01-21 08:54:26 +00:00
void initialize ( Poco : : Util : : Application & self ) override
2017-04-01 07:20:54 +00:00
{
Poco : : Util : : Application : : initialize ( self ) ;
2017-03-13 17:35:17 +00:00
2017-04-01 07:20:54 +00:00
const char * home_path_cstr = getenv ( " HOME " ) ;
if ( home_path_cstr )
home_path = home_path_cstr ;
2017-03-13 17:35:17 +00:00
2018-11-28 14:41:51 +00:00
configReadClient ( config ( ) , home_path ) ;
2017-03-13 17:35:17 +00:00
2017-04-01 07:20:54 +00:00
context . setApplicationType ( Context : : ApplicationType : : CLIENT ) ;
2019-10-02 19:54:40 +00:00
context . setQueryParameters ( query_parameters ) ;
2017-03-13 17:35:17 +00:00
2017-04-01 07:20:54 +00:00
/// settings and limits could be specified in config file, but passed settings has higher priority
2020-03-13 14:50:26 +00:00
for ( const auto & setting : context . getSettingsRef ( ) )
2019-04-25 14:08:20 +00:00
{
const String & name = setting . getName ( ) . toString ( ) ;
if ( config ( ) . has ( name ) & & ! setting . isChanged ( ) )
2020-03-13 14:50:26 +00:00
context . setSetting ( name , config ( ) . getString ( name ) ) ;
2019-04-25 14:08:20 +00:00
}
2017-03-13 17:35:17 +00:00
2019-01-23 19:31:08 +00:00
/// Set path for format schema files
if ( config ( ) . has ( " format_schema_path " ) )
2019-01-27 09:15:32 +00:00
context . setFormatSchemaPath ( Poco : : Path ( config ( ) . getString ( " format_schema_path " ) ) . toString ( ) ) ;
2017-04-01 07:20:54 +00:00
}
2020-01-21 08:54:26 +00:00
int main ( const std : : vector < std : : string > & /*args*/ ) override
2017-04-01 07:20:54 +00:00
{
try
{
2017-12-02 02:47:12 +00:00
return mainImpl ( ) ;
2017-04-01 07:20:54 +00:00
}
catch ( const Exception & e )
{
bool print_stack_trace = config ( ) . getBool ( " stacktrace " , false ) ;
std : : string text = e . displayText ( ) ;
/** If exception is received from server, then stack trace is embedded in message.
* If exception is thrown on client , then stack trace is in separate field .
*/
auto embedded_stack_trace_pos = text . find ( " Stack trace " ) ;
if ( std : : string : : npos ! = embedded_stack_trace_pos & & ! print_stack_trace )
text . resize ( embedded_stack_trace_pos ) ;
std : : cerr < < " Code: " < < e . code ( ) < < " . " < < text < < std : : endl < < std : : endl ;
/// Don't print the stack trace on the client if it was logged on the server.
/// Also don't print the stack trace in case of network errors.
if ( print_stack_trace
& & e . code ( ) ! = ErrorCodes : : NETWORK_ERROR
& & std : : string : : npos = = embedded_stack_trace_pos )
{
std : : cerr < < " Stack trace: " < < std : : endl
2020-01-02 06:56:53 +00:00
< < e . getStackTraceString ( ) ;
2017-04-01 07:20:54 +00:00
}
/// If exception code isn't zero, we should return non-zero return code anyway.
return e . code ( ) ? e . code ( ) : - 1 ;
}
catch ( . . . )
{
2018-08-19 20:26:11 +00:00
std : : cerr < < getCurrentExceptionMessage ( false ) < < std : : endl ;
return getCurrentExceptionCode ( ) ;
2017-04-01 07:20:54 +00:00
}
}
/// Should we celebrate a bit?
2020-03-18 00:57:00 +00:00
static bool isNewYearMode ( )
2017-04-01 07:20:54 +00:00
{
2017-08-04 14:00:26 +00:00
time_t current_time = time ( nullptr ) ;
2017-04-01 07:20:54 +00:00
/// It's bad to be intrusive.
if ( current_time % 3 ! = 0 )
return false ;
LocalDate now ( current_time ) ;
return ( now . month ( ) = = 12 & & now . day ( ) > = 20 )
| | ( now . month ( ) = = 1 & & now . day ( ) < = 5 ) ;
}
2020-03-18 00:57:00 +00:00
static bool isChineseNewYearMode ( const String & local_tz )
2020-01-02 08:49:20 +00:00
{
/// Days of Dec. 20 in Chinese calendar starting from year 2019 to year 2105
static constexpr UInt16 chineseNewYearIndicators [ ]
= { 18275 , 18659 , 19014 , 19368 , 19752 , 20107 , 20491 , 20845 , 21199 , 21583 , 21937 , 22292 , 22676 , 23030 , 23414 , 23768 , 24122 , 24506 ,
24860 , 25215 , 25599 , 25954 , 26308 , 26692 , 27046 , 27430 , 27784 , 28138 , 28522 , 28877 , 29232 , 29616 , 29970 , 30354 , 30708 , 31062 ,
31446 , 31800 , 32155 , 32539 , 32894 , 33248 , 33632 , 33986 , 34369 , 34724 , 35078 , 35462 , 35817 , 36171 , 36555 , 36909 , 37293 , 37647 ,
38002 , 38386 , 38740 , 39095 , 39479 , 39833 , 40187 , 40571 , 40925 , 41309 , 41664 , 42018 , 42402 , 42757 , 43111 , 43495 , 43849 , 44233 ,
44587 , 44942 , 45326 , 45680 , 46035 , 46418 , 46772 , 47126 , 47510 , 47865 , 48249 , 48604 , 48958 , 49342 } ;
/// All time zone names are acquired from https://www.iana.org/time-zones
static constexpr const char * chineseNewYearTimeZoneIndicators [ ] = {
/// Time zones celebrating Chinese new year.
" Asia/Shanghai " ,
" Asia/Chongqing " ,
" Asia/Harbin " ,
" Asia/Urumqi " ,
" Asia/Hong_Kong " ,
" Asia/Chungking " ,
" Asia/Macao " ,
" Asia/Macau " ,
" Asia/Taipei " ,
" Asia/Singapore " ,
/// Time zones celebrating Chinese new year but with different festival names. Let's not print the message for now.
// "Asia/Brunei",
// "Asia/Ho_Chi_Minh",
// "Asia/Hovd",
// "Asia/Jakarta",
// "Asia/Jayapura",
// "Asia/Kashgar",
// "Asia/Kuala_Lumpur",
// "Asia/Kuching",
// "Asia/Makassar",
// "Asia/Pontianak",
// "Asia/Pyongyang",
// "Asia/Saigon",
// "Asia/Seoul",
// "Asia/Ujung_Pandang",
// "Asia/Ulaanbaatar",
// "Asia/Ulan_Bator",
} ;
static constexpr size_t M = sizeof ( chineseNewYearTimeZoneIndicators ) / sizeof ( chineseNewYearTimeZoneIndicators [ 0 ] ) ;
time_t current_time = time ( nullptr ) ;
if ( chineseNewYearTimeZoneIndicators + M
= = std : : find_if ( chineseNewYearTimeZoneIndicators , chineseNewYearTimeZoneIndicators + M , [ & local_tz ] ( const char * tz )
{
return tz = = local_tz ;
} ) )
return false ;
/// It's bad to be intrusive.
if ( current_time % 3 ! = 0 )
return false ;
auto days = DateLUT : : instance ( ) . toDayNum ( current_time ) . toUnderType ( ) ;
2020-03-09 03:41:03 +00:00
for ( auto d : chineseNewYearIndicators )
2020-01-02 08:49:20 +00:00
{
/// Let's celebrate until Lantern Festival
if ( d < = days & & d + 25u > = days )
return true ;
else if ( d > days )
return false ;
}
return false ;
}
2020-06-04 22:45:04 +00:00
# if USE_REPLXX
static void highlight ( const String & query , std : : vector < replxx : : Replxx : : Color > & colors )
{
using namespace replxx ;
static const std : : unordered_map < TokenType , Replxx : : Color > token_to_color =
{
{ TokenType : : Whitespace , Replxx : : Color : : DEFAULT } ,
{ TokenType : : Comment , Replxx : : Color : : GRAY } ,
2020-06-04 23:05:37 +00:00
{ TokenType : : BareWord , Replxx : : Color : : DEFAULT } ,
{ TokenType : : Number , Replxx : : Color : : GREEN } ,
{ TokenType : : StringLiteral , Replxx : : Color : : CYAN } ,
{ TokenType : : QuotedIdentifier , Replxx : : Color : : MAGENTA } ,
{ TokenType : : OpeningRoundBracket , Replxx : : Color : : BROWN } ,
{ TokenType : : ClosingRoundBracket , Replxx : : Color : : BROWN } ,
2020-06-04 22:45:04 +00:00
{ TokenType : : OpeningSquareBracket , Replxx : : Color : : BROWN } ,
{ TokenType : : ClosingSquareBracket , Replxx : : Color : : BROWN } ,
2020-06-06 21:20:17 +00:00
{ TokenType : : OpeningCurlyBrace , Replxx : : Color : : INTENSE } ,
{ TokenType : : ClosingCurlyBrace , Replxx : : Color : : INTENSE } ,
{ TokenType : : Comma , Replxx : : Color : : INTENSE } ,
{ TokenType : : Semicolon , Replxx : : Color : : INTENSE } ,
{ TokenType : : Dot , Replxx : : Color : : INTENSE } ,
{ TokenType : : Asterisk , Replxx : : Color : : INTENSE } ,
{ TokenType : : Plus , Replxx : : Color : : INTENSE } ,
{ TokenType : : Minus , Replxx : : Color : : INTENSE } ,
{ TokenType : : Slash , Replxx : : Color : : INTENSE } ,
{ TokenType : : Percent , Replxx : : Color : : INTENSE } ,
{ TokenType : : Arrow , Replxx : : Color : : INTENSE } ,
{ TokenType : : QuestionMark , Replxx : : Color : : INTENSE } ,
{ TokenType : : Colon , Replxx : : Color : : INTENSE } ,
{ TokenType : : Equals , Replxx : : Color : : INTENSE } ,
{ TokenType : : NotEquals , Replxx : : Color : : INTENSE } ,
{ TokenType : : Less , Replxx : : Color : : INTENSE } ,
{ TokenType : : Greater , Replxx : : Color : : INTENSE } ,
{ TokenType : : LessOrEquals , Replxx : : Color : : INTENSE } ,
{ TokenType : : GreaterOrEquals , Replxx : : Color : : INTENSE } ,
{ TokenType : : Concatenation , Replxx : : Color : : INTENSE } ,
{ TokenType : : At , Replxx : : Color : : INTENSE } ,
2020-06-20 16:05:49 +00:00
{ TokenType : : DoubleAt , Replxx : : Color : : MAGENTA } ,
2020-06-04 22:45:04 +00:00
{ TokenType : : EndOfStream , Replxx : : Color : : DEFAULT } ,
{ TokenType : : Error , Replxx : : Color : : RED } ,
{ TokenType : : ErrorMultilineCommentIsNotClosed , Replxx : : Color : : RED } ,
{ TokenType : : ErrorSingleQuoteIsNotClosed , Replxx : : Color : : RED } ,
{ TokenType : : ErrorDoubleQuoteIsNotClosed , Replxx : : Color : : RED } ,
{ TokenType : : ErrorSinglePipeMark , Replxx : : Color : : RED } ,
{ TokenType : : ErrorWrongNumber , Replxx : : Color : : RED } ,
{ TokenType : : ErrorMaxQuerySizeExceeded , Replxx : : Color : : RED }
} ;
const Replxx : : Color unknown_token_color = Replxx : : Color : : RED ;
Lexer lexer ( query . data ( ) , query . data ( ) + query . size ( ) ) ;
size_t pos = 0 ;
for ( Token token = lexer . nextToken ( ) ; ! token . isEnd ( ) ; token = lexer . nextToken ( ) )
{
size_t utf8_len = UTF8 : : countCodePoints ( reinterpret_cast < const UInt8 * > ( token . begin ) , token . size ( ) ) ;
for ( size_t code_point_index = 0 ; code_point_index < utf8_len ; + + code_point_index )
{
if ( token_to_color . find ( token . type ) ! = token_to_color . end ( ) )
colors [ pos + code_point_index ] = token_to_color . at ( token . type ) ;
else
colors [ pos + code_point_index ] = unknown_token_color ;
}
pos + = utf8_len ;
}
}
# endif
2017-12-02 02:47:12 +00:00
int mainImpl ( )
2017-04-01 07:20:54 +00:00
{
2018-09-06 18:05:33 +00:00
UseSSL use_ssl ;
2017-07-13 00:04:06 +00:00
registerFunctions ( ) ;
registerAggregateFunctions ( ) ;
2017-04-01 07:20:54 +00:00
/// Batch mode is enabled if one of the following is true:
/// - -e (--query) command line option is present.
/// The value of the option is used as the text of query (or of multiple queries).
/// If stdin is not a terminal, INSERT data for the first query is read from it.
/// - stdin is not a terminal. In this case queries are read from it.
2020-02-21 20:01:38 +00:00
if ( ! stdin_is_a_tty | | config ( ) . has ( " query " ) )
2017-04-01 07:20:54 +00:00
is_interactive = false ;
std : : cout < < std : : fixed < < std : : setprecision ( 3 ) ;
std : : cerr < < std : : fixed < < std : : setprecision ( 3 ) ;
if ( is_interactive )
showClientVersion ( ) ;
is_default_format = ! config ( ) . has ( " vertical " ) & & ! config ( ) . has ( " format " ) ;
if ( config ( ) . has ( " vertical " ) )
format = config ( ) . getString ( " format " , " Vertical " ) ;
else
format = config ( ) . getString ( " format " , is_interactive ? " PrettyCompact " : " TabSeparated " ) ;
format_max_block_size = config ( ) . getInt ( " format_max_block_size " , context . getSettingsRef ( ) . max_block_size ) ;
insert_format = " Values " ;
2018-06-13 19:01:07 +00:00
/// Setting value from cmd arg overrides one from config
if ( context . getSettingsRef ( ) . max_insert_block_size . changed )
insert_format_max_block_size = context . getSettingsRef ( ) . max_insert_block_size ;
else
insert_format_max_block_size = config ( ) . getInt ( " insert_format_max_block_size " , context . getSettingsRef ( ) . max_insert_block_size ) ;
2017-04-01 07:20:54 +00:00
if ( ! is_interactive )
{
need_render_progress = config ( ) . getBool ( " progress " , false ) ;
echo_queries = config ( ) . getBool ( " echo " , false ) ;
2018-08-08 02:11:16 +00:00
ignore_error = config ( ) . getBool ( " ignore-error " , false ) ;
2017-04-01 07:20:54 +00:00
}
2020-04-15 01:58:10 +00:00
ClientInfo & client_info = context . getClientInfo ( ) ;
client_info . setInitialQuery ( ) ;
client_info . quota_key = config ( ) . getString ( " quota_key " , " " ) ;
2017-04-01 07:20:54 +00:00
connect ( ) ;
/// Initialize DateLUT here to avoid counting time spent here as query execution time.
2020-01-02 08:49:20 +00:00
const auto local_tz = DateLUT : : instance ( ) . getTimeZone ( ) ;
2017-04-01 07:20:54 +00:00
if ( ! context . getSettingsRef ( ) . use_client_time_zone )
{
2019-03-07 23:13:39 +00:00
const auto & time_zone = connection - > getServerTimezone ( connection_parameters . timeouts ) ;
2017-04-01 07:20:54 +00:00
if ( ! time_zone . empty ( ) )
{
try
{
DateLUT : : setDefaultTimezone ( time_zone ) ;
}
catch ( . . . )
{
std : : cerr < < " Warning: could not switch to server time zone: " < < time_zone
< < " , reason: " < < getCurrentExceptionMessage ( /* with_stacktrace = */ false ) < < std : : endl
< < " Proceeding with local time zone. "
< < std : : endl < < std : : endl ;
}
}
else
{
std : : cerr < < " Warning: could not determine server time zone. "
< < " Proceeding with local time zone. "
< < std : : endl < < std : : endl ;
}
}
2018-02-26 13:24:06 +00:00
Strings keys ;
2018-03-23 16:55:35 +00:00
prompt_by_server_display_name = config ( ) . getRawString ( " prompt_by_server_display_name.default " , " {display_name} :) " ) ;
2018-02-26 13:48:07 +00:00
2018-03-08 08:55:27 +00:00
config ( ) . keys ( " prompt_by_server_display_name " , keys ) ;
2018-02-26 13:24:06 +00:00
for ( const String & key : keys )
{
2018-03-23 16:55:35 +00:00
if ( key ! = " default " & & server_display_name . find ( key ) ! = std : : string : : npos )
2018-02-26 13:24:06 +00:00
{
2018-03-23 16:55:35 +00:00
prompt_by_server_display_name = config ( ) . getRawString ( " prompt_by_server_display_name. " + key ) ;
2018-02-26 13:24:06 +00:00
break ;
}
}
2018-03-23 16:55:35 +00:00
/// Prompt may contain escape sequences including \e[ or \x1b[ sequences to set terminal color.
2018-03-08 08:55:27 +00:00
{
2018-03-23 16:55:35 +00:00
String unescaped_prompt_by_server_display_name ;
ReadBufferFromString in ( prompt_by_server_display_name ) ;
readEscapedString ( unescaped_prompt_by_server_display_name , in ) ;
prompt_by_server_display_name = std : : move ( unescaped_prompt_by_server_display_name ) ;
2018-03-08 08:55:27 +00:00
}
2018-03-23 16:55:35 +00:00
/// Prompt may contain the following substitutions in a form of {name}.
2018-03-23 18:20:54 +00:00
std : : map < String , String > prompt_substitutions
2018-03-23 16:55:35 +00:00
{
2018-03-23 18:20:54 +00:00
{ " host " , connection_parameters . host } ,
{ " port " , toString ( connection_parameters . port ) } ,
{ " user " , connection_parameters . user } ,
2018-03-08 08:55:27 +00:00
{ " display_name " , server_display_name } ,
} ;
2018-03-23 16:55:35 +00:00
/// Quite suboptimal.
2018-03-23 18:20:54 +00:00
for ( const auto & [ key , value ] : prompt_substitutions )
2018-03-08 08:55:27 +00:00
boost : : replace_all ( prompt_by_server_display_name , " { " + key + " } " , value ) ;
2017-04-01 07:20:54 +00:00
if ( is_interactive )
{
2018-03-25 22:44:54 +00:00
if ( ! query_id . empty ( ) )
throw Exception ( " query_id could be specified only in non-interactive mode " , ErrorCodes : : BAD_ARGUMENTS ) ;
2017-04-01 07:20:54 +00:00
if ( print_time_to_stderr )
throw Exception ( " time option could be specified only in non-interactive mode " , ErrorCodes : : BAD_ARGUMENTS ) ;
2020-01-01 19:22:57 +00:00
if ( server_revision > = Suggest : : MIN_SERVER_REVISION & & ! config ( ) . getBool ( " disable_suggestion " , false ) )
2020-02-25 08:30:11 +00:00
{
2020-01-01 19:22:57 +00:00
/// Load suggestion data from the server.
2020-01-19 00:23:35 +00:00
Suggest : : instance ( ) . load ( connection_parameters , config ( ) . getInt ( " suggestion_limit " ) ) ;
2020-02-25 08:30:11 +00:00
}
2020-01-01 19:22:57 +00:00
2017-04-01 07:20:54 +00:00
/// Load command history if present.
if ( config ( ) . has ( " history_file " ) )
history_file = config ( ) . getString ( " history_file " ) ;
2019-09-05 23:35:25 +00:00
else
{
2020-05-18 08:08:55 +00:00
auto * history_file_from_env = getenv ( " CLICKHOUSE_HISTORY_FILE " ) ;
2019-09-05 23:35:25 +00:00
if ( history_file_from_env )
history_file = history_file_from_env ;
else if ( ! home_path . empty ( ) )
history_file = home_path + " /.clickhouse-client-history " ;
}
2017-04-01 07:20:54 +00:00
2019-12-26 15:30:25 +00:00
if ( ! history_file . empty ( ) & & ! Poco : : File ( history_file ) . exists ( ) )
Poco : : File ( history_file ) . createFile ( ) ;
2018-08-22 03:06:13 +00:00
2020-06-02 03:25:19 +00:00
LineReader : : Patterns query_extenders = { " \\ " } ;
LineReader : : Patterns query_delimiters = { " ; " , " \\ G " } ;
2020-01-23 08:18:19 +00:00
# if USE_REPLXX
2020-06-06 16:59:16 +00:00
replxx : : Replxx : : highlighter_callback_t highlight_callback { } ;
if ( config ( ) . getBool ( " highlight " ) )
highlight_callback = highlight ;
ReplxxLineReader lr (
Suggest : : instance ( ) ,
history_file ,
config ( ) . has ( " multiline " ) ,
query_extenders ,
query_delimiters ,
highlight_callback ) ;
2020-03-21 19:49:26 +00:00
# elif defined(USE_READLINE) && USE_READLINE
2020-06-02 03:25:19 +00:00
ReadlineLineReader lr ( Suggest : : instance ( ) , history_file , config ( ) . has ( " multiline " ) , query_extenders , query_delimiters ) ;
2020-01-23 08:18:19 +00:00
# else
2020-06-02 03:25:19 +00:00
LineReader lr ( history_file , config ( ) . has ( " multiline " ) , query_extenders , query_delimiters ) ;
2020-01-23 08:18:19 +00:00
# endif
2018-08-22 03:06:13 +00:00
2020-02-25 08:30:11 +00:00
/// Enable bracketed-paste-mode only when multiquery is enabled and multiline is
/// disabled, so that we are able to paste and execute multiline queries in a whole
/// instead of erroring out, while be less intrusive.
if ( config ( ) . has ( " multiquery " ) & & ! config ( ) . has ( " multiline " ) )
lr . enableBracketedPaste ( ) ;
2019-12-26 15:30:25 +00:00
do
2019-11-14 00:41:55 +00:00
{
2019-12-26 15:30:25 +00:00
auto input = lr . readLine ( prompt ( ) , " :-] " ) ;
if ( input . empty ( ) )
break ;
2019-10-26 05:57:48 +00:00
2020-01-26 18:43:44 +00:00
has_vertical_output_suffix = false ;
2020-01-23 11:18:47 +00:00
if ( input . ends_with ( " \\ G " ) )
{
input . resize ( input . size ( ) - 2 ) ;
has_vertical_output_suffix = true ;
}
2019-12-26 15:30:25 +00:00
try
2018-08-22 03:06:13 +00:00
{
2020-06-26 01:34:06 +00:00
if ( ! processQueryText ( input ) )
2019-12-26 15:30:25 +00:00
break ;
2018-08-22 03:06:13 +00:00
}
2019-12-26 15:30:25 +00:00
catch ( const Exception & e )
2018-08-22 03:06:13 +00:00
{
2019-12-26 15:30:25 +00:00
actual_client_error = e . code ( ) ;
if ( ! actual_client_error | | actual_client_error ! = expected_client_error )
{
std : : cerr < < std : : endl
< < " Exception on client: " < < std : : endl
< < " Code: " < < e . code ( ) < < " . " < < e . displayText ( ) < < std : : endl ;
2018-08-22 03:06:13 +00:00
2019-12-26 15:30:25 +00:00
if ( config ( ) . getBool ( " stacktrace " , false ) )
2020-01-09 16:21:04 +00:00
std : : cerr < < " Stack trace: " < < std : : endl < < e . getStackTraceString ( ) < < std : : endl ;
2018-08-22 03:06:13 +00:00
2019-12-26 15:30:25 +00:00
std : : cerr < < std : : endl ;
}
/// Client-side exception during query execution can result in the loss of
/// sync in the connection protocol.
/// So we reconnect and allow to enter the next query.
connect ( ) ;
}
}
while ( true ) ;
2017-04-01 07:20:54 +00:00
2020-01-02 08:49:20 +00:00
if ( isNewYearMode ( ) )
std : : cout < < " Happy new year. " < < std : : endl ;
else if ( isChineseNewYearMode ( local_tz ) )
std : : cout < < " Happy Chinese new year. 春节快乐! " < < std : : endl ;
else
std : : cout < < " Bye. " < < std : : endl ;
2017-04-01 07:20:54 +00:00
return 0 ;
}
else
{
2018-03-24 23:50:30 +00:00
query_id = config ( ) . getString ( " query_id " , " " ) ;
2020-06-30 09:25:23 +00:00
if ( query_fuzzer_runs )
{
nonInteractiveWithFuzzing ( ) ;
}
else
{
nonInteractive ( ) ;
}
2017-04-01 07:20:54 +00:00
2018-05-21 13:49:54 +00:00
/// If exception code isn't zero, we should return non-zero return code anyway.
2020-06-26 01:34:06 +00:00
if ( last_exception_received_from_server )
return last_exception_received_from_server - > code ( ) ! = 0 ? last_exception_received_from_server - > code ( ) : - 1 ;
2017-04-01 07:20:54 +00:00
return 0 ;
}
}
void connect ( )
{
2018-07-02 12:32:40 +00:00
connection_parameters = ConnectionParameters ( config ( ) ) ;
2017-04-01 07:20:54 +00:00
if ( is_interactive )
std : : cout < < " Connecting to "
2018-03-23 18:20:54 +00:00
< < ( ! connection_parameters . default_database . empty ( ) ? " database " + connection_parameters . default_database + " at " : " " )
< < connection_parameters . host < < " : " < < connection_parameters . port
< < ( ! connection_parameters . user . empty ( ) ? " as user " + connection_parameters . user : " " )
2017-04-01 07:20:54 +00:00
< < " . " < < std : : endl ;
2018-01-10 01:43:54 +00:00
connection = std : : make_unique < Connection > (
2018-03-23 18:20:54 +00:00
connection_parameters . host ,
connection_parameters . port ,
connection_parameters . default_database ,
connection_parameters . user ,
connection_parameters . password ,
" client " ,
connection_parameters . compression ,
connection_parameters . security ) ;
2018-01-10 01:43:54 +00:00
2017-11-02 14:45:23 +00:00
String server_name ;
UInt64 server_version_major = 0 ;
UInt64 server_version_minor = 0 ;
2018-07-31 21:36:18 +00:00
UInt64 server_version_patch = 0 ;
2017-04-01 07:20:54 +00:00
2017-12-13 20:53:53 +00:00
if ( max_client_network_bandwidth )
2017-12-08 15:02:17 +00:00
{
2018-11-26 00:56:50 +00:00
ThrottlerPtr throttler = std : : make_shared < Throttler > ( max_client_network_bandwidth , 0 , " " ) ;
2017-12-08 15:02:17 +00:00
connection - > setThrottler ( throttler ) ;
}
2019-03-07 23:13:39 +00:00
connection - > getServerVersion ( connection_parameters . timeouts ,
server_name , server_version_major , server_version_minor , server_version_patch , server_revision ) ;
2017-04-01 07:20:54 +00:00
2018-07-31 21:36:18 +00:00
server_version = toString ( server_version_major ) + " . " + toString ( server_version_minor ) + " . " + toString ( server_version_patch ) ;
2018-03-08 08:55:27 +00:00
2020-04-15 01:12:32 +00:00
if ( server_display_name = connection - > getServerDisplayName ( connection_parameters . timeouts ) ; server_display_name . empty ( ) )
2018-03-05 14:47:10 +00:00
{
server_display_name = config ( ) . getString ( " host " , " localhost " ) ;
}
2017-11-02 14:45:23 +00:00
if ( is_interactive )
{
2017-04-01 07:20:54 +00:00
std : : cout < < " Connected to " < < server_name
2019-09-10 19:28:33 +00:00
< < " server version " < < server_version
< < " revision " < < server_revision
< < " . " < < std : : endl < < std : : endl ;
2020-05-02 17:19:43 +00:00
auto client_version_tuple = std : : make_tuple ( VERSION_MAJOR , VERSION_MINOR , VERSION_PATCH ) ;
auto server_version_tuple = std : : make_tuple ( server_version_major , server_version_minor , server_version_patch ) ;
if ( client_version_tuple < server_version_tuple )
2019-09-10 20:10:23 +00:00
{
2019-09-10 19:28:33 +00:00
std : : cout < < " ClickHouse client version is older than ClickHouse server. "
< < " It may lack support for new features. "
< < std : : endl < < std : : endl ;
}
2020-05-02 17:19:43 +00:00
else if ( client_version_tuple > server_version_tuple )
{
std : : cout < < " ClickHouse server version is older than ClickHouse client. "
< < " It may indicate that the server is out of date and can be upgraded. "
< < std : : endl < < std : : endl ;
}
2017-04-01 07:20:54 +00:00
}
}
2020-03-09 03:41:03 +00:00
inline String prompt ( ) const
2018-02-26 13:24:06 +00:00
{
2018-03-12 13:23:15 +00:00
return boost : : replace_all_copy ( prompt_by_server_display_name , " {database} " , config ( ) . getString ( " database " , " default " ) ) ;
2018-02-26 13:24:06 +00:00
}
2017-04-01 07:20:54 +00:00
void nonInteractive ( )
{
2017-12-19 19:46:02 +00:00
String text ;
2017-04-01 07:20:54 +00:00
if ( config ( ) . has ( " query " ) )
2019-08-28 15:05:38 +00:00
text = config ( ) . getRawString ( " query " ) ; /// Poco configuration should not process substitutions in form of ${...} inside query.
2017-04-01 07:20:54 +00:00
else
{
/// If 'query' parameter is not set, read a query from stdin.
/// The query is read entirely into memory (streaming is disabled).
ReadBufferFromFileDescriptor in ( STDIN_FILENO ) ;
2017-12-19 19:46:02 +00:00
readStringUntilEOF ( text , in ) ;
2017-04-01 07:20:54 +00:00
}
2020-06-26 01:34:06 +00:00
processQueryText ( text ) ;
2017-04-01 07:20:54 +00:00
}
2020-06-30 09:25:23 +00:00
void nonInteractiveWithFuzzing ( )
{
if ( config ( ) . has ( " query " ) )
{
// Poco configuration should not process substitutions in form of
// ${...} inside query
processWithFuzzing ( config ( ) . getRawString ( " query " ) ) ;
return ;
}
2020-07-07 12:35:53 +00:00
// Try to stream the queries from stdin, without reading all of them
// into memory. The interface of the parser does not support streaming,
// in particular, it can't distinguish the end of partial input buffer
// and the final end of input file. This means we have to try to split
// the input into separate queries here. Two patterns of input are
// especially interesing:
// 1) multiline query:
// select 1
// from system.numbers;
//
// 2) csv insert with in-place data:
// insert into t format CSV 1;2
//
// (1) means we can't split on new line, and (2) means we can't split on
// semicolon. Solution: split on ';\n'. This sequence is frequent enough
// in the SQL tests which are our principal input for fuzzing. Now we
// have another interesting case:
// 3) escaped semicolon followed by newline, e.g.
// select ';
// '
//
// To handle (3), parse until we can, and read more data if the parser
// complains. Hopefully this should be enough...
2020-06-30 09:25:23 +00:00
ReadBufferFromFileDescriptor in ( STDIN_FILENO ) ;
std : : string text ;
while ( ! in . eof ( ) )
{
2020-07-07 12:35:53 +00:00
// Read until separator.
2020-07-03 12:52:16 +00:00
while ( ! in . eof ( ) )
2020-06-30 09:25:23 +00:00
{
2020-07-07 12:35:53 +00:00
char * next_separator = find_first_symbols < ' ; ' > ( in . position ( ) ,
2020-07-03 12:52:16 +00:00
in . buffer ( ) . end ( ) ) ;
2020-07-07 12:35:53 +00:00
if ( next_separator < in . buffer ( ) . end ( ) )
2020-07-03 12:52:16 +00:00
{
2020-07-07 12:35:53 +00:00
next_separator + + ;
if ( next_separator < in . buffer ( ) . end ( )
& & * next_separator = = ' \n ' )
{
// Found ';\n', append it to the query text and try to
// parse.
next_separator + + ;
text . append ( in . position ( ) , next_separator - in . position ( ) ) ;
in . position ( ) = next_separator ;
break ;
}
2020-07-03 12:52:16 +00:00
}
// Didn't find the semicolon and reached the end of buffer.
2020-07-07 12:35:53 +00:00
text . append ( in . position ( ) , next_separator - in . position ( ) ) ;
in . position ( ) = next_separator ;
2020-07-03 12:52:16 +00:00
if ( text . size ( ) > 1024 * 1024 )
{
2020-07-07 12:35:53 +00:00
// We've read a lot of text and still haven't seen a separator.
2020-07-03 12:52:16 +00:00
// Likely some pathological input, just fall through to prevent
// too long loops.
break ;
}
2020-06-30 09:25:23 +00:00
}
2020-07-07 12:35:53 +00:00
// Parse and execute what we've read.
2020-06-30 09:25:23 +00:00
fprintf ( stderr , " will now parse '%s' \n " , text . c_str ( ) ) ;
2020-07-07 12:35:53 +00:00
const auto * new_end = processWithFuzzing ( text ) ;
2020-06-30 09:25:23 +00:00
if ( new_end > & text [ 0 ] )
{
const auto rest_size = text . size ( ) - ( new_end - & text [ 0 ] ) ;
fprintf ( stderr , " total %zd, rest %zd \n " , text . size ( ) , rest_size ) ;
memcpy ( & text [ 0 ] , new_end , rest_size ) ;
text . resize ( rest_size ) ;
}
else
{
fprintf ( stderr , " total %zd, can't parse \n " , text . size ( ) ) ;
}
if ( ! connection - > isConnected ( ) )
{
// Uh-oh...
std : : cerr < < " Lost connection to the server. " < < std : : endl ;
2020-07-07 12:35:53 +00:00
last_exception_received_from_server
= std : : make_unique < Exception > ( 210 , " ~ " ) ;
2020-06-30 09:25:23 +00:00
return ;
}
2020-07-03 12:52:16 +00:00
if ( text . size ( ) > 4 * 1024 )
2020-06-30 09:25:23 +00:00
{
2020-07-07 12:35:53 +00:00
// Some pathological situation where the text is larger than 4kB
2020-06-30 09:25:23 +00:00
// and we still cannot parse a single query in it. Abort.
std : : cerr < < " Read too much text and still can't parse a query. "
" Aborting. " < < std : : endl ;
2020-07-07 12:35:53 +00:00
last_exception_received_from_server
= std : : make_unique < Exception > ( 1 , " ~ " ) ;
2020-07-03 12:52:16 +00:00
// return;
exit ( 1 ) ;
2020-06-30 09:25:23 +00:00
}
}
}
2020-06-26 01:34:06 +00:00
bool processQueryText ( const String & text )
2017-04-01 07:20:54 +00:00
{
2019-11-13 15:45:19 +00:00
if ( exit_strings . end ( ) ! = exit_strings . find ( trim ( text , [ ] ( char c ) { return isWhitespaceASCII ( c ) | | c = = ' ; ' ; } ) ) )
return false ;
2020-06-26 01:34:06 +00:00
if ( ! config ( ) . has ( " multiquery " ) )
2017-04-01 07:20:54 +00:00
{
2020-06-26 05:44:42 +00:00
assert ( ! query_fuzzer_runs ) ;
2020-06-26 04:37:18 +00:00
processTextAsSingleQuery ( text ) ;
2020-06-26 01:34:06 +00:00
return true ;
}
2018-12-27 13:09:11 +00:00
2020-06-26 05:44:42 +00:00
if ( query_fuzzer_runs )
{
processWithFuzzing ( text ) ;
return true ;
}
2020-06-26 01:34:06 +00:00
return processMultiQuery ( text ) ;
}
2017-04-01 07:20:54 +00:00
2020-06-26 01:34:06 +00:00
bool processMultiQuery ( const String & text )
{
const bool test_mode = config ( ) . has ( " testmode " ) ;
2017-04-01 07:20:54 +00:00
2020-06-26 01:34:06 +00:00
{ /// disable logs if expects errors
TestHint test_hint ( test_mode , text ) ;
if ( test_hint . clientError ( ) | | test_hint . serverError ( ) )
2020-06-26 04:37:18 +00:00
processTextAsSingleQuery ( " SET send_logs_level = 'none' " ) ;
2020-06-26 01:34:06 +00:00
}
2018-08-20 03:21:26 +00:00
2020-06-26 01:34:06 +00:00
/// Several queries separated by ';'.
/// INSERT data is ended by the end of line, not ';'.
2017-12-22 18:29:03 +00:00
2020-06-26 01:34:06 +00:00
const char * begin = text . data ( ) ;
const char * end = begin + text . size ( ) ;
2017-04-01 07:20:54 +00:00
2020-06-26 01:34:06 +00:00
while ( begin < end )
{
const char * pos = begin ;
ASTPtr orig_ast = parseQuery ( pos , end , true ) ;
2017-04-01 07:20:54 +00:00
2020-06-26 01:34:06 +00:00
if ( ! orig_ast )
{
if ( ignore_error )
2017-12-19 23:14:37 +00:00
{
2020-06-26 01:34:06 +00:00
Tokens tokens ( begin , end ) ;
IParser : : Pos token_iterator ( tokens , context . getSettingsRef ( ) . max_parser_depth ) ;
while ( token_iterator - > type ! = TokenType : : Semicolon & & token_iterator . isValid ( ) )
+ + token_iterator ;
begin = token_iterator - > end ;
continue ;
2017-12-19 23:14:37 +00:00
}
2020-06-26 01:34:06 +00:00
return true ;
}
2017-04-01 07:20:54 +00:00
2020-06-26 01:34:06 +00:00
auto * insert = orig_ast - > as < ASTInsertQuery > ( ) ;
2017-04-01 07:20:54 +00:00
2020-06-26 01:34:06 +00:00
if ( insert & & insert - > data )
{
pos = find_first_symbols < ' \n ' > ( insert - > data , end ) ;
insert - > end = pos ;
}
2017-04-01 07:20:54 +00:00
2020-06-26 01:34:06 +00:00
String str = text . substr ( begin - text . data ( ) , pos - begin ) ;
2018-07-27 17:19:22 +00:00
2020-06-26 01:34:06 +00:00
begin = pos ;
while ( isWhitespaceASCII ( * begin ) | | * begin = = ' ; ' )
+ + begin ;
2019-02-07 16:09:06 +00:00
2020-06-26 01:34:06 +00:00
TestHint test_hint ( test_mode , str ) ;
expected_client_error = test_hint . clientError ( ) ;
expected_server_error = test_hint . serverError ( ) ;
2017-04-01 07:20:54 +00:00
2020-06-26 01:34:06 +00:00
try
{
auto ast_to_process = orig_ast ;
if ( insert & & insert - > data )
2020-06-26 04:37:18 +00:00
{
2020-06-26 01:34:06 +00:00
ast_to_process = nullptr ;
2020-06-26 04:37:18 +00:00
processTextAsSingleQuery ( str ) ;
}
else
{
parsed_query = ast_to_process ;
full_query = str ;
query_to_send = str ;
processParsedSingleQuery ( ) ;
}
2020-06-26 01:34:06 +00:00
}
catch ( . . . )
{
last_exception_received_from_server = std : : make_unique < Exception > ( getCurrentExceptionMessage ( true ) , getCurrentExceptionCode ( ) ) ;
actual_client_error = last_exception_received_from_server - > code ( ) ;
if ( ! ignore_error & & ( ! actual_client_error | | actual_client_error ! = expected_client_error ) )
std : : cerr < < " Error on processing query: " < < str < < std : : endl < < last_exception_received_from_server - > message ( ) ;
received_exception_from_server = true ;
2017-04-01 07:20:54 +00:00
}
2020-06-26 01:34:06 +00:00
if ( ! test_hint . checkActual ( actual_server_error , actual_client_error , received_exception_from_server , last_exception_received_from_server ) )
connection - > forceConnected ( connection_parameters . timeouts ) ;
if ( received_exception_from_server & & ! ignore_error )
{
if ( is_interactive )
break ;
else
return false ;
}
2017-04-01 07:20:54 +00:00
}
2020-06-26 01:34:06 +00:00
return true ;
2017-04-01 07:20:54 +00:00
}
2020-06-30 09:25:23 +00:00
// Returns the last position we could parse.
const char * processWithFuzzing ( const String & text )
2020-06-26 05:44:42 +00:00
{
/// Several queries separated by ';'.
/// INSERT data is ended by the end of line, not ';'.
const char * begin = text . data ( ) ;
const char * end = begin + text . size ( ) ;
while ( begin < end )
{
// Skip whitespace before the query
while ( isWhitespaceASCII ( * begin ) | | * begin = = ' ; ' )
{
+ + begin ;
}
2020-07-07 12:35:53 +00:00
const auto * this_query_begin = begin ;
2020-06-26 05:44:42 +00:00
ASTPtr orig_ast = parseQuery ( begin , end , true ) ;
if ( ! orig_ast )
{
// Can't continue after a parsing error
2020-06-30 09:25:23 +00:00
return begin ;
2020-06-26 05:44:42 +00:00
}
2020-07-07 12:35:53 +00:00
auto * as_insert = orig_ast - > as < ASTInsertQuery > ( ) ;
2020-06-26 05:44:42 +00:00
if ( as_insert & & as_insert - > data )
{
// INSERT data is ended by newline
as_insert - > end = find_first_symbols < ' \n ' > ( as_insert - > data , end ) ;
begin = as_insert - > end ;
}
full_query = text . substr ( this_query_begin - text . data ( ) ,
begin - text . data ( ) ) ;
ASTPtr fuzz_base = orig_ast ;
for ( int fuzz_step = 0 ; fuzz_step < query_fuzzer_runs ; fuzz_step + + )
{
2020-07-03 12:52:16 +00:00
fprintf ( stderr , " fuzzing step %d for query at pos %zd \n " ,
fuzz_step , this_query_begin - text . data ( ) ) ;
2020-06-26 05:44:42 +00:00
ASTPtr ast_to_process ;
try
{
2020-07-07 12:35:53 +00:00
std : : stringstream dump_before_fuzz ;
fuzz_base - > dumpTree ( dump_before_fuzz ) ;
2020-06-26 05:44:42 +00:00
auto base_before_fuzz = fuzz_base - > formatForErrorMessage ( ) ;
2020-07-07 12:35:53 +00:00
2020-06-26 05:44:42 +00:00
ast_to_process = fuzz_base - > clone ( ) ;
fuzzer . fuzzMain ( ast_to_process ) ;
2020-07-07 12:35:53 +00:00
2020-06-26 05:44:42 +00:00
auto base_after_fuzz = fuzz_base - > formatForErrorMessage ( ) ;
// Debug AST cloning errors.
2020-07-03 12:52:16 +00:00
if ( base_before_fuzz ! = base_after_fuzz )
{
fprintf ( stderr , " base before fuzz: %s \n "
" base after fuzz: %s \n " , base_before_fuzz . c_str ( ) ,
base_after_fuzz . c_str ( ) ) ;
2020-07-07 12:35:53 +00:00
fprintf ( stderr , " dump before fuzz: \n %s \n " ,
dump_before_fuzz . str ( ) . c_str ( ) ) ;
fprintf ( stderr , " dump after fuzz: \n " ) ;
fuzz_base - > dumpTree ( std : : cerr ) ;
2020-07-03 12:52:16 +00:00
assert ( false ) ;
}
2020-06-26 05:44:42 +00:00
auto fuzzed_text = ast_to_process - > formatForErrorMessage ( ) ;
if ( fuzz_step > 0 & & fuzzed_text = = base_before_fuzz )
{
fprintf ( stderr , " got boring ast \n " ) ;
continue ;
}
parsed_query = ast_to_process ;
query_to_send = parsed_query - > formatForErrorMessage ( ) ;
processParsedSingleQuery ( ) ;
}
catch ( . . . )
{
last_exception_received_from_server = std : : make_unique < Exception > ( getCurrentExceptionMessage ( true ) , getCurrentExceptionCode ( ) ) ;
received_exception_from_server = true ;
std : : cerr < < " Error on processing query: " < < ast_to_process - > formatForErrorMessage ( ) < < std : : endl < < last_exception_received_from_server - > message ( ) ;
}
2020-07-07 12:35:53 +00:00
if ( received_exception_from_server )
2020-06-26 05:44:42 +00:00
{
2020-07-07 12:35:53 +00:00
// Query completed with error, ignore it and fuzz again.
fprintf ( stderr , " Got error, will fuzz again \n " ) ;
received_exception_from_server = false ;
last_exception_received_from_server . reset ( ) ;
2020-06-26 05:44:42 +00:00
continue ;
}
else if ( ast_to_process - > formatForErrorMessage ( ) . size ( ) > 500 )
{
2020-07-03 12:52:16 +00:00
// ast too long, start from original ast
2020-06-26 05:44:42 +00:00
fprintf ( stderr , " current ast too long, won't elaborate \n " ) ;
fuzz_base = orig_ast ;
}
else
{
// fuzz starting from this successful query
fprintf ( stderr , " using this ast as etalon \n " ) ;
fuzz_base = ast_to_process ;
}
}
}
2020-07-03 12:52:16 +00:00
return begin ;
2020-06-26 05:44:42 +00:00
}
2020-06-26 04:37:18 +00:00
void processTextAsSingleQuery ( const String & text_ )
{
full_query = text_ ;
/// Some parts of a query (result output and formatting) are executed
/// client-side. Thus we need to parse the query.
const char * begin = full_query . data ( ) ;
parsed_query = parseQuery ( begin , begin + full_query . size ( ) , false ) ;
if ( ! parsed_query )
return ;
// An INSERT query may have the data that follow query text. Remove the
/// Send part of query without data, because data will be sent separately.
auto * insert = parsed_query - > as < ASTInsertQuery > ( ) ;
if ( insert & & insert - > data )
{
query_to_send = full_query . substr ( 0 , insert - > data - full_query . data ( ) ) ;
}
else
{
query_to_send = full_query ;
}
processParsedSingleQuery ( ) ;
}
// Parameters are in global variables:
// 'parsed_query' -- the query AST,
// 'query_to_send' -- the query text that is sent to server,
// 'full_query' -- for INSERT queries, contains the query and the data that
// follow it. Its memory is referenced by ASTInsertQuery::begin, end.
void processParsedSingleQuery ( )
2017-04-01 07:20:54 +00:00
{
resetOutput ( ) ;
2020-07-07 12:35:53 +00:00
last_exception_received_from_server . reset ( ) ;
2020-06-26 01:34:06 +00:00
received_exception_from_server = false ;
2017-04-01 07:20:54 +00:00
if ( echo_queries )
{
2020-06-26 04:37:18 +00:00
writeString ( full_query , std_out ) ;
2017-04-01 07:20:54 +00:00
writeChar ( ' \n ' , std_out ) ;
std_out . next ( ) ;
}
watch . restart ( ) ;
processed_rows = 0 ;
progress . reset ( ) ;
show_progress_bar = false ;
written_progress_chars = 0 ;
written_first_block = false ;
2019-03-26 13:07:00 +00:00
{
/// Temporarily apply query settings to context.
std : : optional < Settings > old_settings ;
SCOPE_EXIT ( { if ( old_settings ) context . setSettings ( * old_settings ) ; } ) ;
auto apply_query_settings = [ & ] ( const IAST & settings_ast )
{
if ( ! old_settings )
old_settings . emplace ( context . getSettingsRef ( ) ) ;
2019-04-18 23:29:32 +00:00
context . applySettingsChanges ( settings_ast . as < ASTSetQuery > ( ) - > changes ) ;
2019-03-26 13:07:00 +00:00
} ;
const auto * insert = parsed_query - > as < ASTInsertQuery > ( ) ;
if ( insert & & insert - > settings_ast )
apply_query_settings ( * insert - > settings_ast ) ;
/// FIXME: try to prettify this cast using `as<>()`
const auto * with_output = dynamic_cast < const ASTQueryWithOutput * > ( parsed_query . get ( ) ) ;
if ( with_output & & with_output - > settings_ast )
apply_query_settings ( * with_output - > settings_ast ) ;
2017-04-17 16:16:04 +00:00
2019-03-07 23:13:39 +00:00
connection - > forceConnected ( connection_parameters . timeouts ) ;
2019-03-26 13:07:00 +00:00
2019-05-30 20:12:44 +00:00
ASTPtr input_function ;
if ( insert & & insert - > select )
2019-05-30 21:33:06 +00:00
insert - > tryFindInputFunction ( input_function ) ;
2019-05-30 20:12:44 +00:00
/// INSERT query for which data transfer is needed (not an INSERT SELECT or input()) is processed separately.
2020-04-25 11:33:47 +00:00
if ( insert & & ( ! insert - > select | | input_function ) & & ! insert - > watch )
2019-05-30 20:12:44 +00:00
{
if ( input_function & & insert - > format . empty ( ) )
2019-09-05 13:17:01 +00:00
throw Exception ( " FORMAT must be specified for function input() " , ErrorCodes::INVALID_USAGE_OF_INPUT) ;
2019-03-26 13:07:00 +00:00
processInsertQuery ( ) ;
2019-05-30 20:12:44 +00:00
}
2019-03-26 13:07:00 +00:00
else
processOrdinaryQuery ( ) ;
}
2017-04-01 07:20:54 +00:00
/// Do not change context (current DB, settings) in case of an exception.
2020-06-26 01:34:06 +00:00
if ( ! received_exception_from_server )
2017-04-01 07:20:54 +00:00
{
2019-03-11 13:22:51 +00:00
if ( const auto * set_query = parsed_query - > as < ASTSetQuery > ( ) )
2017-04-01 07:20:54 +00:00
{
/// Save all changes in settings to avoid losing them if the connection is lost.
2017-06-07 12:54:35 +00:00
for ( const auto & change : set_query - > changes )
2017-04-01 07:20:54 +00:00
{
2017-06-07 12:54:35 +00:00
if ( change . name = = " profile " )
current_profile = change . value . safeGet < String > ( ) ;
2017-04-01 07:20:54 +00:00
else
2019-04-18 23:29:32 +00:00
context . applySettingChange ( change ) ;
2017-04-01 07:20:54 +00:00
}
}
2019-03-11 13:22:51 +00:00
if ( const auto * use_query = parsed_query - > as < ASTUseQuery > ( ) )
2017-04-01 07:20:54 +00:00
{
const String & new_database = use_query - > database ;
/// If the client initiates the reconnection, it takes the settings from the config.
config ( ) . setString ( " database " , new_database ) ;
/// If the connection initiates the reconnection, it uses its variable.
connection - > setDefaultDatabase ( new_database ) ;
}
}
if ( is_interactive )
{
std : : cout < < std : : endl
< < processed_rows < < " rows in set. Elapsed: " < < watch . elapsedSeconds ( ) < < " sec. " ;
2019-05-20 13:24:53 +00:00
if ( progress . read_rows > = 1000 )
2017-04-01 07:20:54 +00:00
writeFinalProgress ( ) ;
std : : cout < < std : : endl < < std : : endl ;
}
else if ( print_time_to_stderr )
{
std : : cerr < < watch . elapsedSeconds ( ) < < " \n " ;
}
}
/// Convert external tables to ExternalTableData and send them using the connection.
void sendExternalTables ( )
{
2019-03-11 13:22:51 +00:00
const auto * select = parsed_query - > as < ASTSelectWithUnionQuery > ( ) ;
2017-04-01 07:20:54 +00:00
if ( ! select & & ! external_tables . empty ( ) )
throw Exception ( " External tables could be sent only with select query " , ErrorCodes : : BAD_ARGUMENTS ) ;
2020-02-19 15:12:21 +00:00
std : : vector < ExternalTableDataPtr > data ;
2017-04-01 07:20:54 +00:00
for ( auto & table : external_tables )
data . emplace_back ( table . getData ( context ) ) ;
connection - > sendExternalTablesData ( data ) ;
}
2019-01-22 19:56:53 +00:00
/// Process the query that doesn't require transferring data blocks to the server.
2017-04-01 07:20:54 +00:00
void processOrdinaryQuery ( )
{
2020-06-15 20:07:39 +00:00
/// Rewrite query only when we have query parameters.
/// Note that if query is rewritten, comments in query are lost.
/// But the user often wants to see comments in server logs, query log, processlist, etc.
if ( ! query_parameters . empty ( ) )
2019-06-29 16:58:32 +00:00
{
/// Replace ASTQueryParameter with ASTLiteral for prepared statements.
ReplaceQueryParameterVisitor visitor ( query_parameters ) ;
visitor . visit ( parsed_query ) ;
2019-06-15 18:22:48 +00:00
2019-06-29 16:58:32 +00:00
/// Get new query after substitutions. Note that it cannot be done for INSERT query with embedded data.
2020-06-26 04:37:18 +00:00
query_to_send = serializeAST ( * parsed_query ) ;
2019-06-29 16:58:32 +00:00
}
2019-06-15 18:22:48 +00:00
2020-06-26 01:34:06 +00:00
int retries_left = 10 ;
for ( ; ; )
2020-03-05 13:12:00 +00:00
{
2020-06-26 01:34:06 +00:00
assert ( retries_left > 0 ) ;
2020-03-05 13:12:00 +00:00
try
{
connection - > sendQuery (
connection_parameters . timeouts ,
2020-06-26 04:37:18 +00:00
query_to_send ,
2020-03-05 13:12:00 +00:00
query_id ,
QueryProcessingStage : : Complete ,
& context . getSettingsRef ( ) ,
2020-04-15 01:58:10 +00:00
& context . getClientInfo ( ) ,
2020-03-05 13:12:00 +00:00
true ) ;
sendExternalTables ( ) ;
receiveResult ( ) ;
break ;
}
catch ( const Exception & e )
{
2020-06-26 01:34:06 +00:00
/// Retry when the server said "Client should retry" and no rows
/// has been received yet.
if ( processed_rows = = 0
& & e . code ( ) = = ErrorCodes : : DEADLOCK_AVOIDED
& & - - retries_left )
{
std : : cerr < < " Got a transient error from the server, will "
< < " retry ( " < < retries_left < < " retries left) " ;
}
else
{
throw ;
}
2020-03-05 13:12:00 +00:00
}
}
2017-04-01 07:20:54 +00:00
}
2019-01-22 19:56:53 +00:00
/// Process the query that requires transferring data blocks to the server.
2017-04-01 07:20:54 +00:00
void processInsertQuery ( )
{
2020-06-26 04:37:18 +00:00
const auto parsed_insert_query = parsed_query - > as < ASTInsertQuery & > ( ) ;
2020-02-21 20:01:38 +00:00
if ( ! parsed_insert_query . data & & ( is_interactive | | ( ! stdin_is_a_tty & & std_in . eof ( ) ) ) )
2017-04-01 07:20:54 +00:00
throw Exception ( " No data to insert " , ErrorCodes : : NO_DATA_TO_INSERT ) ;
2020-04-15 01:58:10 +00:00
connection - > sendQuery (
connection_parameters . timeouts ,
2020-06-26 04:37:18 +00:00
query_to_send ,
2020-04-15 01:58:10 +00:00
query_id ,
QueryProcessingStage : : Complete ,
& context . getSettingsRef ( ) ,
& context . getClientInfo ( ) ,
true ) ;
2017-04-01 07:20:54 +00:00
sendExternalTables ( ) ;
/// Receive description of table structure.
Block sample ;
2018-12-04 20:03:04 +00:00
ColumnsDescription columns_description ;
if ( receiveSampleBlock ( sample , columns_description ) )
2017-04-01 07:20:54 +00:00
{
/// If structure was received (thus, server has not thrown an exception),
/// send our data with that structure.
2018-12-04 20:03:04 +00:00
sendData ( sample , columns_description ) ;
2018-06-08 19:50:15 +00:00
receiveEndOfQuery ( ) ;
2017-04-01 07:20:54 +00:00
}
}
2017-07-12 03:03:56 +00:00
ASTPtr parseQuery ( const char * & pos , const char * end , bool allow_multi_statements )
2017-04-01 07:20:54 +00:00
{
2018-10-09 14:32:11 +00:00
ParserQuery parser ( end , true ) ;
2017-04-01 07:20:54 +00:00
ASTPtr res ;
2020-04-15 20:28:05 +00:00
const auto & settings = context . getSettingsRef ( ) ;
2020-04-16 02:59:16 +00:00
size_t max_length = 0 ;
if ( ! allow_multi_statements )
max_length = settings . max_query_size ;
2020-04-15 20:28:05 +00:00
2017-12-22 18:29:03 +00:00
if ( is_interactive | | ignore_error )
2017-04-01 07:20:54 +00:00
{
String message ;
2020-04-16 02:59:16 +00:00
res = tryParseQuery ( parser , pos , end , message , true , " " , allow_multi_statements , max_length , settings . max_parser_depth ) ;
2017-04-01 07:20:54 +00:00
if ( ! res )
{
std : : cerr < < std : : endl < < message < < std : : endl < < std : : endl ;
return nullptr ;
}
}
else
2020-04-16 02:59:16 +00:00
res = parseQueryAndMovePosition ( parser , pos , end , " " , allow_multi_statements , max_length , settings . max_parser_depth ) ;
2017-04-01 07:20:54 +00:00
if ( is_interactive )
{
std : : cout < < std : : endl ;
formatAST ( * res , std : : cout ) ;
std : : cout < < std : : endl < < std : : endl ;
}
return res ;
}
2018-12-04 20:03:04 +00:00
void sendData ( Block & sample , const ColumnsDescription & columns_description )
2017-04-01 07:20:54 +00:00
{
/// If INSERT data must be sent.
2019-03-15 13:12:11 +00:00
const auto * parsed_insert_query = parsed_query - > as < ASTInsertQuery > ( ) ;
2017-04-01 07:20:54 +00:00
if ( ! parsed_insert_query )
return ;
if ( parsed_insert_query - > data )
{
/// Send data contained in the query.
ReadBufferFromMemory data_in ( parsed_insert_query - > data , parsed_insert_query - > end - parsed_insert_query - > data ) ;
2018-12-04 20:03:04 +00:00
sendDataFrom ( data_in , sample , columns_description ) ;
2017-04-01 07:20:54 +00:00
}
else if ( ! is_interactive )
{
/// Send data read from stdin.
2018-12-04 20:03:04 +00:00
sendDataFrom ( std_in , sample , columns_description ) ;
2017-04-01 07:20:54 +00:00
}
else
throw Exception ( " No data to insert " , ErrorCodes : : NO_DATA_TO_INSERT ) ;
}
2018-12-04 20:03:04 +00:00
void sendDataFrom ( ReadBuffer & buf , Block & sample , const ColumnsDescription & columns_description )
2017-04-01 07:20:54 +00:00
{
String current_format = insert_format ;
/// Data format can be specified in the INSERT query.
2019-03-11 13:22:51 +00:00
if ( const auto * insert = parsed_query - > as < ASTInsertQuery > ( ) )
2019-02-19 19:26:41 +00:00
{
2017-04-01 07:20:54 +00:00
if ( ! insert - > format . empty ( ) )
current_format = insert - > format ;
2019-02-19 19:26:41 +00:00
}
2017-04-01 07:20:54 +00:00
BlockInputStreamPtr block_input = context . getInputFormat (
current_format , buf , sample , insert_format_max_block_size ) ;
2019-03-14 15:20:51 +00:00
const auto & column_defaults = columns_description . getDefaults ( ) ;
2018-07-11 12:05:04 +00:00
if ( ! column_defaults . empty ( ) )
block_input = std : : make_shared < AddingDefaultsBlockInputStream > ( block_input , column_defaults , context ) ;
2018-12-04 20:03:04 +00:00
2018-07-11 12:05:04 +00:00
BlockInputStreamPtr async_block_input = std : : make_shared < AsynchronousBlockInputStream > ( block_input ) ;
2017-04-01 07:20:54 +00:00
async_block_input - > readPrefix ( ) ;
while ( true )
{
Block block = async_block_input - > read ( ) ;
2018-06-08 19:50:15 +00:00
/// Check if server send Log packet
2019-08-29 17:30:16 +00:00
receiveLogs ( ) ;
2019-08-29 12:36:06 +00:00
/// Check if server send Exception packet
2018-06-08 19:50:15 +00:00
auto packet_type = connection - > checkPacket ( ) ;
2019-08-28 17:52:17 +00:00
if ( packet_type & & * packet_type = = Protocol : : Server : : Exception )
2019-10-01 10:48:46 +00:00
{
2019-11-19 18:42:51 +00:00
/*
* We ' re exiting with error , so it makes sense to kill the
* input stream without waiting for it to complete .
*/
async_block_input - > cancel ( true ) ;
2019-08-28 17:52:17 +00:00
return ;
2019-10-01 10:48:46 +00:00
}
2017-04-01 07:20:54 +00:00
connection - > sendData ( block ) ;
processed_rows + = block . rows ( ) ;
2018-06-13 19:01:07 +00:00
2017-04-01 07:20:54 +00:00
if ( ! block )
break ;
}
async_block_input - > readSuffix ( ) ;
}
/// Flush all buffers.
void resetOutput ( )
{
2018-06-07 18:46:53 +00:00
block_out_stream . reset ( ) ;
logs_out_stream . reset ( ) ;
2018-06-06 20:57:07 +00:00
2017-04-01 07:20:54 +00:00
if ( pager_cmd )
{
pager_cmd - > in . close ( ) ;
pager_cmd - > wait ( ) ;
}
pager_cmd = nullptr ;
2018-06-06 20:57:07 +00:00
2017-04-01 07:20:54 +00:00
if ( out_file_buf )
{
out_file_buf - > next ( ) ;
2017-11-20 04:41:56 +00:00
out_file_buf . reset ( ) ;
2017-04-01 07:20:54 +00:00
}
2018-06-06 20:57:07 +00:00
2018-06-07 18:46:53 +00:00
if ( out_logs_buf )
{
out_logs_buf - > next ( ) ;
out_logs_buf . reset ( ) ;
}
2017-04-01 07:20:54 +00:00
std_out . next ( ) ;
}
/// Receives and processes packets coming from server.
/// Also checks if query execution should be cancelled.
void receiveResult ( )
{
InterruptListener interrupt_listener ;
bool cancelled = false ;
2018-11-06 11:23:19 +00:00
// TODO: get the poll_interval from commandline.
2019-03-07 23:13:39 +00:00
const auto receive_timeout = connection_parameters . timeouts . receive_timeout ;
2019-02-07 13:47:16 +00:00
constexpr size_t default_poll_interval = 1000000 ; /// in microseconds
constexpr size_t min_poll_interval = 5000 ; /// in microseconds
2018-11-06 11:23:19 +00:00
const size_t poll_interval
= std : : max ( min_poll_interval , std : : min < size_t > ( receive_timeout . totalMicroseconds ( ) , default_poll_interval ) ) ;
2017-04-01 07:20:54 +00:00
while ( true )
{
2019-02-05 10:19:34 +00:00
Stopwatch receive_watch ( CLOCK_MONOTONIC_COARSE ) ;
2018-11-06 11:23:19 +00:00
while ( true )
2017-04-01 07:20:54 +00:00
{
2018-11-06 11:23:19 +00:00
/// Has the Ctrl+C been pressed and thus the query should be cancelled?
/// If this is the case, inform the server about it and receive the remaining packets
/// to avoid losing sync.
if ( ! cancelled )
2017-04-01 07:20:54 +00:00
{
2020-04-11 19:51:04 +00:00
auto cancel_query = [ & ]
{
2018-11-06 11:23:19 +00:00
connection - > sendCancel ( ) ;
cancelled = true ;
if ( is_interactive )
2020-04-11 19:51:04 +00:00
{
if ( written_progress_chars )
clearProgress ( ) ;
2018-11-06 11:23:19 +00:00
std : : cout < < " Cancelling query. " < < std : : endl ;
2020-04-11 19:51:04 +00:00
}
2017-04-01 07:20:54 +00:00
2018-11-06 11:23:19 +00:00
/// Pressing Ctrl+C twice results in shut down.
interrupt_listener . unblock ( ) ;
} ;
2017-04-01 07:20:54 +00:00
2018-11-06 11:23:19 +00:00
if ( interrupt_listener . check ( ) )
{
2020-03-23 02:12:31 +00:00
cancel_query ( ) ;
2019-02-01 14:22:27 +00:00
}
else
{
2019-02-05 10:19:34 +00:00
double elapsed = receive_watch . elapsedSeconds ( ) ;
2018-11-06 11:23:19 +00:00
if ( elapsed > receive_timeout . totalSeconds ( ) )
{
std : : cout < < " Timeout exceeded while receiving data from server. "
< < " Waited for " < < static_cast < size_t > ( elapsed ) < < " seconds, "
< < " timeout is " < < receive_timeout . totalSeconds ( ) < < " seconds. " < < std : : endl ;
2020-03-23 02:12:31 +00:00
cancel_query ( ) ;
2018-11-06 11:23:19 +00:00
}
}
2017-04-01 07:20:54 +00:00
}
2018-11-06 11:23:19 +00:00
/// Poll for changes after a cancellation check, otherwise it never reached
/// because of progress updates from server.
if ( connection - > poll ( poll_interval ) )
2020-05-31 18:49:19 +00:00
break ;
2017-04-01 07:20:54 +00:00
}
2020-06-01 18:58:38 +00:00
if ( ! receiveAndProcessPacket ( cancelled ) )
2017-04-01 07:20:54 +00:00
break ;
}
if ( cancelled & & is_interactive )
std : : cout < < " Query was cancelled. " < < std : : endl ;
}
/// Receive a part of the result, or progress info or an exception and process it.
/// Returns true if one should continue receiving packets.
2020-06-01 18:58:38 +00:00
/// Output of result is suppressed if query was cancelled.
bool receiveAndProcessPacket ( bool cancelled )
2017-04-01 07:20:54 +00:00
{
2019-11-15 16:34:43 +00:00
Packet packet = connection - > receivePacket ( ) ;
2017-04-01 07:20:54 +00:00
switch ( packet . type )
{
case Protocol : : Server : : Data :
2020-06-01 18:58:38 +00:00
if ( ! cancelled )
onData ( packet . block ) ;
2017-04-01 07:20:54 +00:00
return true ;
case Protocol : : Server : : Progress :
onProgress ( packet . progress ) ;
return true ;
case Protocol : : Server : : ProfileInfo :
onProfileInfo ( packet . profile_info ) ;
return true ;
case Protocol : : Server : : Totals :
2020-06-01 18:58:38 +00:00
if ( ! cancelled )
onTotals ( packet . block ) ;
2017-04-01 07:20:54 +00:00
return true ;
case Protocol : : Server : : Extremes :
2020-06-01 18:58:38 +00:00
if ( ! cancelled )
onExtremes ( packet . block ) ;
2017-04-01 07:20:54 +00:00
return true ;
case Protocol : : Server : : Exception :
2020-06-26 01:34:06 +00:00
onReceiveExceptionFromServer ( * packet . exception ) ;
last_exception_received_from_server = std : : move ( packet . exception ) ;
2017-04-01 07:20:54 +00:00
return false ;
2018-06-06 20:57:07 +00:00
case Protocol : : Server : : Log :
onLogData ( packet . block ) ;
return true ;
2017-04-01 07:20:54 +00:00
case Protocol : : Server : : EndOfStream :
onEndOfStream ( ) ;
return false ;
default :
throw Exception ( " Unknown packet from server " , ErrorCodes : : UNKNOWN_PACKET_FROM_SERVER ) ;
}
}
/// Receive the block that serves as an example of the structure of table where data will be inserted.
2018-12-04 20:03:04 +00:00
bool receiveSampleBlock ( Block & out , ColumnsDescription & columns_description )
2017-04-01 07:20:54 +00:00
{
2018-06-08 19:50:15 +00:00
while ( true )
{
2019-11-15 16:34:43 +00:00
Packet packet = connection - > receivePacket ( ) ;
2017-04-01 07:20:54 +00:00
2018-06-08 19:50:15 +00:00
switch ( packet . type )
{
case Protocol : : Server : : Data :
out = packet . block ;
return true ;
case Protocol : : Server : : Exception :
2020-06-26 01:34:06 +00:00
onReceiveExceptionFromServer ( * packet . exception ) ;
last_exception_received_from_server = std : : move ( packet . exception ) ;
2018-06-08 19:50:15 +00:00
return false ;
case Protocol : : Server : : Log :
onLogData ( packet . block ) ;
break ;
2018-12-04 20:03:04 +00:00
case Protocol : : Server : : TableColumns :
columns_description = ColumnsDescription : : parse ( packet . multistring_message [ 1 ] ) ;
return receiveSampleBlock ( out , columns_description ) ;
2018-06-08 19:50:15 +00:00
default :
throw NetException ( " Unexpected packet from server (expected Data, Exception or Log, got "
+ String ( Protocol : : Server : : toString ( packet . type ) ) + " ) " , ErrorCodes : : UNEXPECTED_PACKET_FROM_SERVER ) ;
}
}
}
2019-01-22 19:56:53 +00:00
/// Process Log packets, exit when receive Exception or EndOfStream
2018-06-08 19:50:15 +00:00
bool receiveEndOfQuery ( )
{
while ( true )
2017-04-01 07:20:54 +00:00
{
2019-11-15 16:34:43 +00:00
Packet packet = connection - > receivePacket ( ) ;
2017-04-01 07:20:54 +00:00
2018-06-08 19:50:15 +00:00
switch ( packet . type )
{
case Protocol : : Server : : EndOfStream :
onEndOfStream ( ) ;
return true ;
2017-04-01 07:20:54 +00:00
2018-06-08 19:50:15 +00:00
case Protocol : : Server : : Exception :
2020-06-26 01:34:06 +00:00
onReceiveExceptionFromServer ( * packet . exception ) ;
last_exception_received_from_server = std : : move ( packet . exception ) ;
2018-06-08 19:50:15 +00:00
return false ;
2018-07-06 15:49:33 +00:00
2018-06-08 19:50:15 +00:00
case Protocol : : Server : : Log :
onLogData ( packet . block ) ;
break ;
default :
throw NetException ( " Unexpected packet from server (expected Exception, EndOfStream or Log, got "
+ String ( Protocol : : Server : : toString ( packet . type ) ) + " ) " , ErrorCodes : : UNEXPECTED_PACKET_FROM_SERVER ) ;
}
2017-04-01 07:20:54 +00:00
}
}
2019-08-29 17:30:16 +00:00
/// Process Log packets, used when inserting data by blocks
void receiveLogs ( )
{
auto packet_type = connection - > checkPacket ( ) ;
while ( packet_type & & * packet_type = = Protocol : : Server : : Log )
{
2020-06-01 18:58:38 +00:00
receiveAndProcessPacket ( false ) ;
2019-08-29 17:30:16 +00:00
packet_type = connection - > checkPacket ( ) ;
}
}
2017-04-01 07:20:54 +00:00
void initBlockOutputStream ( const Block & block )
{
if ( ! block_out_stream )
{
WriteBuffer * out_buf = nullptr ;
String pager = config ( ) . getString ( " pager " , " " ) ;
if ( ! pager . empty ( ) )
{
2017-12-10 14:57:08 +00:00
signal ( SIGPIPE , SIG_IGN ) ;
2017-04-01 07:20:54 +00:00
pager_cmd = ShellCommand : : execute ( pager , true ) ;
out_buf = & pager_cmd - > in ;
}
else
{
out_buf = & std_out ;
}
String current_format = format ;
/// The query can specify output format or output file.
2019-03-12 12:41:57 +00:00
/// FIXME: try to prettify this cast using `as<>()`
if ( const auto * query_with_output = dynamic_cast < const ASTQueryWithOutput * > ( parsed_query . get ( ) ) )
2017-04-01 07:20:54 +00:00
{
2019-03-08 09:45:34 +00:00
if ( query_with_output - > out_file )
2017-04-01 07:20:54 +00:00
{
2019-03-15 13:12:11 +00:00
const auto & out_file_node = query_with_output - > out_file - > as < ASTLiteral & > ( ) ;
2017-04-01 07:20:54 +00:00
const auto & out_file = out_file_node . value . safeGet < std : : string > ( ) ;
2019-03-08 09:45:34 +00:00
2017-04-01 07:20:54 +00:00
out_file_buf . emplace ( out_file , DBMS_DEFAULT_BUFFER_SIZE , O_WRONLY | O_EXCL | O_CREAT ) ;
2017-10-04 00:22:00 +00:00
out_buf = & * out_file_buf ;
2017-04-01 07:20:54 +00:00
// We are writing to file, so default format is the same as in non-interactive mode.
if ( is_interactive & & is_default_format )
current_format = " TabSeparated " ;
}
if ( query_with_output - > format ! = nullptr )
{
if ( has_vertical_output_suffix )
throw Exception ( " Output format already specified " , ErrorCodes : : CLIENT_OUTPUT_FORMAT_SPECIFIED ) ;
2019-03-15 13:12:11 +00:00
const auto & id = query_with_output - > format - > as < ASTIdentifier & > ( ) ;
2017-04-01 07:20:54 +00:00
current_format = id . name ;
}
}
if ( has_vertical_output_suffix )
current_format = " Vertical " ;
block_out_stream = context . getOutputFormat ( current_format , * out_buf , block ) ;
block_out_stream - > writePrefix ( ) ;
}
}
2018-06-06 20:57:07 +00:00
void initLogsOutputStream ( )
{
if ( ! logs_out_stream )
{
2018-06-07 18:46:53 +00:00
WriteBuffer * wb = out_logs_buf . get ( ) ;
if ( ! out_logs_buf )
{
if ( server_logs_file . empty ( ) )
{
/// Use stderr by default
out_logs_buf = std : : make_unique < WriteBufferFromFileDescriptor > ( STDERR_FILENO ) ;
wb = out_logs_buf . get ( ) ;
}
else if ( server_logs_file = = " - " )
{
/// Use stdout if --server_logs_file=- specified
wb = & std_out ;
}
else
{
2020-06-22 20:31:44 +00:00
out_logs_buf = std : : make_unique < WriteBufferFromFile > (
server_logs_file , DBMS_DEFAULT_BUFFER_SIZE , O_WRONLY | O_APPEND | O_CREAT ) ;
2018-06-07 18:46:53 +00:00
wb = out_logs_buf . get ( ) ;
}
}
2020-02-21 20:01:38 +00:00
logs_out_stream = std : : make_shared < InternalTextLogsRowOutputStream > ( * wb , stdout_is_a_tty ) ;
2018-06-06 20:57:07 +00:00
logs_out_stream - > writePrefix ( ) ;
}
}
2017-04-01 07:20:54 +00:00
void onData ( Block & block )
{
if ( written_progress_chars )
clearProgress ( ) ;
if ( ! block )
return ;
processed_rows + = block . rows ( ) ;
initBlockOutputStream ( block ) ;
2020-07-03 12:52:16 +00:00
/// The header block containing zero rows was used to initialize
/// block_out_stream, do not output it.
/// Also do not output too much data if we're fuzzing.
if ( block . rows ( ) ! = 0
& & ( query_fuzzer_runs = = 0 | | processed_rows < 100 ) )
2017-04-01 07:20:54 +00:00
{
block_out_stream - > write ( block ) ;
written_first_block = true ;
}
/// Received data block is immediately displayed to the user.
block_out_stream - > flush ( ) ;
2019-03-26 14:36:56 +00:00
/// Restore progress bar after data block.
writeProgress ( ) ;
2017-04-01 07:20:54 +00:00
}
2018-06-06 20:57:07 +00:00
void onLogData ( Block & block )
{
2020-06-22 19:16:48 +00:00
has_received_logs = true ;
2018-06-06 20:57:07 +00:00
initLogsOutputStream ( ) ;
logs_out_stream - > write ( block ) ;
logs_out_stream - > flush ( ) ;
}
2017-04-01 07:20:54 +00:00
void onTotals ( Block & block )
{
initBlockOutputStream ( block ) ;
block_out_stream - > setTotals ( block ) ;
}
void onExtremes ( Block & block )
{
initBlockOutputStream ( block ) ;
block_out_stream - > setExtremes ( block ) ;
}
void onProgress ( const Progress & value )
{
2019-02-07 13:47:16 +00:00
if ( ! progress . incrementPiecewiseAtomically ( value ) )
{
// Just a keep-alive update.
return ;
}
2018-01-10 00:04:08 +00:00
if ( block_out_stream )
block_out_stream - > onProgress ( value ) ;
2017-04-01 07:20:54 +00:00
writeProgress ( ) ;
}
void clearProgress ( )
{
written_progress_chars = 0 ;
2020-06-22 19:16:48 +00:00
if ( ! has_received_logs )
2020-04-11 19:51:04 +00:00
std : : cerr < < " \r " CLEAR_TO_END_OF_LINE ;
2017-04-01 07:20:54 +00:00
}
void writeProgress ( )
{
if ( ! need_render_progress )
return ;
2019-03-26 14:36:56 +00:00
/// Output all progress bar commands to stderr at once to avoid flicker.
WriteBufferFromFileDescriptor message ( STDERR_FILENO , 1024 ) ;
2017-04-01 07:20:54 +00:00
static size_t increment = 0 ;
static const char * indicators [ 8 ] =
{
" \033 [1;30m→ \033 [0m " ,
" \033 [1;31m↘ \033 [0m " ,
" \033 [1;32m↓ \033 [0m " ,
" \033 [1;33m↙ \033 [0m " ,
" \033 [1;34m← \033 [0m " ,
" \033 [1;35m↖ \033 [0m " ,
" \033 [1;36m↑ \033 [0m " ,
" \033 [1m↗ \033 [0m " ,
} ;
2020-05-18 08:08:55 +00:00
const char * indicator = indicators [ increment % 8 ] ;
2019-03-26 14:36:56 +00:00
2020-06-22 19:16:48 +00:00
if ( ! has_received_logs & & written_progress_chars )
2020-04-11 19:51:04 +00:00
message < < ' \r ' ;
2019-03-26 14:36:56 +00:00
size_t prefix_size = message . count ( ) ;
2017-04-01 07:20:54 +00:00
2020-04-11 19:51:04 +00:00
message < < indicator < < " Progress: " ;
2017-04-01 07:20:54 +00:00
message
2019-05-20 13:24:53 +00:00
< < formatReadableQuantity ( progress . read_rows ) < < " rows, "
< < formatReadableSizeWithDecimalSuffix ( progress . read_bytes ) ;
2017-04-01 07:20:54 +00:00
size_t elapsed_ns = watch . elapsed ( ) ;
if ( elapsed_ns )
message < < " ( "
2019-05-20 13:24:53 +00:00
< < formatReadableQuantity ( progress . read_rows * 1000000000.0 / elapsed_ns ) < < " rows/s., "
< < formatReadableSizeWithDecimalSuffix ( progress . read_bytes * 1000000000.0 / elapsed_ns ) < < " /s.) " ;
2017-04-01 07:20:54 +00:00
else
message < < " . " ;
2020-04-11 19:51:04 +00:00
written_progress_chars = message . count ( ) - prefix_size - ( strlen ( indicator ) - 2 ) ; /// Don't count invisible output (escape sequences).
2017-04-01 07:20:54 +00:00
2017-04-12 19:50:49 +00:00
/// If the approximate number of rows to process is known, we can display a progress bar and percentage.
2019-05-20 13:24:53 +00:00
if ( progress . total_rows_to_read > 0 )
2017-04-01 07:20:54 +00:00
{
2019-05-20 13:24:53 +00:00
size_t total_rows_corrected = std : : max ( progress . read_rows , progress . total_rows_to_read ) ;
2017-04-16 04:28:04 +00:00
2017-04-12 19:50:49 +00:00
/// To avoid flicker, display progress bar only if .5 seconds have passed since query execution start
/// and the query is less than halfway done.
if ( elapsed_ns > 500000000 )
{
/// Trigger to start displaying progress bar. If query is mostly done, don't display it.
2019-05-20 13:24:53 +00:00
if ( progress . read_rows * 2 < total_rows_corrected )
2017-04-12 19:50:49 +00:00
show_progress_bar = true ;
if ( show_progress_bar )
{
2020-04-11 19:51:04 +00:00
ssize_t width_of_progress_bar = static_cast < ssize_t > ( getTerminalWidth ( ) ) - written_progress_chars - strlen ( " 99% " ) ;
2017-04-12 19:50:49 +00:00
if ( width_of_progress_bar > 0 )
{
2019-05-20 13:24:53 +00:00
std : : string bar = UnicodeBar : : render ( UnicodeBar : : getWidth ( progress . read_rows , 0 , total_rows_corrected , width_of_progress_bar ) ) ;
2019-03-26 14:36:56 +00:00
message < < " \033 [0;32m " < < bar < < " \033 [0m " ;
2017-04-12 19:50:49 +00:00
if ( width_of_progress_bar > static_cast < ssize_t > ( bar . size ( ) / UNICODE_BAR_CHAR_SIZE ) )
2019-03-26 14:36:56 +00:00
message < < std : : string ( width_of_progress_bar - bar . size ( ) / UNICODE_BAR_CHAR_SIZE , ' ' ) ;
2017-04-12 19:50:49 +00:00
}
}
}
/// Underestimate percentage a bit to avoid displaying 100%.
2019-05-20 13:24:53 +00:00
message < < ' ' < < ( 99 * progress . read_rows / total_rows_corrected ) < < ' % ' ;
2017-04-01 07:20:54 +00:00
}
2017-04-12 19:50:49 +00:00
2020-04-11 19:51:04 +00:00
message < < CLEAR_TO_END_OF_LINE ;
2020-06-22 19:16:48 +00:00
if ( has_received_logs )
2020-03-12 18:35:25 +00:00
message < < ' \n ' ;
2017-04-01 07:20:54 +00:00
+ + increment ;
2019-03-26 14:36:56 +00:00
message . next ( ) ;
2017-04-01 07:20:54 +00:00
}
void writeFinalProgress ( )
{
std : : cout < < " Processed "
2019-05-20 13:24:53 +00:00
< < formatReadableQuantity ( progress . read_rows ) < < " rows, "
< < formatReadableSizeWithDecimalSuffix ( progress . read_bytes ) ;
2017-04-01 07:20:54 +00:00
size_t elapsed_ns = watch . elapsed ( ) ;
if ( elapsed_ns )
std : : cout < < " ( "
2019-05-20 13:24:53 +00:00
< < formatReadableQuantity ( progress . read_rows * 1000000000.0 / elapsed_ns ) < < " rows/s., "
< < formatReadableSizeWithDecimalSuffix ( progress . read_bytes * 1000000000.0 / elapsed_ns ) < < " /s.) " ;
2017-04-01 07:20:54 +00:00
else
std : : cout < < " . " ;
}
2020-06-26 01:34:06 +00:00
void onReceiveExceptionFromServer ( const Exception & e )
2017-04-01 07:20:54 +00:00
{
resetOutput ( ) ;
2020-06-26 01:34:06 +00:00
received_exception_from_server = true ;
2017-04-01 07:20:54 +00:00
2018-07-27 17:19:22 +00:00
actual_server_error = e . code ( ) ;
if ( expected_server_error )
{
if ( actual_server_error = = expected_server_error )
return ;
std : : cerr < < " Expected error code: " < < expected_server_error < < " but got: " < < actual_server_error < < " . " < < std : : endl ;
}
2017-04-01 07:20:54 +00:00
std : : string text = e . displayText ( ) ;
auto embedded_stack_trace_pos = text . find ( " Stack trace " ) ;
if ( std : : string : : npos ! = embedded_stack_trace_pos & & ! config ( ) . getBool ( " stacktrace " , false ) )
text . resize ( embedded_stack_trace_pos ) ;
2020-05-30 16:39:34 +00:00
/// If we probably have progress bar, we should add additional newline,
/// otherwise exception may display concatenated with the progress bar.
if ( need_render_progress )
std : : cerr < < ' \n ' ;
std : : cerr < < " Received exception from server (version " < < server_version < < " ): " < < std : : endl
2017-04-01 07:20:54 +00:00
< < " Code: " < < e . code ( ) < < " . " < < text < < std : : endl ;
}
void onProfileInfo ( const BlockStreamProfileInfo & profile_info )
{
if ( profile_info . hasAppliedLimit ( ) & & block_out_stream )
block_out_stream - > setRowsBeforeLimit ( profile_info . getRowsBeforeLimit ( ) ) ;
}
void onEndOfStream ( )
{
if ( block_out_stream )
block_out_stream - > writeSuffix ( ) ;
2018-06-06 20:57:07 +00:00
if ( logs_out_stream )
logs_out_stream - > writeSuffix ( ) ;
2017-04-01 07:20:54 +00:00
resetOutput ( ) ;
if ( is_interactive & & ! written_first_block )
2020-04-11 19:51:04 +00:00
{
if ( written_progress_chars )
clearProgress ( ) ;
2017-04-01 07:20:54 +00:00
std : : cout < < " Ok. " < < std : : endl ;
2020-04-11 19:51:04 +00:00
}
2017-04-01 07:20:54 +00:00
}
2020-03-18 00:57:00 +00:00
static void showClientVersion ( )
2017-04-01 07:20:54 +00:00
{
2019-04-03 14:06:59 +00:00
std : : cout < < DBMS_NAME < < " client version " < < VERSION_STRING < < VERSION_OFFICIAL < < " . " < < std : : endl ;
2017-04-01 07:20:54 +00:00
}
2017-03-13 17:35:17 +00:00
public :
2017-04-01 07:20:54 +00:00
void init ( int argc , char * * argv )
{
/// Don't parse options with Poco library. We need more sophisticated processing.
stopOptionsProcessing ( ) ;
/** We allow different groups of arguments:
* - common arguments ;
* - arguments for any number of external tables each in form " --external args... " ,
2019-05-18 21:07:23 +00:00
* where possible args are file , name , format , structure , types ;
* - param arguments for prepared statements .
2017-04-01 07:20:54 +00:00
* Split these groups before processing .
*/
2019-06-14 14:00:37 +00:00
using Arguments = std : : vector < std : : string > ;
2017-04-01 07:20:54 +00:00
Arguments common_arguments { " " } ; /// 0th argument is ignored.
std : : vector < Arguments > external_tables_arguments ;
bool in_external_group = false ;
for ( int arg_num = 1 ; arg_num < argc ; + + arg_num )
{
const char * arg = argv [ arg_num ] ;
if ( 0 = = strcmp ( arg , " --external " ) )
{
in_external_group = true ;
external_tables_arguments . emplace_back ( Arguments { " " } ) ;
}
/// Options with value after equal sign.
else if ( in_external_group
2017-07-12 00:49:30 +00:00
& & ( 0 = = strncmp ( arg , " --file= " , strlen ( " --file= " ) )
| | 0 = = strncmp ( arg , " --name= " , strlen ( " --name= " ) )
| | 0 = = strncmp ( arg , " --format= " , strlen ( " --format= " ) )
| | 0 = = strncmp ( arg , " --structure= " , strlen ( " --structure= " ) )
| | 0 = = strncmp ( arg , " --types= " , strlen ( " --types= " ) ) ) )
2017-04-01 07:20:54 +00:00
{
external_tables_arguments . back ( ) . emplace_back ( arg ) ;
}
/// Options with value after whitespace.
else if ( in_external_group
& & ( 0 = = strcmp ( arg , " --file " )
| | 0 = = strcmp ( arg , " --name " )
| | 0 = = strcmp ( arg , " --format " )
| | 0 = = strcmp ( arg , " --structure " )
| | 0 = = strcmp ( arg , " --types " ) ) )
{
if ( arg_num + 1 < argc )
{
external_tables_arguments . back ( ) . emplace_back ( arg ) ;
+ + arg_num ;
arg = argv [ arg_num ] ;
external_tables_arguments . back ( ) . emplace_back ( arg ) ;
}
else
break ;
}
else
{
in_external_group = false ;
2019-05-18 21:07:23 +00:00
/// Parameter arg after underline.
if ( startsWith ( arg , " --param_ " ) )
2019-06-15 18:56:32 +00:00
{
const char * param_continuation = arg + strlen ( " --param_ " ) ;
const char * equal_pos = strchr ( param_continuation , ' = ' ) ;
if ( equal_pos = = param_continuation )
throw Exception ( " Parameter name cannot be empty " , ErrorCodes : : BAD_ARGUMENTS ) ;
if ( equal_pos )
{
/// param_name=value
query_parameters . emplace ( String ( param_continuation , equal_pos ) , String ( equal_pos + 1 ) ) ;
}
else
{
/// param_name value
+ + arg_num ;
arg = argv [ arg_num ] ;
query_parameters . emplace ( String ( param_continuation ) , String ( arg ) ) ;
}
}
2019-05-18 21:07:23 +00:00
else
common_arguments . emplace_back ( arg ) ;
2017-04-01 07:20:54 +00:00
}
}
2017-03-13 17:35:17 +00:00
2020-02-21 20:01:38 +00:00
stdin_is_a_tty = isatty ( STDIN_FILENO ) ;
stdout_is_a_tty = isatty ( STDOUT_FILENO ) ;
2018-04-20 19:31:19 +00:00
2020-04-11 19:51:04 +00:00
uint64_t terminal_width = 0 ;
2020-02-21 20:01:38 +00:00
if ( stdin_is_a_tty )
2019-08-23 13:19:12 +00:00
terminal_width = getTerminalWidth ( ) ;
2018-05-23 16:12:46 +00:00
2019-08-22 14:03:37 +00:00
namespace po = boost : : program_options ;
2018-05-23 16:12:46 +00:00
2017-04-01 07:20:54 +00:00
/// Main commandline options related to client functionality and all parameters from Settings.
2019-08-23 15:47:27 +00:00
po : : options_description main_description = createOptionsDescription ( " Main options " , terminal_width ) ;
2017-04-01 07:20:54 +00:00
main_description . add_options ( )
( " help " , " produce help message " )
2019-02-02 14:40:29 +00:00
( " config-file,C " , po : : value < std : : string > ( ) , " config-file path " )
( " config,c " , po : : value < std : : string > ( ) , " config-file path (another shorthand) " )
2018-06-06 20:57:07 +00:00
( " host,h " , po : : value < std : : string > ( ) - > default_value ( " localhost " ) , " server host " )
( " port " , po : : value < int > ( ) - > default_value ( 9000 ) , " server port " )
2018-08-25 13:41:50 +00:00
( " secure,s " , " Use TLS connection " )
2018-06-06 20:57:07 +00:00
( " user,u " , po : : value < std : : string > ( ) - > default_value ( " default " ) , " user " )
2019-02-02 13:43:08 +00:00
/** If "--password [value]" is used but the value is omitted, the bad argument exception will be thrown.
* implicit_value is used to avoid this exception ( to allow user to type just " --password " )
* Since currently boost provides no way to check if a value has been set implicitly for an option ,
* the " \n " is used to distinguish this case because there is hardly a chance an user would use " \n "
* as the password .
*/
2019-08-23 20:35:36 +00:00
( " password " , po : : value < std : : string > ( ) - > implicit_value ( " \n " , " " ) , " password " )
2018-05-21 17:27:18 +00:00
( " ask-password " , " ask-password " )
2020-04-15 01:12:32 +00:00
( " quota_key " , po : : value < std : : string > ( ) , " A string to differentiate quotas when the user have keyed quotas configured on server " )
2018-06-06 20:57:07 +00:00
( " query_id " , po : : value < std : : string > ( ) , " query_id " )
( " query,q " , po : : value < std : : string > ( ) , " query " )
( " database,d " , po : : value < std : : string > ( ) , " database " )
( " pager " , po : : value < std : : string > ( ) , " pager " )
2018-08-25 16:19:00 +00:00
( " disable_suggestion,A " , " Disable loading suggestion data. Note that suggestion data is loaded asynchronously through a second connection to ClickHouse server. Also it is reasonable to disable suggestion if you want to paste a query with TAB characters. Shorthand option -A is for those who get used to mysql client. " )
2018-08-25 13:41:50 +00:00
( " suggestion_limit " , po : : value < int > ( ) - > default_value ( 10000 ) ,
" Suggestion limit for how many databases, tables and columns to fetch. " )
2017-07-12 00:49:30 +00:00
( " multiline,m " , " multiline " )
2017-09-29 22:21:17 +00:00
( " multiquery,n " , " multiquery " )
2018-06-06 20:57:07 +00:00
( " format,f " , po : : value < std : : string > ( ) , " default output format " )
2018-07-27 17:19:22 +00:00
( " testmode,T " , " enable test hints in comments " )
( " ignore-error " , " do not stop processing in multiquery mode " )
2017-07-12 00:49:30 +00:00
( " vertical,E " , " vertical output format, same as --format=Vertical or FORMAT Vertical or \\ G at end of command " )
( " time,t " , " print query execution time to stderr in non-interactive mode (for benchmarks) " )
( " stacktrace " , " print stack traces of exceptions " )
( " progress " , " print progress even in non-interactive mode " )
( " version,V " , " print version information and exit " )
2018-08-10 19:53:49 +00:00
( " version-clean " , " print version in machine-readable format and exit " )
2017-07-12 00:49:30 +00:00
( " echo " , " in batch mode, print query before execution " )
2018-06-06 20:57:07 +00:00
( " max_client_network_bandwidth " , po : : value < int > ( ) , " the maximum speed of data exchange over the network for the client in bytes per second. " )
( " compression " , po : : value < bool > ( ) , " enable or disable compression " )
2020-06-06 16:59:16 +00:00
( " highlight " , po : : value < bool > ( ) - > default_value ( true ) , " enable or disable basic syntax highlight in interactive command line " )
2018-06-09 15:29:08 +00:00
( " log-level " , po : : value < std : : string > ( ) , " client log level " )
2018-06-07 18:46:53 +00:00
( " server_logs_file " , po : : value < std : : string > ( ) , " put server logs into specified file " )
2020-06-26 05:44:42 +00:00
( " query-fuzzer-runs " , po : : value < int > ( ) - > default_value ( 0 ) , " query fuzzer runs " )
2017-04-01 07:20:54 +00:00
;
2019-04-25 14:08:20 +00:00
2020-03-13 14:50:26 +00:00
Settings cmd_settings ;
cmd_settings . addProgramOptions ( main_description ) ;
2017-03-13 17:35:17 +00:00
2017-04-01 07:20:54 +00:00
/// Commandline options related to external tables.
2019-08-23 15:47:27 +00:00
po : : options_description external_description = createOptionsDescription ( " External tables options " , terminal_width ) ;
2017-04-01 07:20:54 +00:00
external_description . add_options ( )
2018-06-06 20:57:07 +00:00
( " file " , po : : value < std : : string > ( ) , " data file or - for stdin " )
( " name " , po : : value < std : : string > ( ) - > default_value ( " _data " ) , " name of the table " )
( " format " , po : : value < std : : string > ( ) - > default_value ( " TabSeparated " ) , " data format " )
( " structure " , po : : value < std : : string > ( ) , " structure " )
( " types " , po : : value < std : : string > ( ) , " types " )
2017-04-01 07:20:54 +00:00
;
2019-05-18 21:07:23 +00:00
2017-04-01 07:20:54 +00:00
/// Parse main commandline options.
2019-06-14 14:00:37 +00:00
po : : parsed_options parsed = po : : command_line_parser ( common_arguments ) . options ( main_description ) . run ( ) ;
2020-06-24 16:34:49 +00:00
auto unrecognized_options = po : : collect_unrecognized ( parsed . options , po : : collect_unrecognized_mode : : include_positional ) ;
2020-06-25 10:10:27 +00:00
// unrecognized_options[0] is "", I don't understand why we need "" as the first argument which unused
2020-06-24 16:34:49 +00:00
if ( unrecognized_options . size ( ) > 1 )
{
throw Exception ( " Unrecognized option ' " + unrecognized_options [ 1 ] + " ' " , ErrorCodes : : UNRECOGNIZED_ARGUMENTS ) ;
}
2018-06-06 20:57:07 +00:00
po : : variables_map options ;
po : : store ( parsed , options ) ;
2019-04-25 14:08:20 +00:00
po : : notify ( options ) ;
2017-04-01 07:20:54 +00:00
if ( options . count ( " version " ) | | options . count ( " V " ) )
{
showClientVersion ( ) ;
exit ( 0 ) ;
}
2018-08-10 19:53:49 +00:00
if ( options . count ( " version-clean " ) )
{
std : : cout < < VERSION_STRING ;
exit ( 0 ) ;
}
2017-04-01 07:20:54 +00:00
/// Output of help message.
if ( options . count ( " help " )
| | ( options . count ( " host " ) & & options [ " host " ] . as < std : : string > ( ) = = " elp " ) ) /// If user writes -help instead of --help.
{
std : : cout < < main_description < < " \n " ;
std : : cout < < external_description < < " \n " ;
2019-06-15 18:56:32 +00:00
std : : cout < < " In addition, --param_name=value can be specified for substitution of parameters for parametrized queries. \n " ;
2017-04-01 07:20:54 +00:00
exit ( 0 ) ;
}
2018-06-06 20:57:07 +00:00
if ( options . count ( " log-level " ) )
Poco : : Logger : : root ( ) . setLevel ( options [ " log-level " ] . as < std : : string > ( ) ) ;
2017-04-01 07:20:54 +00:00
size_t number_of_external_tables_with_stdin_source = 0 ;
for ( size_t i = 0 ; i < external_tables_arguments . size ( ) ; + + i )
{
/// Parse commandline options related to external tables.
2019-06-14 14:00:37 +00:00
po : : parsed_options parsed_tables = po : : command_line_parser ( external_tables_arguments [ i ] ) . options ( external_description ) . run ( ) ;
2018-06-06 20:57:07 +00:00
po : : variables_map external_options ;
2019-01-04 13:32:08 +00:00
po : : store ( parsed_tables , external_options ) ;
2017-04-01 07:20:54 +00:00
try
{
external_tables . emplace_back ( external_options ) ;
if ( external_tables . back ( ) . file = = " - " )
+ + number_of_external_tables_with_stdin_source ;
if ( number_of_external_tables_with_stdin_source > 1 )
throw Exception ( " Two or more external tables has stdin (-) set as - - file field " , ErrorCodes::BAD_ARGUMENTS) ;
}
catch ( const Exception & e )
{
std : : string text = e . displayText ( ) ;
std : : cerr < < " Code: " < < e . code ( ) < < " . " < < text < < std : : endl ;
std : : cerr < < " Table № " < < i < < std : : endl < < std : : endl ;
2020-06-11 17:57:03 +00:00
/// Avoid the case when error exit code can possibly overflow to normal (zero).
2020-06-11 13:08:24 +00:00
auto exit_code = e . code ( ) % 256 ;
if ( exit_code = = 0 )
exit_code = 255 ;
exit ( exit_code ) ;
2017-04-01 07:20:54 +00:00
}
}
2020-03-13 14:50:26 +00:00
context . makeGlobalContext ( ) ;
context . setSettings ( cmd_settings ) ;
2019-04-25 14:08:20 +00:00
/// Copy settings-related program options to config.
/// TODO: Is this code necessary?
for ( const auto & setting : context . getSettingsRef ( ) )
{
const String name = setting . getName ( ) . toString ( ) ;
if ( options . count ( name ) )
config ( ) . setString ( name , options [ name ] . as < std : : string > ( ) ) ;
2018-11-06 11:23:19 +00:00
}
2017-03-13 17:35:17 +00:00
2019-02-02 14:40:29 +00:00
if ( options . count ( " config-file " ) & & options . count ( " config " ) )
2019-02-02 14:43:36 +00:00
throw Exception ( " Two or more configuration files referenced in arguments " , ErrorCodes : : BAD_ARGUMENTS ) ;
2019-02-02 14:40:29 +00:00
2017-04-01 07:20:54 +00:00
/// Save received data into the internal config.
2019-02-02 14:29:54 +00:00
if ( options . count ( " config-file " ) )
2017-04-01 07:20:54 +00:00
config ( ) . setString ( " config-file " , options [ " config-file " ] . as < std : : string > ( ) ) ;
2019-02-02 14:40:29 +00:00
if ( options . count ( " config " ) )
config ( ) . setString ( " config-file " , options [ " config " ] . as < std : : string > ( ) ) ;
2017-04-01 07:20:54 +00:00
if ( options . count ( " host " ) & & ! options [ " host " ] . defaulted ( ) )
config ( ) . setString ( " host " , options [ " host " ] . as < std : : string > ( ) ) ;
2018-03-24 23:22:58 +00:00
if ( options . count ( " query_id " ) )
config ( ) . setString ( " query_id " , options [ " query_id " ] . as < std : : string > ( ) ) ;
2017-04-01 07:20:54 +00:00
if ( options . count ( " query " ) )
config ( ) . setString ( " query " , options [ " query " ] . as < std : : string > ( ) ) ;
if ( options . count ( " database " ) )
config ( ) . setString ( " database " , options [ " database " ] . as < std : : string > ( ) ) ;
if ( options . count ( " pager " ) )
config ( ) . setString ( " pager " , options [ " pager " ] . as < std : : string > ( ) ) ;
if ( options . count ( " port " ) & & ! options [ " port " ] . defaulted ( ) )
config ( ) . setInt ( " port " , options [ " port " ] . as < int > ( ) ) ;
2018-03-29 01:41:06 +00:00
if ( options . count ( " secure " ) )
config ( ) . setBool ( " secure " , true ) ;
2018-08-21 10:04:27 +00:00
if ( options . count ( " user " ) & & ! options [ " user " ] . defaulted ( ) )
2017-04-01 07:20:54 +00:00
config ( ) . setString ( " user " , options [ " user " ] . as < std : : string > ( ) ) ;
if ( options . count ( " password " ) )
config ( ) . setString ( " password " , options [ " password " ] . as < std : : string > ( ) ) ;
2018-05-21 17:27:18 +00:00
if ( options . count ( " ask-password " ) )
config ( ) . setBool ( " ask-password " , true ) ;
2020-04-15 01:12:32 +00:00
if ( options . count ( " quota_key " ) )
config ( ) . setString ( " quota_key " , options [ " quota_key " ] . as < std : : string > ( ) ) ;
2017-04-01 07:20:54 +00:00
if ( options . count ( " multiline " ) )
config ( ) . setBool ( " multiline " , true ) ;
if ( options . count ( " multiquery " ) )
config ( ) . setBool ( " multiquery " , true ) ;
2018-07-27 17:19:22 +00:00
if ( options . count ( " testmode " ) )
config ( ) . setBool ( " testmode " , true ) ;
2017-12-22 18:29:03 +00:00
if ( options . count ( " ignore-error " ) )
config ( ) . setBool ( " ignore-error " , true ) ;
2017-04-01 07:20:54 +00:00
if ( options . count ( " format " ) )
config ( ) . setString ( " format " , options [ " format " ] . as < std : : string > ( ) ) ;
if ( options . count ( " vertical " ) )
config ( ) . setBool ( " vertical " , true ) ;
if ( options . count ( " stacktrace " ) )
config ( ) . setBool ( " stacktrace " , true ) ;
if ( options . count ( " progress " ) )
config ( ) . setBool ( " progress " , true ) ;
if ( options . count ( " echo " ) )
config ( ) . setBool ( " echo " , true ) ;
if ( options . count ( " time " ) )
print_time_to_stderr = true ;
2017-12-13 20:53:53 +00:00
if ( options . count ( " max_client_network_bandwidth " ) )
2017-12-13 21:26:17 +00:00
max_client_network_bandwidth = options [ " max_client_network_bandwidth " ] . as < int > ( ) ;
2017-04-01 07:20:54 +00:00
if ( options . count ( " compression " ) )
config ( ) . setBool ( " compression " , options [ " compression " ] . as < bool > ( ) ) ;
2018-06-07 18:46:53 +00:00
if ( options . count ( " server_logs_file " ) )
server_logs_file = options [ " server_logs_file " ] . as < std : : string > ( ) ;
2018-08-25 13:41:50 +00:00
if ( options . count ( " disable_suggestion " ) )
config ( ) . setBool ( " disable_suggestion " , true ) ;
2018-06-06 16:37:04 +00:00
if ( options . count ( " suggestion_limit " ) )
config ( ) . setInt ( " suggestion_limit " , options [ " suggestion_limit " ] . as < int > ( ) ) ;
2020-06-06 16:59:16 +00:00
if ( options . count ( " highlight " ) )
config ( ) . setBool ( " highlight " , options [ " highlight " ] . as < bool > ( ) ) ;
2019-06-14 14:00:37 +00:00
2020-06-26 05:44:42 +00:00
if ( ( query_fuzzer_runs = options [ " query-fuzzer-runs " ] . as < int > ( ) ) )
{
2020-06-30 09:25:23 +00:00
// Fuzzer implies multiquery.
2020-06-26 05:44:42 +00:00
config ( ) . setBool ( " multiquery " , true ) ;
2020-06-30 09:25:23 +00:00
// Ignore errors in parsing queries.
// TODO stop using parseQuery.
config ( ) . setBool ( " ignore-error " , true ) ;
ignore_error = true ;
2020-06-26 05:44:42 +00:00
}
2019-06-14 14:00:37 +00:00
argsToConfig ( common_arguments , config ( ) , 100 ) ;
2020-06-14 19:23:05 +00:00
clearPasswordFromCommandLine ( argc , argv ) ;
2017-04-01 07:20:54 +00:00
}
2017-03-13 17:35:17 +00:00
} ;
}
2020-06-30 09:25:23 +00:00
# pragma GCC diagnostic ignored "-Wunused-function"
# pragma GCC diagnostic ignored "-Wmissing-declarations"
2017-03-24 15:05:54 +00:00
int mainEntryClickHouseClient ( int argc , char * * argv )
2017-03-13 17:35:17 +00:00
{
2017-04-01 07:20:54 +00:00
try
{
2019-02-02 13:49:24 +00:00
DB : : Client client ;
2017-04-01 07:20:54 +00:00
client . init ( argc , argv ) ;
2019-02-02 13:49:24 +00:00
return client . run ( ) ;
2017-04-01 07:20:54 +00:00
}
catch ( const boost : : program_options : : error & e )
{
std : : cerr < < " Bad arguments: " < < e . what ( ) < < std : : endl ;
return 1 ;
}
2020-06-24 16:34:49 +00:00
catch ( const DB : : Exception & e )
{
std : : string text = e . displayText ( ) ;
std : : cerr < < " Code: " < < e . code ( ) < < " . " < < text < < std : : endl ;
return 1 ;
}
2018-06-06 20:57:07 +00:00
catch ( . . . )
{
std : : cerr < < DB : : getCurrentExceptionMessage ( true ) < < std : : endl ;
return 1 ;
}
2017-03-13 17:35:17 +00:00
}