2019-02-15 11:46:07 +00:00
# include "HTTPHandler.h"
2017-06-02 18:48:33 +00:00
# include <chrono>
2012-03-09 04:45:27 +00:00
# include <iomanip>
2017-02-09 10:10:13 +00:00
# include <Poco/File.h>
2017-08-09 14:33:07 +00:00
# include <Poco/Net/HTTPBasicCredentials.h>
# include <Poco/Net/HTTPServerRequest.h>
2019-02-01 01:48:25 +00:00
# include <Poco/Net/HTTPServerRequestImpl.h>
2017-08-09 14:33:07 +00:00
# include <Poco/Net/HTTPServerResponse.h>
# include <Poco/Net/NetException.h>
2017-06-06 17:18:32 +00:00
# include <ext/scope_guard.h>
2018-08-20 02:34:00 +00:00
# include <Core/ExternalTable.h>
2018-01-15 19:07:47 +00:00
# include <Common/StringUtils/StringUtils.h>
2017-04-01 09:19:00 +00:00
# include <Common/escapeForFileName.h>
2020-03-18 18:54:27 +00:00
# include <common/getFQDNOrHostName.h>
2018-06-01 19:39:32 +00:00
# include <Common/CurrentThread.h>
2018-08-31 00:59:48 +00:00
# include <Common/setThreadName.h>
2019-10-04 13:32:21 +00:00
# include <Common/SettingsChanges.h>
2020-01-19 14:26:28 +00:00
# include <Disks/DiskSpaceMonitor.h>
2019-02-10 17:12:22 +00:00
# include <Compression/CompressedReadBuffer.h>
# include <Compression/CompressedWriteBuffer.h>
2017-04-01 09:19:00 +00:00
# include <IO/ReadBufferFromIStream.h>
# include <IO/ReadBufferFromString.h>
2017-08-09 14:33:07 +00:00
# include <IO/WriteBufferFromString.h>
2017-04-01 09:19:00 +00:00
# include <IO/WriteBufferFromHTTPServerResponse.h>
# include <IO/WriteBufferFromFile.h>
# include <IO/WriteHelpers.h>
# include <IO/copyData.h>
# include <IO/ConcatReadBuffer.h>
# include <IO/CascadeWriteBuffer.h>
# include <IO/MemoryReadWriteBuffer.h>
# include <IO/WriteBufferFromTemporaryFile.h>
2019-01-23 14:48:50 +00:00
# include <DataStreams/IBlockInputStream.h>
2017-04-01 09:19:00 +00:00
# include <Interpreters/executeQuery.h>
2017-07-13 20:58:19 +00:00
# include <Common/typeid_cast.h>
2017-09-23 23:32:26 +00:00
# include <Poco/Net/HTTPStream.h>
2020-04-16 12:31:57 +00:00
# if !defined(ARCADIA_BUILD)
# include <Common / config.h>
# endif
2012-03-09 03:06:09 +00:00
namespace DB
{
2016-01-11 21:46:36 +00:00
namespace ErrorCodes
{
2017-04-01 07:20:54 +00:00
2020-02-25 18:02:41 +00:00
extern const int LOGICAL_ERROR ;
2017-04-01 07:20:54 +00:00
extern const int CANNOT_PARSE_TEXT ;
extern const int CANNOT_PARSE_ESCAPE_SEQUENCE ;
extern const int CANNOT_PARSE_QUOTED_STRING ;
extern const int CANNOT_PARSE_DATE ;
extern const int CANNOT_PARSE_DATETIME ;
extern const int CANNOT_PARSE_NUMBER ;
extern const int CANNOT_OPEN_FILE ;
extern const int UNKNOWN_ELEMENT_IN_AST ;
extern const int UNKNOWN_TYPE_OF_AST_NODE ;
extern const int TOO_DEEP_AST ;
extern const int TOO_BIG_AST ;
extern const int UNEXPECTED_AST_STRUCTURE ;
2018-10-29 18:00:36 +00:00
extern const int SYNTAX_ERROR ;
2019-08-01 13:16:09 +00:00
extern const int INCORRECT_DATA ;
extern const int TYPE_MISMATCH ;
2017-04-01 07:20:54 +00:00
extern const int UNKNOWN_TABLE ;
extern const int UNKNOWN_FUNCTION ;
extern const int UNKNOWN_IDENTIFIER ;
extern const int UNKNOWN_TYPE ;
extern const int UNKNOWN_STORAGE ;
extern const int UNKNOWN_DATABASE ;
extern const int UNKNOWN_SETTING ;
extern const int UNKNOWN_DIRECTION_OF_SORTING ;
extern const int UNKNOWN_AGGREGATE_FUNCTION ;
extern const int UNKNOWN_FORMAT ;
extern const int UNKNOWN_DATABASE_ENGINE ;
extern const int UNKNOWN_TYPE_OF_QUERY ;
extern const int QUERY_IS_TOO_LARGE ;
extern const int NOT_IMPLEMENTED ;
extern const int SOCKET_TIMEOUT ;
extern const int UNKNOWN_USER ;
extern const int WRONG_PASSWORD ;
extern const int REQUIRED_PASSWORD ;
2017-06-02 18:48:33 +00:00
extern const int INVALID_SESSION_TIMEOUT ;
2017-09-25 19:45:15 +00:00
extern const int HTTP_LENGTH_REQUIRED ;
2016-01-11 21:46:36 +00:00
}
2017-06-02 21:01:17 +00:00
2017-02-07 05:21:31 +00:00
static Poco : : Net : : HTTPResponse : : HTTPStatus exceptionCodeToHTTPStatus ( int exception_code )
2017-02-06 08:23:02 +00:00
{
2017-04-01 07:20:54 +00:00
using namespace Poco : : Net ;
2019-08-02 05:07:10 +00:00
if ( exception_code = = ErrorCodes : : REQUIRED_PASSWORD )
return HTTPResponse : : HTTP_UNAUTHORIZED ;
else if ( exception_code = = ErrorCodes : : CANNOT_PARSE_TEXT | |
exception_code = = ErrorCodes : : CANNOT_PARSE_ESCAPE_SEQUENCE | |
exception_code = = ErrorCodes : : CANNOT_PARSE_QUOTED_STRING | |
exception_code = = ErrorCodes : : CANNOT_PARSE_DATE | |
exception_code = = ErrorCodes : : CANNOT_PARSE_DATETIME | |
2019-08-02 05:11:02 +00:00
exception_code = = ErrorCodes : : CANNOT_PARSE_NUMBER | |
exception_code = = ErrorCodes : : UNKNOWN_ELEMENT_IN_AST | |
2019-08-02 05:07:10 +00:00
exception_code = = ErrorCodes : : UNKNOWN_TYPE_OF_AST_NODE | |
exception_code = = ErrorCodes : : TOO_DEEP_AST | |
exception_code = = ErrorCodes : : TOO_BIG_AST | |
2019-08-02 05:11:02 +00:00
exception_code = = ErrorCodes : : UNEXPECTED_AST_STRUCTURE | |
exception_code = = ErrorCodes : : SYNTAX_ERROR | |
exception_code = = ErrorCodes : : INCORRECT_DATA | |
2019-08-02 05:07:10 +00:00
exception_code = = ErrorCodes : : TYPE_MISMATCH )
return HTTPResponse : : HTTP_BAD_REQUEST ;
else if ( exception_code = = ErrorCodes : : UNKNOWN_TABLE | |
exception_code = = ErrorCodes : : UNKNOWN_FUNCTION | |
exception_code = = ErrorCodes : : UNKNOWN_IDENTIFIER | |
exception_code = = ErrorCodes : : UNKNOWN_TYPE | |
exception_code = = ErrorCodes : : UNKNOWN_STORAGE | |
exception_code = = ErrorCodes : : UNKNOWN_DATABASE | |
exception_code = = ErrorCodes : : UNKNOWN_SETTING | |
exception_code = = ErrorCodes : : UNKNOWN_DIRECTION_OF_SORTING | |
exception_code = = ErrorCodes : : UNKNOWN_AGGREGATE_FUNCTION | |
exception_code = = ErrorCodes : : UNKNOWN_FORMAT | |
2019-08-02 05:11:02 +00:00
exception_code = = ErrorCodes : : UNKNOWN_DATABASE_ENGINE | |
exception_code = = ErrorCodes : : UNKNOWN_TYPE_OF_QUERY )
2019-08-02 05:07:10 +00:00
return HTTPResponse : : HTTP_NOT_FOUND ;
else if ( exception_code = = ErrorCodes : : QUERY_IS_TOO_LARGE )
return HTTPResponse : : HTTP_REQUESTENTITYTOOLARGE ;
else if ( exception_code = = ErrorCodes : : NOT_IMPLEMENTED )
return HTTPResponse : : HTTP_NOT_IMPLEMENTED ;
else if ( exception_code = = ErrorCodes : : SOCKET_TIMEOUT | |
exception_code = = ErrorCodes : : CANNOT_OPEN_FILE )
return HTTPResponse : : HTTP_SERVICE_UNAVAILABLE ;
else if ( exception_code = = ErrorCodes : : HTTP_LENGTH_REQUIRED )
return HTTPResponse : : HTTP_LENGTH_REQUIRED ;
return HTTPResponse : : HTTP_INTERNAL_SERVER_ERROR ;
2017-02-06 08:23:02 +00:00
}
2017-06-02 18:48:33 +00:00
2020-03-06 18:14:33 +00:00
static std : : chrono : : steady_clock : : duration parseSessionTimeout (
2017-08-24 14:51:13 +00:00
const Poco : : Util : : AbstractConfiguration & config ,
const HTMLForm & params )
2017-06-02 18:48:33 +00:00
{
2020-03-06 18:14:33 +00:00
unsigned session_timeout = config . getInt ( " default_session_timeout " , 60 ) ;
2017-06-02 18:48:33 +00:00
if ( params . has ( " session_timeout " ) )
{
2020-03-06 18:14:33 +00:00
unsigned max_session_timeout = config . getUInt ( " max_session_timeout " , 3600 ) ;
2017-06-02 21:01:17 +00:00
std : : string session_timeout_str = params . get ( " session_timeout " ) ;
2017-06-02 18:48:33 +00:00
2017-06-27 15:58:33 +00:00
ReadBufferFromString buf ( session_timeout_str ) ;
if ( ! tryReadIntText ( session_timeout , buf ) | | ! buf . eof ( ) )
throw Exception ( " Invalid session timeout: ' " + session_timeout_str + " ' " , ErrorCodes : : INVALID_SESSION_TIMEOUT ) ;
2017-06-02 18:48:33 +00:00
2017-06-02 21:01:17 +00:00
if ( session_timeout > max_session_timeout )
throw Exception ( " Session timeout ' " + session_timeout_str + " ' is larger than max_session_timeout: " + toString ( max_session_timeout )
+ " . Maximum session timeout could be modified in configuration file. " ,
2017-06-02 18:48:33 +00:00
ErrorCodes : : INVALID_SESSION_TIMEOUT ) ;
}
2020-03-06 18:14:33 +00:00
return std : : chrono : : seconds ( session_timeout ) ;
2017-06-02 18:48:33 +00:00
}
2017-08-09 11:57:09 +00:00
HTTPHandler : : HTTPHandler ( IServer & server_ )
2019-11-04 18:36:28 +00:00
: server ( server_ ) , log ( & Logger : : get ( " HTTPHandler " ) )
2017-02-06 08:23:02 +00:00
{
2018-03-08 07:36:58 +00:00
server_display_name = server . config ( ) . getString ( " display_name " , getFQDNOrHostName ( ) ) ;
2017-02-06 08:23:02 +00:00
}
2016-01-11 21:46:36 +00:00
2017-06-02 21:01:17 +00:00
2019-11-04 18:36:28 +00:00
HTTPHandler : : SessionContextHolder : : ~ SessionContextHolder ( )
2012-03-09 03:06:09 +00:00
{
2019-11-04 18:36:28 +00:00
if ( session_context )
session_context - > releaseSession ( session_id , session_timeout ) ;
}
2018-06-19 20:30:35 +00:00
2018-06-01 19:39:32 +00:00
2019-11-04 18:36:28 +00:00
HTTPHandler : : SessionContextHolder : : SessionContextHolder ( IServer & accepted_server , HTMLForm & params )
{
session_id = params . get ( " session_id " , " " ) ;
context = std : : make_unique < Context > ( accepted_server . context ( ) ) ;
2017-04-01 07:20:54 +00:00
2019-11-04 18:36:28 +00:00
if ( ! session_id . empty ( ) )
2019-11-01 10:03:21 +00:00
{
2019-11-04 18:36:28 +00:00
session_timeout = parseSessionTimeout ( accepted_server . config ( ) , params ) ;
2019-11-06 07:02:10 +00:00
session_context = context - > acquireSession ( session_id , session_timeout , params . check < String > ( " session_check " , " 1 " ) ) ;
2019-11-01 10:03:21 +00:00
2019-11-04 18:36:28 +00:00
context = std : : make_unique < Context > ( * session_context ) ;
context - > setSessionContext ( * session_context ) ;
}
}
2017-04-01 07:20:54 +00:00
2019-11-04 18:36:28 +00:00
void HTTPHandler : : SessionContextHolder : : authentication ( HTTPServerRequest & request , HTMLForm & params )
{
auto user = request . get ( " X-ClickHouse-User " , " " ) ;
auto password = request . get ( " X-ClickHouse-Key " , " " ) ;
auto quota_key = request . get ( " X-ClickHouse-Quota " , " " ) ;
2017-04-01 07:20:54 +00:00
2017-09-21 11:16:18 +00:00
if ( user . empty ( ) & & password . empty ( ) & & quota_key . empty ( ) )
2017-04-01 07:20:54 +00:00
{
2017-09-21 11:16:18 +00:00
/// User name and password can be passed using query parameters
/// or using HTTP Basic auth (both methods are insecure).
if ( request . hasCredentials ( ) )
{
Poco : : Net : : HTTPBasicCredentials credentials ( request ) ;
user = credentials . getUsername ( ) ;
password = credentials . getPassword ( ) ;
2017-09-23 23:18:48 +00:00
}
else
{
2017-09-21 11:16:18 +00:00
user = params . get ( " user " , " default " ) ;
password = params . get ( " password " , " " ) ;
}
2017-04-01 07:20:54 +00:00
2017-09-21 11:16:18 +00:00
quota_key = params . get ( " quota_key " , " " ) ;
}
else
{
/// It is prohibited to mix different authorization schemes.
if ( request . hasCredentials ( )
| | params . has ( " user " )
| | params . has ( " password " )
| | params . has ( " quota_key " ) )
{
2017-09-23 23:20:58 +00:00
throw Exception ( " Invalid authentication: it is not allowed to use X-ClickHouse HTTP headers and other authentication methods simultaneously " , ErrorCodes : : REQUIRED_PASSWORD ) ;
2017-09-21 11:16:18 +00:00
}
2017-04-01 07:20:54 +00:00
}
std : : string query_id = params . get ( " query_id " , " " ) ;
2019-11-04 18:36:28 +00:00
context - > setUser ( user , password , request . clientAddress ( ) , quota_key ) ;
context - > setCurrentQueryId ( query_id ) ;
}
2019-02-01 01:48:25 +00:00
2019-11-06 07:02:10 +00:00
void HTTPHandler : : processQuery ( Context & context , HTTPRequest & request , HTMLForm & params , HTTPResponse & response )
2019-11-04 18:36:28 +00:00
{
2019-11-06 11:01:02 +00:00
const auto & name_with_custom_executor = context . getCustomExecutor ( request , params ) ;
2019-11-06 13:40:38 +00:00
LOG_TRACE ( log , " Using ' " < < name_with_custom_executor . first < < " ' CustomExecutor to execute URI: " < < request . getURI ( ) ) ;
2019-02-01 01:48:25 +00:00
2019-11-06 07:02:10 +00:00
ExtractorClientInfo { context . getClientInfo ( ) } . extract ( request ) ;
ExtractorContextChange { context , name_with_custom_executor . second } . extract ( request , params ) ;
2019-02-01 01:48:25 +00:00
2019-11-06 07:02:10 +00:00
HTTPInputStreams input_streams { context , request , params } ;
HTTPOutputStreams output_streams = HTTPOutputStreams ( context , request , response , params , getKeepAliveTimeout ( ) ) ;
2019-11-06 11:01:02 +00:00
name_with_custom_executor . second - > executeQuery ( context , request , response , params , input_streams , output_streams ) ;
2012-03-09 03:06:09 +00:00
}
2019-11-06 07:02:10 +00:00
void HTTPHandler : : trySendExceptionToClient (
const std : : string & message , int exception_code , Poco : : Net : : HTTPServerRequest & request , Poco : : Net : : HTTPServerResponse & response , bool compression )
2012-03-09 03:06:09 +00:00
{
2017-04-01 07:20:54 +00:00
try
{
2020-01-22 16:32:18 +00:00
response . set ( " X-ClickHouse-Exception-Code " , toString < int > ( exception_code ) ) ;
2020-01-23 13:40:16 +00:00
2017-04-01 07:20:54 +00:00
/// If HTTP method is POST and Keep-Alive is turned on, we should read the whole request body
/// to avoid reading part of the current request body in the next request.
2019-11-06 07:02:10 +00:00
if ( request . getMethod ( ) = = Poco : : Net : : HTTPRequest : : HTTP_POST & & response . getKeepAlive ( )
& & ! request . stream ( ) . eof ( ) & & exception_code ! = ErrorCodes : : HTTP_LENGTH_REQUIRED )
2017-04-01 07:20:54 +00:00
request . stream ( ) . ignore ( std : : numeric_limits < std : : streamsize > : : max ( ) ) ;
2019-11-07 03:51:11 +00:00
if ( exception_code = = ErrorCodes : : UNKNOWN_USER | | exception_code = = ErrorCodes : : WRONG_PASSWORD
| | exception_code = = ErrorCodes : : REQUIRED_PASSWORD | | exception_code = = ErrorCodes : : HTTP_LENGTH_REQUIRED )
2017-04-01 07:20:54 +00:00
{
2019-11-07 03:51:11 +00:00
if ( exception_code = = ErrorCodes : : HTTP_LENGTH_REQUIRED )
response . setStatusAndReason ( exceptionCodeToHTTPStatus ( exception_code ) ) ;
else
response . requireAuthentication ( " ClickHouse server HTTP API " ) ;
2019-11-06 07:02:10 +00:00
response . send ( ) < < message < < std : : endl ;
2017-04-01 07:20:54 +00:00
}
else
{
response . setStatusAndReason ( exceptionCodeToHTTPStatus ( exception_code ) ) ;
2019-11-06 07:02:10 +00:00
HTTPOutputStreams output_streams ( request , response , compression , getKeepAliveTimeout ( ) ) ;
2017-04-01 07:20:54 +00:00
2019-11-06 07:02:10 +00:00
writeString ( message , * output_streams . out_maybe_compressed ) ;
writeChar ( ' \n ' , * output_streams . out_maybe_compressed ) ;
2017-04-01 07:20:54 +00:00
2019-11-06 07:02:10 +00:00
output_streams . finalize ( ) ;
2017-04-01 07:20:54 +00:00
}
}
catch ( . . . )
{
tryLogCurrentException ( log , " Cannot send exception to client " ) ;
}
2014-07-14 19:46:06 +00:00
}
2012-06-25 05:07:34 +00:00
2014-07-14 19:46:06 +00:00
void HTTPHandler : : handleRequest ( Poco : : Net : : HTTPServerRequest & request , Poco : : Net : : HTTPServerResponse & response )
{
2018-08-31 00:59:48 +00:00
setThreadName ( " HTTPHandler " ) ;
2019-01-15 18:39:54 +00:00
ThreadStatus thread_status ;
2018-08-31 00:59:48 +00:00
2017-04-01 07:20:54 +00:00
/// In case of exception, send stack trace to client.
2019-11-06 07:02:10 +00:00
bool with_stacktrace = false , internal_compression = false ;
2016-06-25 07:22:12 +00:00
2017-04-01 07:20:54 +00:00
try
{
2019-11-04 18:36:28 +00:00
response . set ( " Content-Type " , " text/plain; charset=UTF-8 " ) ;
2018-03-08 07:36:58 +00:00
response . set ( " X-ClickHouse-Server-Display-Name " , server_display_name ) ;
2019-11-04 18:36:28 +00:00
2017-04-01 07:20:54 +00:00
/// For keep-alive to work.
if ( request . getVersion ( ) = = Poco : : Net : : HTTPServerRequest : : HTTP_1_1 )
response . setChunkedTransferEncoding ( true ) ;
2014-07-14 19:46:06 +00:00
2017-04-01 07:20:54 +00:00
HTMLForm params ( request ) ;
with_stacktrace = params . getParsed < bool > ( " stacktrace " , false ) ;
2019-11-06 07:02:10 +00:00
internal_compression = params . getParsed < bool > ( " compress " , false ) ;
2016-06-25 07:22:12 +00:00
2017-09-25 19:45:15 +00:00
/// Workaround. Poco does not detect 411 Length Required case.
2019-11-04 18:36:28 +00:00
if ( request . getMethod ( ) = = Poco : : Net : : HTTPRequest : : HTTP_POST & & ! request . getChunkedTransferEncoding ( ) & & ! request . hasContentLength ( ) )
2020-02-29 18:27:34 +00:00
throw Exception ( " The Transfer-Encoding is not chunked and there is no Content-Length header for POST request " , ErrorCodes : : HTTP_LENGTH_REQUIRED ) ;
2017-09-25 19:45:15 +00:00
2019-11-04 18:36:28 +00:00
{
SessionContextHolder holder { server , params } ;
CurrentThread : : QueryScope query_scope ( * holder . context ) ;
holder . authentication ( request , params ) ;
2019-11-06 07:02:10 +00:00
processQuery ( * holder . context , request , params , response ) ;
LOG_INFO ( log , " Done processing query " ) ;
2019-11-04 18:36:28 +00:00
}
2017-04-01 07:20:54 +00:00
}
catch ( . . . )
{
tryLogCurrentException ( log ) ;
2016-06-25 07:22:12 +00:00
2017-04-01 07:20:54 +00:00
/** If exception is received from remote server, then stack trace is embedded in message.
* If exception is thrown on local server , then stack trace is in separate field .
*/
int exception_code = getCurrentExceptionCode ( ) ;
2019-11-04 18:36:28 +00:00
std : : string exception_message = getCurrentExceptionMessage ( with_stacktrace , true ) ;
2019-11-06 07:02:10 +00:00
trySendExceptionToClient ( exception_message , exception_code , request , response , internal_compression ) ;
2017-04-01 07:20:54 +00:00
}
2012-03-09 03:06:09 +00:00
}
}