2021-02-19 12:51:26 +00:00
# include <Server/HTTPHandler.h>
2019-02-15 11:46:07 +00:00
2021-03-11 20:41:10 +00:00
# include <Access/Authentication.h>
# include <Access/Credentials.h>
# include <Access/ExternalAuthenticators.h>
2020-11-11 12:34:28 +00:00
# include <Compression/CompressedReadBuffer.h>
# include <Compression/CompressedWriteBuffer.h>
2021-02-19 12:51:26 +00:00
# include <Core/ExternalTable.h>
# include <Disks/StoragePolicy.h>
# include <IO/CascadeWriteBuffer.h>
# include <IO/ConcatReadBuffer.h>
# include <IO/MemoryReadWriteBuffer.h>
2020-11-11 12:34:28 +00:00
# include <IO/ReadBufferFromIStream.h>
# include <IO/ReadBufferFromString.h>
# include <IO/WriteBufferFromFile.h>
2021-02-19 12:51:26 +00:00
# include <IO/WriteBufferFromString.h>
# include <IO/WriteBufferFromTemporaryFile.h>
2020-11-11 12:34:28 +00:00
# include <IO/WriteHelpers.h>
# include <IO/copyData.h>
2020-12-10 22:05:02 +00:00
# include <Interpreters/Context.h>
2021-02-19 12:51:26 +00:00
# include <Interpreters/QueryParameterVisitor.h>
# include <Interpreters/executeQuery.h>
2021-08-01 14:12:34 +00:00
# include <Interpreters/Session.h>
2021-02-19 12:51:26 +00:00
# include <Server/HTTPHandlerFactory.h>
# include <Server/HTTPHandlerRequestFilter.h>
# include <Server/IServer.h>
2021-10-05 14:43:33 +00:00
# include <base/logger_useful.h>
2021-02-19 12:51:26 +00:00
# include <Common/SettingsChanges.h>
# include <Common/StringUtils/StringUtils.h>
# include <Common/escapeForFileName.h>
# include <Common/setThreadName.h>
2017-07-13 20:58:19 +00:00
# include <Common/typeid_cast.h>
2021-10-04 07:55:09 +00:00
2021-10-05 14:43:33 +00:00
# include <base/getFQDNOrHostName.h>
# include <base/scope_guard.h>
2021-09-22 16:54:36 +00:00
# include <Server/HTTP/HTTPResponse.h>
2017-09-23 23:32:26 +00:00
2020-04-16 12:31:57 +00:00
# if !defined(ARCADIA_BUILD)
# include <Common / config.h>
# endif
2021-03-11 20:41:10 +00:00
# include <Poco/Base64Decoder.h>
# include <Poco/Base64Encoder.h>
2021-02-19 12:51:26 +00:00
# include <Poco/Net/HTTPBasicCredentials.h>
# include <Poco/Net/HTTPStream.h>
# include <Poco/Net/NetException.h>
2021-03-11 20:41:10 +00:00
# include <Poco/MemoryStream.h>
# include <Poco/StreamCopier.h>
# include <Poco/String.h>
2021-02-19 12:51:26 +00:00
# include <chrono>
# include <iomanip>
2021-03-11 20:41:10 +00:00
# include <sstream>
2021-02-19 12:51:26 +00:00
2020-04-16 12:31:57 +00:00
2012-03-09 03:06:09 +00:00
namespace DB
{
2020-11-11 12:34:28 +00:00
2016-01-11 21:46:36 +00:00
namespace ErrorCodes
{
2020-11-11 12:34:28 +00:00
2020-02-25 18:02:41 +00:00
extern const int LOGICAL_ERROR ;
2017-04-01 07:20:54 +00:00
extern const int CANNOT_PARSE_TEXT ;
extern const int CANNOT_PARSE_ESCAPE_SEQUENCE ;
extern const int CANNOT_PARSE_QUOTED_STRING ;
extern const int CANNOT_PARSE_DATE ;
extern const int CANNOT_PARSE_DATETIME ;
extern const int CANNOT_PARSE_NUMBER ;
2020-05-03 19:47:07 +00:00
extern const int CANNOT_PARSE_INPUT_ASSERTION_FAILED ;
2017-04-01 07:20:54 +00:00
extern const int CANNOT_OPEN_FILE ;
2020-04-26 11:06:14 +00:00
extern const int CANNOT_COMPILE_REGEXP ;
2017-04-01 07:20:54 +00:00
extern const int UNKNOWN_ELEMENT_IN_AST ;
extern const int UNKNOWN_TYPE_OF_AST_NODE ;
extern const int TOO_DEEP_AST ;
extern const int TOO_BIG_AST ;
extern const int UNEXPECTED_AST_STRUCTURE ;
2018-10-29 18:00:36 +00:00
extern const int SYNTAX_ERROR ;
2019-08-01 13:16:09 +00:00
extern const int INCORRECT_DATA ;
extern const int TYPE_MISMATCH ;
2017-04-01 07:20:54 +00:00
extern const int UNKNOWN_TABLE ;
extern const int UNKNOWN_FUNCTION ;
extern const int UNKNOWN_IDENTIFIER ;
extern const int UNKNOWN_TYPE ;
extern const int UNKNOWN_STORAGE ;
extern const int UNKNOWN_DATABASE ;
extern const int UNKNOWN_SETTING ;
extern const int UNKNOWN_DIRECTION_OF_SORTING ;
extern const int UNKNOWN_AGGREGATE_FUNCTION ;
extern const int UNKNOWN_FORMAT ;
extern const int UNKNOWN_DATABASE_ENGINE ;
extern const int UNKNOWN_TYPE_OF_QUERY ;
2020-04-04 08:57:16 +00:00
extern const int NO_ELEMENTS_IN_CONFIG ;
2017-04-01 07:20:54 +00:00
extern const int QUERY_IS_TOO_LARGE ;
extern const int NOT_IMPLEMENTED ;
extern const int SOCKET_TIMEOUT ;
extern const int UNKNOWN_USER ;
extern const int WRONG_PASSWORD ;
extern const int REQUIRED_PASSWORD ;
2021-03-11 20:41:10 +00:00
extern const int AUTHENTICATION_FAILED ;
2017-06-02 18:48:33 +00:00
2020-08-28 01:21:08 +00:00
extern const int BAD_REQUEST_PARAMETER ;
2017-06-02 18:48:33 +00:00
extern const int INVALID_SESSION_TIMEOUT ;
2017-09-25 19:45:15 +00:00
extern const int HTTP_LENGTH_REQUIRED ;
2016-01-11 21:46:36 +00:00
}
2021-09-18 09:36:02 +00:00
namespace
{
2021-09-22 16:54:36 +00:00
bool tryAddHeadersFromConfig ( HTTPServerResponse & response , const Poco : : Util : : LayeredConfiguration & config )
{
if ( config . has ( " http_options_response " ) )
2021-09-18 09:36:02 +00:00
{
2021-09-22 16:54:36 +00:00
Strings config_keys ;
config . keys ( " http_options_response " , config_keys ) ;
for ( const std : : string & config_key : config_keys )
2021-09-18 09:36:02 +00:00
{
2021-09-22 16:54:36 +00:00
if ( config_key = = " header " | | config_key . starts_with ( " header[ " ) )
2021-09-18 09:36:02 +00:00
{
2021-09-22 16:54:36 +00:00
/// If there is empty header name, it will not be processed and message about it will be in logs
if ( config . getString ( " http_options_response. " + config_key + " .name " , " " ) . empty ( ) )
LOG_WARNING ( & Poco : : Logger : : get ( " processOptionsRequest " ) , " Empty header was found in config. It will not be processed. " ) ;
2021-09-24 12:40:27 +00:00
else
2021-09-22 16:54:36 +00:00
response . add ( config . getString ( " http_options_response. " + config_key + " .name " , " " ) ,
config . getString ( " http_options_response. " + config_key + " .value " , " " ) ) ;
2021-09-18 09:36:02 +00:00
}
}
2021-09-22 16:54:36 +00:00
return true ;
}
return false ;
}
/// Process options request. Useful for CORS.
void processOptionsRequest ( HTTPServerResponse & response , const Poco : : Util : : LayeredConfiguration & config )
{
/// If can add some headers from config
if ( tryAddHeadersFromConfig ( response , config ) )
{
response . setKeepAlive ( false ) ;
response . setStatusAndReason ( HTTPResponse : : HTTP_NO_CONTENT ) ;
response . send ( ) ;
2021-09-18 09:36:02 +00:00
}
}
2021-09-22 16:54:36 +00:00
}
2021-09-18 09:36:02 +00:00
2021-03-11 20:41:10 +00:00
static String base64Decode ( const String & encoded )
{
String decoded ;
Poco : : MemoryInputStream istr ( encoded . data ( ) , encoded . size ( ) ) ;
Poco : : Base64Decoder decoder ( istr ) ;
Poco : : StreamCopier : : copyToString ( decoder , decoded ) ;
return decoded ;
}
static String base64Encode ( const String & decoded )
{
std : : ostringstream ostr ; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
ostr . exceptions ( std : : ios : : failbit ) ;
Poco : : Base64Encoder encoder ( ostr ) ;
encoder . rdbuf ( ) - > setLineLength ( 0 ) ;
encoder < < decoded ;
encoder . close ( ) ;
return ostr . str ( ) ;
}
2017-06-02 21:01:17 +00:00
2017-02-07 05:21:31 +00:00
static Poco : : Net : : HTTPResponse : : HTTPStatus exceptionCodeToHTTPStatus ( int exception_code )
2017-02-06 08:23:02 +00:00
{
2017-04-01 07:20:54 +00:00
using namespace Poco : : Net ;
2019-08-02 05:07:10 +00:00
if ( exception_code = = ErrorCodes : : REQUIRED_PASSWORD )
2020-05-03 19:47:07 +00:00
{
2019-08-02 05:07:10 +00:00
return HTTPResponse : : HTTP_UNAUTHORIZED ;
2020-05-03 19:47:07 +00:00
}
2021-03-11 20:41:10 +00:00
else if ( exception_code = = ErrorCodes : : UNKNOWN_USER | |
exception_code = = ErrorCodes : : WRONG_PASSWORD | |
exception_code = = ErrorCodes : : AUTHENTICATION_FAILED )
{
return HTTPResponse : : HTTP_FORBIDDEN ;
}
2020-11-11 12:34:28 +00:00
else if ( exception_code = = ErrorCodes : : CANNOT_PARSE_TEXT | |
exception_code = = ErrorCodes : : CANNOT_PARSE_ESCAPE_SEQUENCE | |
exception_code = = ErrorCodes : : CANNOT_PARSE_QUOTED_STRING | |
exception_code = = ErrorCodes : : CANNOT_PARSE_DATE | |
exception_code = = ErrorCodes : : CANNOT_PARSE_DATETIME | |
exception_code = = ErrorCodes : : CANNOT_PARSE_NUMBER | |
exception_code = = ErrorCodes : : CANNOT_PARSE_INPUT_ASSERTION_FAILED | |
exception_code = = ErrorCodes : : UNKNOWN_ELEMENT_IN_AST | |
exception_code = = ErrorCodes : : UNKNOWN_TYPE_OF_AST_NODE | |
exception_code = = ErrorCodes : : TOO_DEEP_AST | |
exception_code = = ErrorCodes : : TOO_BIG_AST | |
exception_code = = ErrorCodes : : UNEXPECTED_AST_STRUCTURE | |
exception_code = = ErrorCodes : : SYNTAX_ERROR | |
exception_code = = ErrorCodes : : INCORRECT_DATA | |
exception_code = = ErrorCodes : : TYPE_MISMATCH )
2020-05-03 19:47:07 +00:00
{
2019-08-02 05:07:10 +00:00
return HTTPResponse : : HTTP_BAD_REQUEST ;
2020-05-03 19:47:07 +00:00
}
2020-11-11 12:34:28 +00:00
else if ( exception_code = = ErrorCodes : : UNKNOWN_TABLE | |
exception_code = = ErrorCodes : : UNKNOWN_FUNCTION | |
exception_code = = ErrorCodes : : UNKNOWN_IDENTIFIER | |
exception_code = = ErrorCodes : : UNKNOWN_TYPE | |
exception_code = = ErrorCodes : : UNKNOWN_STORAGE | |
exception_code = = ErrorCodes : : UNKNOWN_DATABASE | |
exception_code = = ErrorCodes : : UNKNOWN_SETTING | |
exception_code = = ErrorCodes : : UNKNOWN_DIRECTION_OF_SORTING | |
exception_code = = ErrorCodes : : UNKNOWN_AGGREGATE_FUNCTION | |
exception_code = = ErrorCodes : : UNKNOWN_FORMAT | |
exception_code = = ErrorCodes : : UNKNOWN_DATABASE_ENGINE | |
exception_code = = ErrorCodes : : UNKNOWN_TYPE_OF_QUERY )
2020-05-03 19:47:07 +00:00
{
2019-08-02 05:07:10 +00:00
return HTTPResponse : : HTTP_NOT_FOUND ;
2020-05-03 19:47:07 +00:00
}
2019-08-02 05:07:10 +00:00
else if ( exception_code = = ErrorCodes : : QUERY_IS_TOO_LARGE )
2020-05-03 19:47:07 +00:00
{
2019-08-02 05:07:10 +00:00
return HTTPResponse : : HTTP_REQUESTENTITYTOOLARGE ;
2020-05-03 19:47:07 +00:00
}
2019-08-02 05:07:10 +00:00
else if ( exception_code = = ErrorCodes : : NOT_IMPLEMENTED )
2020-05-03 19:47:07 +00:00
{
2019-08-02 05:07:10 +00:00
return HTTPResponse : : HTTP_NOT_IMPLEMENTED ;
2020-05-03 19:47:07 +00:00
}
2020-11-11 12:34:28 +00:00
else if ( exception_code = = ErrorCodes : : SOCKET_TIMEOUT | |
exception_code = = ErrorCodes : : CANNOT_OPEN_FILE )
2020-05-03 19:47:07 +00:00
{
2019-08-02 05:07:10 +00:00
return HTTPResponse : : HTTP_SERVICE_UNAVAILABLE ;
2020-05-03 19:47:07 +00:00
}
2019-08-02 05:07:10 +00:00
else if ( exception_code = = ErrorCodes : : HTTP_LENGTH_REQUIRED )
2020-05-03 19:47:07 +00:00
{
2019-08-02 05:07:10 +00:00
return HTTPResponse : : HTTP_LENGTH_REQUIRED ;
2020-05-03 19:47:07 +00:00
}
2019-08-02 05:07:10 +00:00
return HTTPResponse : : HTTP_INTERNAL_SERVER_ERROR ;
2017-02-06 08:23:02 +00:00
}
2017-06-02 18:48:33 +00:00
2020-11-11 12:34:28 +00:00
static std : : chrono : : steady_clock : : duration parseSessionTimeout (
const Poco : : Util : : AbstractConfiguration & config ,
const HTMLForm & params )
2017-06-02 18:48:33 +00:00
{
2020-03-06 18:14:33 +00:00
unsigned session_timeout = config . getInt ( " default_session_timeout " , 60 ) ;
2017-06-02 18:48:33 +00:00
if ( params . has ( " session_timeout " ) )
{
2020-03-06 18:14:33 +00:00
unsigned max_session_timeout = config . getUInt ( " max_session_timeout " , 3600 ) ;
2017-06-02 21:01:17 +00:00
std : : string session_timeout_str = params . get ( " session_timeout " ) ;
2017-06-02 18:48:33 +00:00
2017-06-27 15:58:33 +00:00
ReadBufferFromString buf ( session_timeout_str ) ;
if ( ! tryReadIntText ( session_timeout , buf ) | | ! buf . eof ( ) )
throw Exception ( " Invalid session timeout: ' " + session_timeout_str + " ' " , ErrorCodes : : INVALID_SESSION_TIMEOUT ) ;
2017-06-02 18:48:33 +00:00
2017-06-02 21:01:17 +00:00
if ( session_timeout > max_session_timeout )
2020-11-11 12:34:28 +00:00
throw Exception ( " Session timeout ' " + session_timeout_str + " ' is larger than max_session_timeout: " + toString ( max_session_timeout )
+ " . Maximum session timeout could be modified in configuration file. " ,
2017-06-02 18:48:33 +00:00
ErrorCodes : : INVALID_SESSION_TIMEOUT ) ;
}
2020-03-06 18:14:33 +00:00
return std : : chrono : : seconds ( session_timeout ) ;
2017-06-02 18:48:33 +00:00
}
2017-02-28 14:15:13 +00:00
void HTTPHandler : : pushDelayedResults ( Output & used_output )
{
2017-04-01 07:20:54 +00:00
std : : vector < WriteBufferPtr > write_buffers ;
2021-03-04 11:10:21 +00:00
ConcatReadBuffer : : Buffers read_buffers ;
2017-04-01 07:20:54 +00:00
2020-05-18 08:08:55 +00:00
auto * cascade_buffer = typeid_cast < CascadeWriteBuffer * > ( used_output . out_maybe_delayed_and_compressed . get ( ) ) ;
2017-04-01 07:20:54 +00:00
if ( ! cascade_buffer )
throw Exception ( " Expected CascadeWriteBuffer " , ErrorCodes : : LOGICAL_ERROR ) ;
cascade_buffer - > getResultBuffers ( write_buffers ) ;
if ( write_buffers . empty ( ) )
throw Exception ( " At least one buffer is expected to overwrite result into HTTP response " , ErrorCodes : : LOGICAL_ERROR ) ;
for ( auto & write_buf : write_buffers )
{
IReadableWriteBuffer * write_buf_concrete ;
ReadBufferPtr reread_buf ;
2020-11-11 12:34:28 +00:00
if ( write_buf
& & ( write_buf_concrete = dynamic_cast < IReadableWriteBuffer * > ( write_buf . get ( ) ) )
2017-04-01 07:20:54 +00:00
& & ( reread_buf = write_buf_concrete - > tryGetReadBuffer ( ) ) )
{
2021-03-04 11:10:21 +00:00
read_buffers . emplace_back ( wrapReadBufferPointer ( reread_buf ) ) ;
2017-04-01 07:20:54 +00:00
}
}
2021-03-04 11:10:21 +00:00
if ( ! read_buffers . empty ( ) )
2021-02-04 15:59:54 +00:00
{
2021-03-04 11:10:21 +00:00
ConcatReadBuffer concat_read_buffer ( std : : move ( read_buffers ) ) ;
2021-02-04 15:59:54 +00:00
copyData ( concat_read_buffer , * used_output . out_maybe_compressed ) ;
}
2017-02-28 14:15:13 +00:00
}
2020-11-11 12:34:28 +00:00
HTTPHandler : : HTTPHandler ( IServer & server_ , const std : : string & name )
: server ( server_ )
, log ( & Poco : : Logger : : get ( name ) )
2021-08-01 14:12:34 +00:00
, default_settings ( server . context ( ) - > getSettingsRef ( ) )
2017-02-06 08:23:02 +00:00
{
2018-03-08 07:36:58 +00:00
server_display_name = server . config ( ) . getString ( " display_name " , getFQDNOrHostName ( ) ) ;
2017-02-06 08:23:02 +00:00
}
2016-01-11 21:46:36 +00:00
2017-06-02 21:01:17 +00:00
2021-03-11 20:41:10 +00:00
/// We need d-tor to be present in this translation unit to make it play well with some
/// forward decls in the header. Other than that, the default d-tor would be OK.
2021-08-01 14:12:34 +00:00
HTTPHandler : : ~ HTTPHandler ( ) = default ;
2021-03-11 20:41:10 +00:00
bool HTTPHandler : : authenticateUser (
2021-02-19 12:51:26 +00:00
HTTPServerRequest & request ,
2017-04-01 07:20:54 +00:00
HTMLForm & params ,
2021-03-11 20:41:10 +00:00
HTTPServerResponse & response )
2012-03-09 03:06:09 +00:00
{
2021-03-11 20:41:10 +00:00
using namespace Poco : : Net ;
2017-04-01 07:20:54 +00:00
2017-09-21 11:16:18 +00:00
/// The user and password can be passed by headers (similar to X-Auth-*),
/// which is used by load balancers to pass authentication information.
std : : string user = request . get ( " X-ClickHouse-User " , " " ) ;
std : : string password = request . get ( " X-ClickHouse-Key " , " " ) ;
std : : string quota_key = request . get ( " X-ClickHouse-Quota " , " " ) ;
2017-04-01 07:20:54 +00:00
2021-03-11 20:41:10 +00:00
std : : string spnego_challenge ;
2017-09-21 11:16:18 +00:00
if ( user . empty ( ) & & password . empty ( ) & & quota_key . empty ( ) )
2017-04-01 07:20:54 +00:00
{
2017-09-21 11:16:18 +00:00
/// User name and password can be passed using query parameters
/// or using HTTP Basic auth (both methods are insecure).
if ( request . hasCredentials ( ) )
{
2021-03-11 20:41:10 +00:00
/// It is prohibited to mix different authorization schemes.
if ( params . has ( " user " ) | | params . has ( " password " ) )
throw Exception ( " Invalid authentication: it is not allowed to use Authorization HTTP header and authentication via parameters simultaneously " , ErrorCodes : : AUTHENTICATION_FAILED ) ;
std : : string scheme ;
std : : string auth_info ;
request . getCredentials ( scheme , auth_info ) ;
if ( Poco : : icompare ( scheme , " Basic " ) = = 0 )
{
HTTPBasicCredentials credentials ( auth_info ) ;
user = credentials . getUsername ( ) ;
password = credentials . getPassword ( ) ;
}
else if ( Poco : : icompare ( scheme , " Negotiate " ) = = 0 )
{
spnego_challenge = auth_info ;
2017-09-21 11:16:18 +00:00
2021-03-11 20:41:10 +00:00
if ( spnego_challenge . empty ( ) )
throw Exception ( " Invalid authentication: SPNEGO challenge is empty " , ErrorCodes : : AUTHENTICATION_FAILED ) ;
}
else
{
throw Exception ( " Invalid authentication: ' " + scheme + " ' HTTP Authorization scheme is not supported " , ErrorCodes : : AUTHENTICATION_FAILED ) ;
}
2017-09-23 23:18:48 +00:00
}
else
{
2017-09-21 11:16:18 +00:00
user = params . get ( " user " , " default " ) ;
password = params . get ( " password " , " " ) ;
}
2017-04-01 07:20:54 +00:00
2017-09-21 11:16:18 +00:00
quota_key = params . get ( " quota_key " , " " ) ;
}
else
{
/// It is prohibited to mix different authorization schemes.
2021-03-11 20:41:10 +00:00
if ( request . hasCredentials ( ) | | params . has ( " user " ) | | params . has ( " password " ) | | params . has ( " quota_key " ) )
throw Exception ( " Invalid authentication: it is not allowed to use X-ClickHouse HTTP headers and other authentication methods simultaneously " , ErrorCodes : : AUTHENTICATION_FAILED ) ;
}
if ( spnego_challenge . empty ( ) ) // I.e., now using user name and password strings ("Basic").
{
if ( ! request_credentials )
request_credentials = std : : make_unique < BasicCredentials > ( ) ;
auto * basic_credentials = dynamic_cast < BasicCredentials * > ( request_credentials . get ( ) ) ;
if ( ! basic_credentials )
throw Exception ( " Invalid authentication: unexpected 'Basic' HTTP Authorization scheme " , ErrorCodes : : AUTHENTICATION_FAILED ) ;
basic_credentials - > setUserName ( user ) ;
basic_credentials - > setPassword ( password ) ;
}
else
{
if ( ! request_credentials )
2021-08-01 14:12:34 +00:00
request_credentials = server . context ( ) - > makeGSSAcceptorContext ( ) ;
2021-03-11 20:41:10 +00:00
auto * gss_acceptor_context = dynamic_cast < GSSAcceptorContext * > ( request_credentials . get ( ) ) ;
if ( ! gss_acceptor_context )
throw Exception ( " Invalid authentication: unexpected 'Negotiate' HTTP Authorization scheme expected " , ErrorCodes : : AUTHENTICATION_FAILED ) ;
# pragma GCC diagnostic push
# pragma GCC diagnostic ignored "-Wunreachable-code"
const auto spnego_response = base64Encode ( gss_acceptor_context - > processToken ( base64Decode ( spnego_challenge ) , log ) ) ;
# pragma GCC diagnostic pop
if ( ! spnego_response . empty ( ) )
response . set ( " WWW-Authenticate " , " Negotiate " + spnego_response ) ;
if ( ! gss_acceptor_context - > isFailed ( ) & & ! gss_acceptor_context - > isReady ( ) )
2017-09-21 11:16:18 +00:00
{
2021-03-11 20:41:10 +00:00
if ( spnego_response . empty ( ) )
throw Exception ( " Invalid authentication: 'Negotiate' HTTP Authorization failure " , ErrorCodes : : AUTHENTICATION_FAILED ) ;
response . setStatusAndReason ( HTTPResponse : : HTTP_UNAUTHORIZED ) ;
response . send ( ) ;
return false ;
2017-09-21 11:16:18 +00:00
}
2017-04-01 07:20:54 +00:00
}
2020-12-02 19:20:47 +00:00
/// Set client info. It will be used for quota accounting parameters in 'setUser' method.
2021-08-01 14:12:34 +00:00
ClientInfo & client_info = session - > getClientInfo ( ) ;
2020-12-02 19:20:47 +00:00
ClientInfo : : HTTPMethod http_method = ClientInfo : : HTTPMethod : : UNKNOWN ;
2021-02-19 12:51:26 +00:00
if ( request . getMethod ( ) = = HTTPServerRequest : : HTTP_GET )
2020-12-02 19:20:47 +00:00
http_method = ClientInfo : : HTTPMethod : : GET ;
2021-02-19 12:51:26 +00:00
else if ( request . getMethod ( ) = = HTTPServerRequest : : HTTP_POST )
2020-12-02 19:20:47 +00:00
http_method = ClientInfo : : HTTPMethod : : POST ;
client_info . http_method = http_method ;
client_info . http_user_agent = request . get ( " User-Agent " , " " ) ;
2021-01-21 22:55:45 +00:00
client_info . http_referer = request . get ( " Referer " , " " ) ;
2020-12-02 19:20:47 +00:00
client_info . forwarded_for = request . get ( " X-Forwarded-For " , " " ) ;
2021-08-01 14:12:34 +00:00
client_info . quota_key = quota_key ;
2020-12-02 19:20:47 +00:00
2021-03-11 20:41:10 +00:00
try
{
2021-08-01 14:12:34 +00:00
session - > authenticate ( * request_credentials , request . clientAddress ( ) ) ;
2021-03-11 20:41:10 +00:00
}
catch ( const Authentication : : Require < BasicCredentials > & required_credentials )
{
request_credentials = std : : make_unique < BasicCredentials > ( ) ;
if ( required_credentials . getRealm ( ) . empty ( ) )
response . set ( " WWW-Authenticate " , " Basic " ) ;
else
response . set ( " WWW-Authenticate " , " Basic realm= \" " + required_credentials . getRealm ( ) + " \" " ) ;
response . setStatusAndReason ( HTTPResponse : : HTTP_UNAUTHORIZED ) ;
response . send ( ) ;
return false ;
}
catch ( const Authentication : : Require < GSSAcceptorContext > & required_credentials )
{
2021-08-01 14:12:34 +00:00
request_credentials = server . context ( ) - > makeGSSAcceptorContext ( ) ;
2021-03-11 20:41:10 +00:00
if ( required_credentials . getRealm ( ) . empty ( ) )
response . set ( " WWW-Authenticate " , " Negotiate " ) ;
else
response . set ( " WWW-Authenticate " , " Negotiate realm= \" " + required_credentials . getRealm ( ) + " \" " ) ;
response . setStatusAndReason ( HTTPResponse : : HTTP_UNAUTHORIZED ) ;
response . send ( ) ;
return false ;
}
request_credentials . reset ( ) ;
return true ;
}
void HTTPHandler : : processQuery (
HTTPServerRequest & request ,
HTMLForm & params ,
HTTPServerResponse & response ,
Output & used_output ,
std : : optional < CurrentThread : : QueryScope > & query_scope )
{
using namespace Poco : : Net ;
LOG_TRACE ( log , " Request URI: {} " , request . getURI ( ) ) ;
2021-03-05 14:57:16 +00:00
if ( ! authenticateUser ( request , params , response ) )
2021-03-11 20:41:10 +00:00
return ; // '401 Unauthorized' response with 'Negotiate' has been sent at this point.
2017-06-02 21:01:17 +00:00
/// The user could specify session identifier and session timeout.
/// It allows to modify settings, create temporary tables and reuse them in subsequent requests.
2017-06-02 18:48:33 +00:00
String session_id ;
2020-03-06 18:14:33 +00:00
std : : chrono : : steady_clock : : duration session_timeout ;
2017-06-02 18:48:33 +00:00
bool session_is_set = params . has ( " session_id " ) ;
2018-06-19 20:30:35 +00:00
const auto & config = server . config ( ) ;
2017-06-02 18:48:33 +00:00
if ( session_is_set )
{
session_id = params . get ( " session_id " ) ;
2017-09-08 11:57:43 +00:00
session_timeout = parseSessionTimeout ( config , params ) ;
2017-06-02 18:48:33 +00:00
std : : string session_check = params . get ( " session_check " , " " ) ;
2021-08-01 14:12:34 +00:00
session - > makeSessionContext ( session_id , session_timeout , session_check = = " 1 " ) ;
2017-06-02 18:48:33 +00:00
}
2020-10-19 21:26:10 +00:00
// Parse the OpenTelemetry traceparent header.
2020-10-20 08:13:21 +00:00
// Disable in Arcadia -- it interferes with the
// test_clickhouse.TestTracing.test_tracing_via_http_proxy[traceparent] test.
2021-08-01 14:12:34 +00:00
ClientInfo client_info = session - > getClientInfo ( ) ;
2020-10-20 08:13:21 +00:00
# if !defined(ARCADIA_BUILD)
2020-08-28 01:21:08 +00:00
if ( request . has ( " traceparent " ) )
{
std : : string opentelemetry_traceparent = request . get ( " traceparent " ) ;
std : : string error ;
2021-03-05 14:57:16 +00:00
if ( ! client_info . client_trace_context . parseTraceparentHeader (
2020-11-11 12:34:28 +00:00
opentelemetry_traceparent , error ) )
2020-08-28 01:21:08 +00:00
{
2020-11-11 12:34:28 +00:00
throw Exception ( ErrorCodes : : BAD_REQUEST_PARAMETER ,
2020-08-28 01:21:08 +00:00
" Failed to parse OpenTelemetry traceparent header '{}': {} " ,
2020-11-11 12:34:28 +00:00
opentelemetry_traceparent , error ) ;
2020-08-28 01:21:08 +00:00
}
2021-03-05 14:57:16 +00:00
client_info . client_trace_context . tracestate = request . get ( " tracestate " , " " ) ;
2020-08-28 01:21:08 +00:00
}
2020-10-20 08:13:21 +00:00
# endif
2020-08-28 01:21:08 +00:00
2021-08-01 14:12:34 +00:00
auto context = session - > makeQueryContext ( std : : move ( client_info ) ) ;
2020-12-02 19:20:47 +00:00
2017-04-01 07:20:54 +00:00
/// The client can pass a HTTP header indicating supported compression method (gzip or deflate).
String http_response_compression_methods = request . get ( " Accept-Encoding " , " " ) ;
2020-01-04 22:59:08 +00:00
CompressionMethod http_response_compression_method = CompressionMethod : : None ;
2017-04-01 07:20:54 +00:00
if ( ! http_response_compression_methods . empty ( ) )
{
2020-01-05 01:51:59 +00:00
/// If client supports brotli - it's preferred.
2017-04-01 07:20:54 +00:00
/// Both gzip and deflate are supported. If the client supports both, gzip is preferred.
/// NOTE parsing of the list of methods is slightly incorrect.
2020-01-05 01:51:59 +00:00
2020-01-05 01:54:58 +00:00
if ( std : : string : : npos ! = http_response_compression_methods . find ( " br " ) )
2020-01-05 01:51:59 +00:00
http_response_compression_method = CompressionMethod : : Brotli ;
else if ( std : : string : : npos ! = http_response_compression_methods . find ( " gzip " ) )
2019-02-13 20:54:12 +00:00
http_response_compression_method = CompressionMethod : : Gzip ;
2017-04-01 07:20:54 +00:00
else if ( std : : string : : npos ! = http_response_compression_methods . find ( " deflate " ) )
2019-02-13 20:54:12 +00:00
http_response_compression_method = CompressionMethod : : Zlib ;
2020-11-11 01:50:56 +00:00
else if ( std : : string : : npos ! = http_response_compression_methods . find ( " xz " ) )
http_response_compression_method = CompressionMethod : : Xz ;
2020-11-29 12:59:56 +00:00
else if ( std : : string : : npos ! = http_response_compression_methods . find ( " zstd " ) )
http_response_compression_method = CompressionMethod : : Zstd ;
2017-04-01 07:20:54 +00:00
}
2020-01-04 22:59:08 +00:00
bool client_supports_http_compression = http_response_compression_method ! = CompressionMethod : : None ;
2017-04-01 07:20:54 +00:00
/// Client can pass a 'compress' flag in the query string. In this case the query result is
/// compressed using internal algorithm. This is not reflected in HTTP headers.
bool internal_compression = params . getParsed < bool > ( " compress " , false ) ;
/// At least, we should postpone sending of first buffer_size result bytes
2020-11-11 12:34:28 +00:00
size_t buffer_size_total = std : : max (
params . getParsed < size_t > ( " buffer_size " , DBMS_DEFAULT_BUFFER_SIZE ) , static_cast < size_t > ( DBMS_DEFAULT_BUFFER_SIZE ) ) ;
2017-04-01 07:20:54 +00:00
/// If it is specified, the whole result will be buffered.
/// First ~buffer_size bytes will be buffered in memory, the remaining bytes will be stored in temporary file.
bool buffer_until_eof = params . getParsed < bool > ( " wait_end_of_query " , false ) ;
size_t buffer_size_http = DBMS_DEFAULT_BUFFER_SIZE ;
size_t buffer_size_memory = ( buffer_size_total > buffer_size_http ) ? buffer_size_total : 0 ;
2017-09-08 16:41:35 +00:00
unsigned keep_alive_timeout = config . getUInt ( " keep_alive_timeout " , 10 ) ;
2017-09-08 11:57:43 +00:00
2017-04-01 07:20:54 +00:00
used_output . out = std : : make_shared < WriteBufferFromHTTPServerResponse > (
2021-02-19 12:51:26 +00:00
response ,
2021-03-11 20:41:10 +00:00
request . getMethod ( ) = = HTTPRequest : : HTTP_HEAD ,
2021-02-19 12:51:26 +00:00
keep_alive_timeout ,
client_supports_http_compression ,
http_response_compression_method ) ;
2020-01-04 22:59:08 +00:00
2017-04-01 07:20:54 +00:00
if ( internal_compression )
used_output . out_maybe_compressed = std : : make_shared < CompressedWriteBuffer > ( * used_output . out ) ;
else
used_output . out_maybe_compressed = used_output . out ;
if ( buffer_size_memory > 0 | | buffer_until_eof )
{
CascadeWriteBuffer : : WriteBufferPtrs cascade_buffer1 ;
CascadeWriteBuffer : : WriteBufferConstructors cascade_buffer2 ;
if ( buffer_size_memory > 0 )
cascade_buffer1 . emplace_back ( std : : make_shared < MemoryWriteBuffer > ( buffer_size_memory ) ) ;
if ( buffer_until_eof )
{
2021-08-01 14:12:34 +00:00
const std : : string tmp_path ( server . context ( ) - > getTemporaryVolume ( ) - > getDisk ( ) - > getPath ( ) ) ;
2020-01-19 14:26:28 +00:00
const std : : string tmp_path_template ( tmp_path + " http_buffers/ " ) ;
2017-04-01 07:20:54 +00:00
2020-11-11 12:34:28 +00:00
auto create_tmp_disk_buffer = [ tmp_path_template ] ( const WriteBufferPtr & )
{
return WriteBufferFromTemporaryFile : : create ( tmp_path_template ) ;
} ;
2017-04-01 07:20:54 +00:00
cascade_buffer2 . emplace_back ( std : : move ( create_tmp_disk_buffer ) ) ;
}
else
{
2020-11-11 12:34:28 +00:00
auto push_memory_buffer_and_continue = [ next_buffer = used_output . out_maybe_compressed ] ( const WriteBufferPtr & prev_buf )
{
2020-05-18 08:08:55 +00:00
auto * prev_memory_buffer = typeid_cast < MemoryWriteBuffer * > ( prev_buf . get ( ) ) ;
2017-04-01 07:20:54 +00:00
if ( ! prev_memory_buffer )
throw Exception ( " Expected MemoryWriteBuffer " , ErrorCodes : : LOGICAL_ERROR ) ;
auto rdbuf = prev_memory_buffer - > tryGetReadBuffer ( ) ;
2020-11-11 12:34:28 +00:00
copyData ( * rdbuf , * next_buffer ) ;
2017-04-01 07:20:54 +00:00
return next_buffer ;
} ;
cascade_buffer2 . emplace_back ( push_memory_buffer_and_continue ) ;
}
2020-11-11 12:34:28 +00:00
used_output . out_maybe_delayed_and_compressed = std : : make_shared < CascadeWriteBuffer > (
std : : move ( cascade_buffer1 ) , std : : move ( cascade_buffer2 ) ) ;
2017-04-01 07:20:54 +00:00
}
else
{
used_output . out_maybe_delayed_and_compressed = used_output . out_maybe_compressed ;
}
/// Request body can be compressed using algorithm specified in the Content-Encoding header.
String http_request_compression_method_str = request . get ( " Content-Encoding " , " " ) ;
2021-02-19 12:51:26 +00:00
auto in_post = wrapReadBufferWithCompressionMethod (
wrapReadBufferReference ( request . getStream ( ) ) , chooseCompressionMethod ( { } , http_request_compression_method_str ) ) ;
2017-04-01 07:20:54 +00:00
/// The data can also be compressed using incompatible internal algorithm. This is indicated by
/// 'decompress' query parameter.
std : : unique_ptr < ReadBuffer > in_post_maybe_compressed ;
bool in_post_compressed = false ;
if ( params . getParsed < bool > ( " decompress " , false ) )
{
in_post_maybe_compressed = std : : make_unique < CompressedReadBuffer > ( * in_post ) ;
in_post_compressed = true ;
}
else
in_post_maybe_compressed = std : : move ( in_post ) ;
std : : unique_ptr < ReadBuffer > in ;
2020-11-11 12:34:28 +00:00
static const NameSet reserved_param_names { " compress " , " decompress " , " user " , " password " , " quota_key " , " query_id " , " stacktrace " ,
" buffer_size " , " wait_end_of_query " , " session_id " , " session_timeout " , " session_check " } ;
2018-06-01 15:32:27 +00:00
Names reserved_param_suffixes ;
2020-11-11 12:34:28 +00:00
auto param_could_be_skipped = [ & ] ( const String & name )
{
2020-06-14 00:56:13 +00:00
/// Empty parameter appears when URL like ?&a=b or a=b&&c=d. Just skip them for user's convenience.
if ( name . empty ( ) )
return true ;
2018-06-01 15:32:27 +00:00
if ( reserved_param_names . count ( name ) )
return true ;
for ( const String & suffix : reserved_param_suffixes )
{
if ( endsWith ( name , suffix ) )
return true ;
}
return false ;
} ;
2017-04-01 07:20:54 +00:00
/// Settings can be overridden in the query.
/// Some parameters (database, default_format, everything used in the code above) do not
/// belong to the Settings class.
/// 'readonly' setting values mean:
/// readonly = 0 - any query is allowed, client can change any setting.
/// readonly = 1 - only readonly queries are allowed, client can't change settings.
/// readonly = 2 - only readonly queries are allowed, client can change any setting except 'readonly'.
/// In theory if initially readonly = 0, the client can change any setting and then set readonly
/// to some other value.
2021-04-10 23:33:54 +00:00
const auto & settings = context - > getSettingsRef ( ) ;
2017-04-01 07:20:54 +00:00
/// Only readonly queries are allowed for HTTP GET requests.
2021-02-19 12:51:26 +00:00
if ( request . getMethod ( ) = = HTTPServerRequest : : HTTP_GET )
2017-04-01 07:20:54 +00:00
{
2018-03-11 00:15:26 +00:00
if ( settings . readonly = = 0 )
2021-04-10 23:33:54 +00:00
context - > setSetting ( " readonly " , 2 ) ;
2017-04-01 07:20:54 +00:00
}
2020-03-09 03:38:43 +00:00
bool has_external_data = startsWith ( request . getContentType ( ) , " multipart/form-data " ) ;
2019-05-28 21:31:19 +00:00
2019-06-15 18:25:27 +00:00
if ( has_external_data )
2019-05-28 21:31:19 +00:00
{
/// Skip unneeded parameters to avoid confusing them later with context settings or query parameters.
reserved_param_suffixes . reserve ( 3 ) ;
/// It is a bug and ambiguity with `date_time_input_format` and `low_cardinality_allow_in_native_format` formats/settings.
reserved_param_suffixes . emplace_back ( " _format " ) ;
reserved_param_suffixes . emplace_back ( " _types " ) ;
reserved_param_suffixes . emplace_back ( " _structure " ) ;
}
2020-07-28 09:58:12 +00:00
std : : string database = request . get ( " X-ClickHouse-Database " , " " ) ;
std : : string default_format = request . get ( " X-ClickHouse-Format " , " " ) ;
2019-04-18 23:29:32 +00:00
SettingsChanges settings_changes ;
2019-05-29 00:23:47 +00:00
for ( const auto & [ key , value ] : params )
2017-04-01 07:20:54 +00:00
{
2019-05-28 21:31:19 +00:00
if ( key = = " database " )
2017-04-01 07:20:54 +00:00
{
2020-07-28 09:58:12 +00:00
if ( database . empty ( ) )
database = value ;
2017-04-01 07:20:54 +00:00
}
2019-05-28 21:31:19 +00:00
else if ( key = = " default_format " )
2017-04-01 07:20:54 +00:00
{
2020-07-28 09:58:12 +00:00
if ( default_format . empty ( ) )
default_format = value ;
2017-04-01 07:20:54 +00:00
}
2019-05-28 21:31:19 +00:00
else if ( param_could_be_skipped ( key ) )
2017-04-01 07:20:54 +00:00
{
}
else
{
2020-04-04 08:57:16 +00:00
/// Other than query parameters are treated as settings.
if ( ! customizeQueryParam ( context , key , value ) )
settings_changes . push_back ( { key , value } ) ;
2017-04-01 07:20:54 +00:00
}
}
2020-07-28 09:58:12 +00:00
if ( ! database . empty ( ) )
2021-04-10 23:33:54 +00:00
context - > setCurrentDatabase ( database ) ;
2020-07-28 09:58:12 +00:00
if ( ! default_format . empty ( ) )
2021-04-10 23:33:54 +00:00
context - > setDefaultFormat ( default_format ) ;
2020-07-28 09:58:12 +00:00
2019-05-28 21:31:19 +00:00
/// For external data we also want settings
2021-04-10 23:33:54 +00:00
context - > checkSettingsConstraints ( settings_changes ) ;
context - > applySettingsChanges ( settings_changes ) ;
2019-04-18 23:29:32 +00:00
2021-08-01 14:12:34 +00:00
// Set the query id supplied by the user, if any, and also update the OpenTelemetry fields.
context - > setCurrentQueryId ( params . get ( " query_id " , request . get ( " X-ClickHouse-Query-Id " , " " ) ) ) ;
2021-08-16 10:58:52 +00:00
const auto & query = getQuery ( request , params , context ) ;
2020-04-04 08:57:16 +00:00
std : : unique_ptr < ReadBuffer > in_param = std : : make_unique < ReadBufferFromString > ( query ) ;
in = has_external_data ? std : : move ( in_param ) : std : : make_unique < ConcatReadBuffer > ( * in_param , * in_post_maybe_compressed ) ;
2019-05-28 21:31:19 +00:00
2017-04-01 07:20:54 +00:00
/// HTTP response compression is turned on only if the client signalled that they support it
/// (using Accept-Encoding header) and 'enable_http_compression' setting is turned on.
used_output . out - > setCompression ( client_supports_http_compression & & settings . enable_http_compression ) ;
if ( client_supports_http_compression )
used_output . out - > setCompressionLevel ( settings . http_zlib_compression_level ) ;
used_output . out - > setSendProgressInterval ( settings . http_headers_progress_interval_ms ) ;
/// If 'http_native_compression_disable_checksumming_on_decompress' setting is turned on,
/// checksums of client data compressed with internal algorithm are not checked.
if ( in_post_compressed & & settings . http_native_compression_disable_checksumming_on_decompress )
static_cast < CompressedReadBuffer & > ( * in_post_maybe_compressed ) . disableChecksumming ( ) ;
2021-10-09 12:16:37 +00:00
/// Add CORS header if 'add_http_cors_header' setting is turned on send * in Access-Control-Allow-Origin,
/// or if config has http_options_response, which means that there
/// are some headers to be sent, and the client passed Origin header.
if ( config . has ( " http_options_response " ) & & ! request . get ( " Origin " , " " ) . empty ( ) )
2021-09-22 16:54:36 +00:00
tryAddHeadersFromConfig ( response , config ) ;
2021-10-09 12:16:37 +00:00
else if ( settings . add_http_cors_header )
used_output . out - > addHeaderCORS ( settings . add_http_cors_header & & ! request . get ( " Origin " , " " ) . empty ( ) ) ;
2017-04-01 07:20:54 +00:00
2021-08-26 00:31:46 +00:00
auto append_callback = [ context = context ] ( ProgressCallback callback )
2020-11-11 12:34:28 +00:00
{
2021-04-10 23:33:54 +00:00
auto prev = context - > getProgressCallback ( ) ;
2019-02-01 01:48:25 +00:00
2021-04-10 23:33:54 +00:00
context - > setProgressCallback ( [ prev , callback ] ( const Progress & progress )
2020-11-11 12:34:28 +00:00
{
2019-02-01 01:48:25 +00:00
if ( prev )
prev ( progress ) ;
callback ( progress ) ;
} ) ;
} ;
2017-04-01 07:20:54 +00:00
/// While still no data has been sent, we will report about query execution progress by sending HTTP headers.
if ( settings . send_progress_in_http_headers )
2020-11-11 12:34:28 +00:00
append_callback ( [ & used_output ] ( const Progress & progress ) { used_output . out - > onProgress ( progress ) ; } ) ;
2019-02-01 01:48:25 +00:00
if ( settings . readonly > 0 & & settings . cancel_http_readonly_queries_on_client_close )
{
2021-08-26 00:31:46 +00:00
append_callback ( [ context = context , & request ] ( const Progress & )
2020-11-11 12:34:28 +00:00
{
2021-02-19 12:51:26 +00:00
/// Assume that at the point this method is called no one is reading data from the socket any more:
/// should be true for read-only queries.
if ( ! request . checkPeerConnected ( ) )
2021-04-10 23:33:54 +00:00
context - > killCurrentQuery ( ) ;
2019-02-01 01:48:25 +00:00
} ) ;
}
2017-04-01 07:20:54 +00:00
2020-04-21 11:30:45 +00:00
customizeContext ( request , context ) ;
2019-03-06 16:41:35 +00:00
2020-10-16 19:22:14 +00:00
query_scope . emplace ( context ) ;
2020-11-11 12:34:28 +00:00
executeQuery ( * in , * used_output . out_maybe_delayed_and_compressed , /* allow_into_outfile = */ false , context ,
[ & response ] ( const String & current_query_id , const String & content_type , const String & format , const String & timezone )
{
2020-01-21 19:03:33 +00:00
response . setContentType ( content_type ) ;
2020-03-03 15:32:41 +00:00
response . add ( " X-ClickHouse-Query-Id " , current_query_id ) ;
2020-01-21 19:03:33 +00:00
response . add ( " X-ClickHouse-Format " , format ) ;
2020-03-03 15:32:41 +00:00
response . add ( " X-ClickHouse-Timezone " , timezone ) ;
2020-11-11 12:34:28 +00:00
}
) ;
2017-04-01 07:20:54 +00:00
if ( used_output . hasDelayed ( ) )
{
/// TODO: set Content-Length if possible
pushDelayedResults ( used_output ) ;
}
2021-04-20 12:57:23 +00:00
/// Send HTTP headers with code 200 if no exception happened and the data is still not sent to the client.
2021-09-07 18:57:38 +00:00
used_output . finalize ( ) ;
2012-03-09 03:06:09 +00:00
}
2021-02-19 12:51:26 +00:00
void HTTPHandler : : trySendExceptionToClient (
const std : : string & s , int exception_code , HTTPServerRequest & request , HTTPServerResponse & response , Output & used_output )
2021-09-03 23:44:29 +00:00
try
2012-03-09 03:06:09 +00:00
{
2021-09-03 23:44:29 +00:00
response . set ( " X-ClickHouse-Exception-Code " , toString < int > ( exception_code ) ) ;
2021-02-19 12:51:26 +00:00
2021-09-03 23:44:29 +00:00
/// FIXME: make sure that no one else is reading from the same stream at the moment.
2017-04-01 07:20:54 +00:00
2021-09-03 23:44:29 +00:00
/// If HTTP method is POST and Keep-Alive is turned on, we should read the whole request body
/// to avoid reading part of the current request body in the next request.
if ( request . getMethod ( ) = = Poco : : Net : : HTTPRequest : : HTTP_POST
& & response . getKeepAlive ( )
& & exception_code ! = ErrorCodes : : HTTP_LENGTH_REQUIRED
& & ! request . getStream ( ) . eof ( ) )
2017-04-01 07:20:54 +00:00
{
2021-09-03 23:44:29 +00:00
request . getStream ( ) . ignoreAll ( ) ;
}
2020-01-23 13:40:16 +00:00
2021-09-03 23:44:29 +00:00
if ( exception_code = = ErrorCodes : : REQUIRED_PASSWORD )
{
response . requireAuthentication ( " ClickHouse server HTTP API " ) ;
}
else
{
response . setStatusAndReason ( exceptionCodeToHTTPStatus ( exception_code ) ) ;
}
2021-02-19 12:51:26 +00:00
2021-09-03 23:44:29 +00:00
if ( ! response . sent ( ) & & ! used_output . out_maybe_compressed )
{
/// If nothing was sent yet and we don't even know if we must compress the response.
* response . send ( ) < < s < < std : : endl ;
}
else if ( used_output . out_maybe_compressed )
{
/// Destroy CascadeBuffer to actualize buffers' positions and reset extra references
if ( used_output . hasDelayed ( ) )
used_output . out_maybe_delayed_and_compressed . reset ( ) ;
2017-04-01 07:20:54 +00:00
2021-09-03 23:44:29 +00:00
/// Send the error message into already used (and possibly compressed) stream.
/// Note that the error message will possibly be sent after some data.
/// Also HTTP code 200 could have already been sent.
2017-04-01 07:20:54 +00:00
2021-09-03 23:44:29 +00:00
/// If buffer has data, and that data wasn't sent yet, then no need to send that data
bool data_sent = used_output . out - > count ( ) ! = used_output . out - > offset ( ) ;
2017-04-01 07:20:54 +00:00
2021-09-03 23:44:29 +00:00
if ( ! data_sent )
2017-04-01 07:20:54 +00:00
{
2021-09-03 23:44:29 +00:00
used_output . out_maybe_compressed - > position ( ) = used_output . out_maybe_compressed - > buffer ( ) . begin ( ) ;
used_output . out - > position ( ) = used_output . out - > buffer ( ) . begin ( ) ;
2017-04-01 07:20:54 +00:00
}
2021-09-03 23:44:29 +00:00
writeString ( s , * used_output . out_maybe_compressed ) ;
writeChar ( ' \n ' , * used_output . out_maybe_compressed ) ;
2017-04-01 07:20:54 +00:00
2021-09-03 23:44:29 +00:00
used_output . out_maybe_compressed - > next ( ) ;
2017-04-01 07:20:54 +00:00
}
2021-09-03 23:44:29 +00:00
else
2017-04-01 07:20:54 +00:00
{
2021-09-03 23:44:29 +00:00
assert ( false ) ;
__builtin_unreachable ( ) ;
2017-04-01 07:20:54 +00:00
}
2021-09-07 18:57:38 +00:00
used_output . finalize ( ) ;
2014-07-14 19:46:06 +00:00
}
2021-09-03 23:44:29 +00:00
catch ( . . . )
{
tryLogCurrentException ( log , " Cannot send exception to client " ) ;
2017-04-01 07:20:54 +00:00
2021-09-07 18:57:38 +00:00
try
2021-09-03 23:44:29 +00:00
{
2021-09-07 18:57:38 +00:00
used_output . finalize ( ) ;
2017-04-01 07:20:54 +00:00
}
catch ( . . . )
{
2021-09-07 18:57:38 +00:00
tryLogCurrentException ( log , " Cannot flush data to client (after sending exception) " ) ;
2017-04-01 07:20:54 +00:00
}
2014-07-14 19:46:06 +00:00
}
2012-06-25 05:07:34 +00:00
2017-01-17 17:21:48 +00:00
2021-02-19 12:51:26 +00:00
void HTTPHandler : : handleRequest ( HTTPServerRequest & request , HTTPServerResponse & response )
2014-07-14 19:46:06 +00:00
{
2018-08-31 00:59:48 +00:00
setThreadName ( " HTTPHandler " ) ;
2019-01-15 18:39:54 +00:00
ThreadStatus thread_status ;
2018-08-31 00:59:48 +00:00
2021-08-01 14:12:34 +00:00
session = std : : make_unique < Session > ( server . context ( ) , ClientInfo : : Interface : : HTTP ) ;
SCOPE_EXIT ( { session . reset ( ) ; } ) ;
2020-10-16 19:22:14 +00:00
std : : optional < CurrentThread : : QueryScope > query_scope ;
2021-08-01 14:12:34 +00:00
2017-04-01 07:20:54 +00:00
Output used_output ;
2014-08-04 19:48:50 +00:00
2017-04-01 07:20:54 +00:00
/// In case of exception, send stack trace to client.
bool with_stacktrace = false ;
2016-06-25 07:22:12 +00:00
2017-04-01 07:20:54 +00:00
try
{
2021-09-18 09:36:02 +00:00
if ( request . getMethod ( ) = = HTTPServerRequest : : HTTP_OPTIONS )
{
processOptionsRequest ( response , server . config ( ) ) ;
return ;
}
2017-04-01 07:20:54 +00:00
response . setContentType ( " text/plain; charset=UTF-8 " ) ;
2018-03-08 07:36:58 +00:00
response . set ( " X-ClickHouse-Server-Display-Name " , server_display_name ) ;
2017-04-01 07:20:54 +00:00
/// For keep-alive to work.
2021-02-19 12:51:26 +00:00
if ( request . getVersion ( ) = = HTTPServerRequest : : HTTP_1_1 )
2017-04-01 07:20:54 +00:00
response . setChunkedTransferEncoding ( true ) ;
2014-07-14 19:46:06 +00:00
2021-08-01 14:12:34 +00:00
HTMLForm params ( default_settings , request ) ;
2017-04-01 07:20:54 +00:00
with_stacktrace = params . getParsed < bool > ( " stacktrace " , false ) ;
2016-06-25 07:22:12 +00:00
2021-04-10 23:33:54 +00:00
/// FIXME: maybe this check is already unnecessary.
2017-09-25 19:45:15 +00:00
/// Workaround. Poco does not detect 411 Length Required case.
2021-02-19 12:51:26 +00:00
if ( request . getMethod ( ) = = HTTPRequest : : HTTP_POST & & ! request . getChunkedTransferEncoding ( ) & & ! request . hasContentLength ( ) )
2017-09-25 19:45:15 +00:00
{
2021-02-19 12:51:26 +00:00
throw Exception (
" The Transfer-Encoding is not chunked and there is no Content-Length header for POST request " ,
ErrorCodes : : HTTP_LENGTH_REQUIRED ) ;
2017-09-25 19:45:15 +00:00
}
2021-03-05 14:57:16 +00:00
processQuery ( request , params , response , used_output , query_scope ) ;
2021-03-11 20:41:10 +00:00
LOG_DEBUG ( log , ( request_credentials ? " Authentication in progress... " : " Done processing query " ) ) ;
2017-04-01 07:20:54 +00:00
}
catch ( . . . )
{
2021-03-11 20:41:10 +00:00
SCOPE_EXIT ( {
request_credentials . reset ( ) ; // ...so that the next requests on the connection have to always start afresh in case of exceptions.
} ) ;
2017-04-01 07:20:54 +00:00
tryLogCurrentException ( log ) ;
2016-06-25 07:22:12 +00:00
2017-04-01 07:20:54 +00:00
/** If exception is received from remote server, then stack trace is embedded in message.
* If exception is thrown on local server , then stack trace is in separate field .
*/
std : : string exception_message = getCurrentExceptionMessage ( with_stacktrace , true ) ;
int exception_code = getCurrentExceptionCode ( ) ;
2016-06-25 07:22:12 +00:00
2017-04-01 07:20:54 +00:00
trySendExceptionToClient ( exception_message , exception_code , request , response , used_output ) ;
}
Fix abnormal server termination when http client goes away
In [1] stress tests found:
2021.02.12 14:20:58.800988 [ 17728 ] {de3e7894-b401-4f7d-8530-90cd5ab06682} <Debug> executeQuery: (from [::1]:45792, using production parser) (comment: /usr/share/clickhouse-test/queries/0_stateless/01520_client_print_query_id.expect) SELECT * FROM numbers(34599)
2021.02.12 14:20:58.916484 [ 17728 ] {de3e7894-b401-4f7d-8530-90cd5ab06682} <Trace> ContextAccess (default): Access granted: CREATE TEMPORARY TABLE ON *.*
2021.02.12 14:20:59.071980 [ 17728 ] {de3e7894-b401-4f7d-8530-90cd5ab06682} <Trace> InterpreterSelectQuery: FetchColumns -> Complete
2021.02.12 14:21:10.708202 [ 17728 ] {de3e7894-b401-4f7d-8530-90cd5ab06682} <Information> executeQuery: Read 34599 rows, 270.30 KiB in 11.876294055 sec., 2913 rows/sec., 22.76 KiB/sec.
2021.02.12 14:22:10.506261 [ 17728 ] {de3e7894-b401-4f7d-8530-90cd5ab06682} <Debug> DynamicQueryHandler: Done processing query
2021.02.12 14:22:18.238037 [ 375 ] {} <Fatal> BaseDaemon: (version 21.3.1.5996, build id: 8DBCED54529C989F7AD4D991F51410774D55DE6C) (from thread 17728) Terminate called for uncaught exception:
Code: 24, e.displayText() = DB::Exception: Cannot write to ostream at offset 262994, Stack trace (when copying this message, always include the lines below):
0. ./obj-x86_64-linux-gnu/../contrib/libcxx/include/exception:0: Poco::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int) @ 0x15c976cb in /usr/bin/clickhouse
1. ./obj-x86_64-linux-gnu/../src/Common/Exception.cpp:56: DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, bool) @ 0x8c9320e in /usr/bin/clickhouse
2. ./obj-x86_64-linux-gnu/../src/IO/WriteBufferFromOStream.cpp:0: DB::WriteBufferFromOStream::nextImpl() @ 0x8d54da5 in /usr/bin/clickhouse
3. ./obj-x86_64-linux-gnu/../src/IO/BufferBase.h:39: DB::WriteBufferFromOStream::~WriteBufferFromOStream() @ 0x8d551d7 in /usr/bin/clickhouse
4. ./obj-x86_64-linux-gnu/../src/IO/WriteBufferFromOStream.cpp:44: DB::Write
2021.02.12 14:22:18.811071 [ 18134 ] {} <Fatal> BaseDaemon: ########################################
2021.02.12 14:22:18.878935 [ 18134 ] {} <Fatal> BaseDaemon: (version 21.3.1.5996, build id: 8DBCED54529C989F7AD4D991F51410774D55DE6C) (from thread 17728) (query_id: de3e7894-b401-4f7d-8530-90cd5ab06682) Received signal Aborted (6)
2021.02.12 14:22:18.943148 [ 18134 ] {} <Fatal> BaseDaemon:
2021.02.12 14:22:19.007073 [ 18134 ] {} <Fatal> BaseDaemon: Stack trace: 0x7f109932018b 0x7f10992ff859 0x8bb33ae 0x8e301dd 0x17dac8c4 0x17dac7c7 0x8c3fe0b 0x8d552c5 0x8d552ea 0x11a29914 0x11a2a2ca 0x12f96092 0x12f8c65e 0x12f84300 0x15b84110 0x15bc0913 0x15bc103f 0x15d29a12 0x15d27fb0 0x15d267b8 0x8badbad 0x7f10994d5609 0x7f10993fc293
2021.02.12 14:22:19.255998 [ 18134 ] {} <Fatal> BaseDaemon: 5. raise @ 0x4618b in /usr/lib/x86_64-linux-gnu/libc-2.31.so
2021.02.12 14:22:19.270203 [ 18134 ] {} <Fatal> BaseDaemon: 6. abort @ 0x25859 in /usr/lib/x86_64-linux-gnu/libc-2.31.so
2021.02.12 14:22:50.108918 [ 370 ] {} <Fatal> Application: Child process was terminated by signal 6.
[1]: https://clickhouse-test-reports.s3.yandex.net/19580/6aecb62416ece880cbb8ee3a803e14d841388dde/stress_test_(thread).html#fail1
Verified locally by commenting out->next() call in
WriteBufferFromHTTPServerResponse::nextImpl(), adding a sleep(1) and
canceling HTTP request before it finished, the stacktrace as follow:
[ 6351 ] {} <Fatal> BaseDaemon: (version 21.3.1.1, build id: 9B40466BF3D2F5AED78A52A995A4A2FD3116787C) (from thread 6677) Terminate called for uncaught exception:
Code: 24, e.displayText() = DB::Exception: Cannot write to ostream at offset 4, Stack trace (when copying this message, always include the lines below):
0. /src/ch/clickhouse/.cmake/../src/Common/StackTrace.cpp:298: StackTrace::tryCapture() @ 0x30a52a in /src/ch/clickhouse/.cmake/src/libclickhouse_common_iod.so
1. /src/ch/clickhouse/.cmake/../src/Common/StackTrace.cpp:260: StackTrace::StackTrace() @ 0x30a4e5 in /src/ch/clickhouse/.cmake/src/libclickhouse_common_iod.so
2. /src/ch/clickhouse/.cmake/../src/Common/Exception.cpp:53: DB::Exception::Exception(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, int, bool) @ 0x2a61ae in /src/ch/clickhouse/.cmake/src/libclickhouse_common_iod.so
3. /src/ch/clickhouse/.cmake/../src/IO/WriteBufferFromOStream.cpp:22: DB::WriteBufferFromOStream::nextImpl() @ 0x3b468a in /src/ch/clickhouse/.cmake/src/libclickhouse_common_iod.so
4. /src/ch/clickhouse/.cmake/../src/IO/WriteBuffer.h:47
<snip>
[ 8966 ] {} <Fatal> BaseDaemon: 7. __cxxabiv1::__terminate(void (*)()) @ 0x1784ca in /src/ch/clickhouse/.cmake/contrib/replxx-cmake/libreplxxd.so
<snip>
[ 8966 ] {} <Fatal> BaseDaemon: 10. /src/ch/clickhouse/.cmake/../src/IO/WriteBufferFromOStream.cpp:0: DB::WriteBufferFromOStream::~WriteBufferFromOStream() @ 0x3b48c1 in /src/ch/clickhouse/.cmake/src/libclickhouse_common_iod.so
[ 8966 ] {} <Fatal> BaseDaemon: 11. /src/ch/clickhouse/.cmake/../src/IO/WriteBufferFromOStream.cpp:44: DB::WriteBufferFromOStream::~WriteBufferFromOStream() @ 0x3b48ec in /src/ch/clickhouse/.cmake/src/libclickhouse_common_iod.so
<snip>
[ 8966 ] {} <Fatal> BaseDaemon: 14. /src/ch/clickhouse/.cmake/../src/IO/WriteBufferFromHTTPServerResponse.cpp:218: DB::WriteBufferFromHTTPServerResponse::~WriteBufferFromHTTPServerResponse() @ 0x3b33cd in /src/ch/clickhouse/.cmake/src/libclickhouse_common_iod.so
<snip>
[ 8966 ] {} <Fatal> BaseDaemon: 22. /src/ch/clickhouse/.cmake/../src/Server/HTTPHandler.h:43: DB::HTTPHandler::Output::~Output() @ 0x260421 in /src/ch/clickhouse/.cmake/src/libclickhouse_serverd.so
[ 8966 ] {} <Fatal> BaseDaemon: 23. /src/ch/clickhouse/.cmake/../src/Server/HTTPHandler.cpp:778: DB::HTTPHandler::handleRequest(Poco::Net::HTTPServerRequest&, Poco::Net::HTTPServerResponse&) @ 0x253fd4 in /src/ch/clickhouse/.cmake/src/libclickhouse_serverd.so
2021-02-13 09:04:03 +00:00
2021-09-07 18:57:38 +00:00
used_output . finalize ( ) ;
2012-03-09 03:06:09 +00:00
}
2020-04-04 08:57:16 +00:00
DynamicQueryHandler : : DynamicQueryHandler ( IServer & server_ , const std : : string & param_name_ )
: HTTPHandler ( server_ , " DynamicQueryHandler " ) , param_name ( param_name_ )
2019-11-07 10:24:30 +00:00
{
2020-04-04 08:57:16 +00:00
}
2019-11-07 10:24:30 +00:00
2021-05-31 14:49:02 +00:00
bool DynamicQueryHandler : : customizeQueryParam ( ContextMutablePtr context , const std : : string & key , const std : : string & value )
2020-04-04 08:57:16 +00:00
{
if ( key = = param_name )
2020-11-11 12:34:28 +00:00
return true ; /// do nothing
2020-04-04 08:57:16 +00:00
if ( startsWith ( key , " param_ " ) )
2019-11-07 10:24:30 +00:00
{
2020-04-04 08:57:16 +00:00
/// Save name and values of substitution in dictionary.
const String parameter_name = key . substr ( strlen ( " param_ " ) ) ;
2021-08-16 10:58:52 +00:00
2021-08-16 21:54:40 +00:00
if ( ! context - > getQueryParameters ( ) . contains ( parameter_name ) )
2021-08-16 10:58:52 +00:00
context - > setQueryParameter ( parameter_name , value ) ;
2020-04-04 08:57:16 +00:00
return true ;
2017-04-01 07:20:54 +00:00
}
2019-11-07 10:24:30 +00:00
2020-04-04 08:57:16 +00:00
return false ;
2012-03-09 03:06:09 +00:00
}
2021-05-31 14:49:02 +00:00
std : : string DynamicQueryHandler : : getQuery ( HTTPServerRequest & request , HTMLForm & params , ContextMutablePtr context )
2020-04-04 08:57:16 +00:00
{
if ( likely ( ! startsWith ( request . getContentType ( ) , " multipart/form-data " ) ) )
{
/// Part of the query can be passed in the 'query' parameter and the rest in the request body
/// (http method need not necessarily be POST). In this case the entire query consists of the
/// contents of the 'query' parameter, a line break and the request body.
std : : string query_param = params . get ( param_name , " " ) ;
return query_param . empty ( ) ? query_param : query_param + " \n " ;
}
/// Support for "external data for query processing".
/// Used in case of POST request with form-data, but it isn't expected to be deleted after that scope.
ExternalTablesHandler handler ( context , params ) ;
2021-02-19 12:51:26 +00:00
params . load ( request , request . getStream ( ) , handler ) ;
2020-04-04 08:57:16 +00:00
std : : string full_query ;
/// Params are of both form params POST and uri (GET params)
for ( const auto & it : params )
2021-08-16 10:58:52 +00:00
{
2020-04-04 08:57:16 +00:00
if ( it . first = = param_name )
2021-08-16 10:58:52 +00:00
{
2020-04-04 08:57:16 +00:00
full_query + = it . second ;
2021-08-16 10:58:52 +00:00
}
else
{
customizeQueryParam ( context , it . first , it . second ) ;
}
}
2020-04-04 08:57:16 +00:00
return full_query ;
}
2020-04-27 20:51:39 +00:00
PredefinedQueryHandler : : PredefinedQueryHandler (
2021-04-10 23:33:54 +00:00
IServer & server_ ,
const NameSet & receive_params_ ,
const std : : string & predefined_query_ ,
const CompiledRegexPtr & url_regex_ ,
const std : : unordered_map < String , CompiledRegexPtr > & header_name_with_regex_ )
: HTTPHandler ( server_ , " PredefinedQueryHandler " )
, receive_params ( receive_params_ )
, predefined_query ( predefined_query_ )
, url_regex ( url_regex_ )
, header_name_with_capture_regex ( header_name_with_regex_ )
2020-04-04 08:57:16 +00:00
{
}
2021-05-31 14:49:02 +00:00
bool PredefinedQueryHandler : : customizeQueryParam ( ContextMutablePtr context , const std : : string & key , const std : : string & value )
2020-04-04 08:57:16 +00:00
{
if ( receive_params . count ( key ) )
{
2021-04-10 23:33:54 +00:00
context - > setQueryParameter ( key , value ) ;
2020-04-04 08:57:16 +00:00
return true ;
}
return false ;
}
2021-05-31 14:49:02 +00:00
void PredefinedQueryHandler : : customizeContext ( HTTPServerRequest & request , ContextMutablePtr context )
2020-04-21 11:30:45 +00:00
{
/// If in the configuration file, the handler's header is regex and contains named capture group
/// We will extract regex named capture groups as query parameters
2020-11-11 12:34:28 +00:00
const auto & set_query_params = [ & ] ( const char * begin , const char * end , const CompiledRegexPtr & compiled_regex )
{
2020-04-21 11:30:45 +00:00
int num_captures = compiled_regex - > NumberOfCapturingGroups ( ) + 1 ;
2020-04-28 07:19:37 +00:00
re2 : : StringPiece matches [ num_captures ] ;
re2 : : StringPiece input ( begin , end - begin ) ;
if ( compiled_regex - > Match ( input , 0 , end - begin , re2 : : RE2 : : Anchor : : ANCHOR_BOTH , matches , num_captures ) )
2020-04-21 11:30:45 +00:00
{
for ( const auto & [ capturing_name , capturing_index ] : compiled_regex - > NamedCapturingGroups ( ) )
{
const auto & capturing_value = matches [ capturing_index ] ;
if ( capturing_value . data ( ) )
2021-04-10 23:33:54 +00:00
context - > setQueryParameter ( capturing_name , String ( capturing_value . data ( ) , capturing_value . size ( ) ) ) ;
2020-04-21 11:30:45 +00:00
}
}
} ;
2020-04-26 11:06:14 +00:00
if ( url_regex )
{
const auto & uri = request . getURI ( ) ;
2020-04-27 22:19:24 +00:00
set_query_params ( uri . data ( ) , find_first_symbols < ' ? ' > ( uri . data ( ) , uri . data ( ) + uri . size ( ) ) , url_regex ) ;
2020-04-26 11:06:14 +00:00
}
2020-04-21 11:30:45 +00:00
for ( const auto & [ header_name , regex ] : header_name_with_capture_regex )
{
2020-04-26 11:06:14 +00:00
const auto & header_value = request . get ( header_name ) ;
set_query_params ( header_value . data ( ) , header_value . data ( ) + header_value . size ( ) , regex ) ;
2020-04-21 11:30:45 +00:00
}
}
2021-05-31 14:49:02 +00:00
std : : string PredefinedQueryHandler : : getQuery ( HTTPServerRequest & request , HTMLForm & params , ContextMutablePtr context )
2020-04-04 08:57:16 +00:00
{
if ( unlikely ( startsWith ( request . getContentType ( ) , " multipart/form-data " ) ) )
{
/// Support for "external data for query processing".
ExternalTablesHandler handler ( context , params ) ;
2021-02-19 12:51:26 +00:00
params . load ( request , request . getStream ( ) , handler ) ;
2020-04-04 08:57:16 +00:00
}
2020-04-27 20:51:39 +00:00
return predefined_query ;
2020-04-04 08:57:16 +00:00
}
2021-02-19 12:51:26 +00:00
HTTPRequestHandlerFactoryPtr createDynamicHandlerFactory ( IServer & server , const std : : string & config_prefix )
2020-04-04 08:57:16 +00:00
{
2021-02-19 12:51:26 +00:00
const auto & query_param_name = server . config ( ) . getString ( config_prefix + " .handler.query_param_name " , " query " ) ;
auto factory = std : : make_shared < HandlingRuleHTTPHandlerFactory < DynamicQueryHandler > > ( server , std : : move ( query_param_name ) ) ;
factory - > addFiltersFromConfig ( server . config ( ) , config_prefix ) ;
return factory ;
2020-04-04 08:57:16 +00:00
}
2020-04-27 22:19:24 +00:00
static inline bool capturingNamedQueryParam ( NameSet receive_params , const CompiledRegexPtr & compiled_regex )
2020-04-26 11:06:14 +00:00
{
const auto & capturing_names = compiled_regex - > NamedCapturingGroups ( ) ;
2020-11-11 12:34:28 +00:00
return std : : count_if ( capturing_names . begin ( ) , capturing_names . end ( ) , [ & ] ( const auto & iterator )
{
return std : : count_if ( receive_params . begin ( ) , receive_params . end ( ) ,
[ & ] ( const auto & param_name ) { return param_name = = iterator . first ; } ) ;
2020-04-26 11:06:14 +00:00
} ) ;
}
2020-04-04 08:57:16 +00:00
2020-04-27 22:19:24 +00:00
static inline CompiledRegexPtr getCompiledRegex ( const std : : string & expression )
{
2020-04-28 07:19:37 +00:00
auto compiled_regex = std : : make_shared < const re2 : : RE2 > ( expression ) ;
2020-04-27 22:19:24 +00:00
if ( ! compiled_regex - > ok ( ) )
2021-02-19 12:51:26 +00:00
throw Exception (
" Cannot compile re2: " + expression + " for http handling rule, error: " + compiled_regex - > error ( )
+ " . Look at https://github.com/google/re2/wiki/Syntax for reference. " ,
ErrorCodes : : CANNOT_COMPILE_REGEXP ) ;
2020-04-27 22:19:24 +00:00
return compiled_regex ;
}
2021-02-19 12:51:26 +00:00
HTTPRequestHandlerFactoryPtr createPredefinedHandlerFactory ( IServer & server , const std : : string & config_prefix )
2020-04-04 08:57:16 +00:00
{
2020-04-26 11:06:14 +00:00
Poco : : Util : : AbstractConfiguration & configuration = server . config ( ) ;
if ( ! configuration . has ( config_prefix + " .handler.query " ) )
2021-02-19 12:51:26 +00:00
throw Exception ( " There is no path ' " + config_prefix + " .handler.query' in configuration file. " , ErrorCodes : : NO_ELEMENTS_IN_CONFIG ) ;
2020-04-04 08:57:16 +00:00
2020-04-27 20:51:39 +00:00
std : : string predefined_query = configuration . getString ( config_prefix + " .handler.query " ) ;
NameSet analyze_receive_params = analyzeReceiveQueryParams ( predefined_query ) ;
2020-04-21 11:30:45 +00:00
2020-04-27 22:19:24 +00:00
std : : unordered_map < String , CompiledRegexPtr > headers_name_with_regex ;
2020-04-26 11:06:14 +00:00
Poco : : Util : : AbstractConfiguration : : Keys headers_name ;
configuration . keys ( config_prefix + " .headers " , headers_name ) ;
2020-04-21 11:30:45 +00:00
2020-04-26 11:06:14 +00:00
for ( const auto & header_name : headers_name )
2020-04-21 11:30:45 +00:00
{
2020-04-26 11:06:14 +00:00
auto expression = configuration . getString ( config_prefix + " .headers. " + header_name ) ;
2020-04-04 08:57:16 +00:00
2020-04-26 11:06:14 +00:00
if ( ! startsWith ( expression , " regex: " ) )
continue ;
expression = expression . substr ( 6 ) ;
2020-04-27 22:19:24 +00:00
auto regex = getCompiledRegex ( expression ) ;
if ( capturingNamedQueryParam ( analyze_receive_params , regex ) )
headers_name_with_regex . emplace ( std : : make_pair ( header_name , regex ) ) ;
2020-04-21 11:30:45 +00:00
}
2020-04-26 11:06:14 +00:00
2021-02-19 12:51:26 +00:00
std : : shared_ptr < HandlingRuleHTTPHandlerFactory < PredefinedQueryHandler > > factory ;
2020-04-26 11:06:14 +00:00
if ( configuration . has ( config_prefix + " .url " ) )
{
auto url_expression = configuration . getString ( config_prefix + " .url " ) ;
if ( startsWith ( url_expression , " regex: " ) )
url_expression = url_expression . substr ( 6 ) ;
2020-04-27 22:19:24 +00:00
auto regex = getCompiledRegex ( url_expression ) ;
if ( capturingNamedQueryParam ( analyze_receive_params , regex ) )
2021-02-19 12:51:26 +00:00
{
factory = std : : make_shared < HandlingRuleHTTPHandlerFactory < PredefinedQueryHandler > > (
server ,
std : : move ( analyze_receive_params ) ,
std : : move ( predefined_query ) ,
std : : move ( regex ) ,
std : : move ( headers_name_with_regex ) ) ;
factory - > addFiltersFromConfig ( configuration , config_prefix ) ;
return factory ;
}
2020-11-11 12:34:28 +00:00
}
2021-02-19 12:51:26 +00:00
factory = std : : make_shared < HandlingRuleHTTPHandlerFactory < PredefinedQueryHandler > > (
server , std : : move ( analyze_receive_params ) , std : : move ( predefined_query ) , CompiledRegexPtr { } , std : : move ( headers_name_with_regex ) ) ;
factory - > addFiltersFromConfig ( configuration , config_prefix ) ;
return factory ;
2020-04-26 11:06:14 +00:00
}
2012-03-09 03:06:09 +00:00
}