2019-02-15 11:46:07 +00:00
# include "HTTPHandler.h"
2020-04-04 08:57:16 +00:00
# include "HTTPHandlerFactory.h"
# include "HTTPHandlerRequestFilter.h"
2017-06-02 18:48:33 +00:00
# include <chrono>
2012-03-09 04:45:27 +00:00
# include <iomanip>
2017-02-09 10:10:13 +00:00
# include <Poco/File.h>
2017-08-09 14:33:07 +00:00
# include <Poco/Net/HTTPBasicCredentials.h>
# include <Poco/Net/HTTPServerRequest.h>
2019-02-01 01:48:25 +00:00
# include <Poco/Net/HTTPServerRequestImpl.h>
2017-08-09 14:33:07 +00:00
# include <Poco/Net/HTTPServerResponse.h>
2020-11-11 12:34:28 +00:00
# include <Poco/Net/HTTPRequestHandlerFactory.h>
2017-08-09 14:33:07 +00:00
# include <Poco/Net/NetException.h>
2020-11-11 12:34:28 +00:00
# include <ext/scope_guard.h>
# include <Core/ExternalTable.h>
2018-01-15 19:07:47 +00:00
# include <Common/StringUtils/StringUtils.h>
2017-04-01 09:19:00 +00:00
# include <Common/escapeForFileName.h>
2020-11-11 12:34:28 +00:00
# include <common/getFQDNOrHostName.h>
2018-08-31 00:59:48 +00:00
# include <Common/setThreadName.h>
2020-11-11 12:34:28 +00:00
# include <Common/SettingsChanges.h>
2021-01-23 15:20:15 +00:00
# include <Disks/IVolume.h>
2020-11-11 12:34:28 +00:00
# include <Compression/CompressedReadBuffer.h>
# include <Compression/CompressedWriteBuffer.h>
# include <IO/ReadBufferFromIStream.h>
# include <IO/ReadBufferFromString.h>
# include <IO/WriteBufferFromString.h>
# include <IO/WriteBufferFromHTTPServerResponse.h>
# include <IO/WriteBufferFromFile.h>
# include <IO/WriteHelpers.h>
# include <IO/copyData.h>
# include <IO/ConcatReadBuffer.h>
# include <IO/CascadeWriteBuffer.h>
# include <IO/MemoryReadWriteBuffer.h>
# include <IO/WriteBufferFromTemporaryFile.h>
# include <DataStreams/IBlockInputStream.h>
# include <Interpreters/executeQuery.h>
# include <Interpreters/QueryParameterVisitor.h>
2020-12-10 22:05:02 +00:00
# include <Interpreters/Context.h>
2017-07-13 20:58:19 +00:00
# include <Common/typeid_cast.h>
2020-11-11 12:34:28 +00:00
# include <Poco/Net/HTTPStream.h>
2017-09-23 23:32:26 +00:00
2020-04-16 12:31:57 +00:00
# if !defined(ARCADIA_BUILD)
# include <Common / config.h>
# endif
2012-03-09 03:06:09 +00:00
namespace DB
{
2020-11-11 12:34:28 +00:00
2016-01-11 21:46:36 +00:00
namespace ErrorCodes
{
2020-11-11 12:34:28 +00:00
2020-02-25 18:02:41 +00:00
extern const int LOGICAL_ERROR ;
2017-04-01 07:20:54 +00:00
extern const int CANNOT_PARSE_TEXT ;
extern const int CANNOT_PARSE_ESCAPE_SEQUENCE ;
extern const int CANNOT_PARSE_QUOTED_STRING ;
extern const int CANNOT_PARSE_DATE ;
extern const int CANNOT_PARSE_DATETIME ;
extern const int CANNOT_PARSE_NUMBER ;
2020-05-03 19:47:07 +00:00
extern const int CANNOT_PARSE_INPUT_ASSERTION_FAILED ;
2017-04-01 07:20:54 +00:00
extern const int CANNOT_OPEN_FILE ;
2020-04-26 11:06:14 +00:00
extern const int CANNOT_COMPILE_REGEXP ;
2017-04-01 07:20:54 +00:00
extern const int UNKNOWN_ELEMENT_IN_AST ;
extern const int UNKNOWN_TYPE_OF_AST_NODE ;
extern const int TOO_DEEP_AST ;
extern const int TOO_BIG_AST ;
extern const int UNEXPECTED_AST_STRUCTURE ;
2018-10-29 18:00:36 +00:00
extern const int SYNTAX_ERROR ;
2019-08-01 13:16:09 +00:00
extern const int INCORRECT_DATA ;
extern const int TYPE_MISMATCH ;
2017-04-01 07:20:54 +00:00
extern const int UNKNOWN_TABLE ;
extern const int UNKNOWN_FUNCTION ;
extern const int UNKNOWN_IDENTIFIER ;
extern const int UNKNOWN_TYPE ;
extern const int UNKNOWN_STORAGE ;
extern const int UNKNOWN_DATABASE ;
extern const int UNKNOWN_SETTING ;
extern const int UNKNOWN_DIRECTION_OF_SORTING ;
extern const int UNKNOWN_AGGREGATE_FUNCTION ;
extern const int UNKNOWN_FORMAT ;
extern const int UNKNOWN_DATABASE_ENGINE ;
extern const int UNKNOWN_TYPE_OF_QUERY ;
2020-04-04 08:57:16 +00:00
extern const int NO_ELEMENTS_IN_CONFIG ;
2017-04-01 07:20:54 +00:00
extern const int QUERY_IS_TOO_LARGE ;
extern const int NOT_IMPLEMENTED ;
extern const int SOCKET_TIMEOUT ;
extern const int UNKNOWN_USER ;
extern const int WRONG_PASSWORD ;
extern const int REQUIRED_PASSWORD ;
2017-06-02 18:48:33 +00:00
2020-08-28 01:21:08 +00:00
extern const int BAD_REQUEST_PARAMETER ;
2017-06-02 18:48:33 +00:00
extern const int INVALID_SESSION_TIMEOUT ;
2017-09-25 19:45:15 +00:00
extern const int HTTP_LENGTH_REQUIRED ;
2016-01-11 21:46:36 +00:00
}
2017-06-02 21:01:17 +00:00
2017-02-07 05:21:31 +00:00
static Poco : : Net : : HTTPResponse : : HTTPStatus exceptionCodeToHTTPStatus ( int exception_code )
2017-02-06 08:23:02 +00:00
{
2017-04-01 07:20:54 +00:00
using namespace Poco : : Net ;
2019-08-02 05:07:10 +00:00
if ( exception_code = = ErrorCodes : : REQUIRED_PASSWORD )
2020-05-03 19:47:07 +00:00
{
2019-08-02 05:07:10 +00:00
return HTTPResponse : : HTTP_UNAUTHORIZED ;
2020-05-03 19:47:07 +00:00
}
2020-11-11 12:34:28 +00:00
else if ( exception_code = = ErrorCodes : : CANNOT_PARSE_TEXT | |
exception_code = = ErrorCodes : : CANNOT_PARSE_ESCAPE_SEQUENCE | |
exception_code = = ErrorCodes : : CANNOT_PARSE_QUOTED_STRING | |
exception_code = = ErrorCodes : : CANNOT_PARSE_DATE | |
exception_code = = ErrorCodes : : CANNOT_PARSE_DATETIME | |
exception_code = = ErrorCodes : : CANNOT_PARSE_NUMBER | |
exception_code = = ErrorCodes : : CANNOT_PARSE_INPUT_ASSERTION_FAILED | |
exception_code = = ErrorCodes : : UNKNOWN_ELEMENT_IN_AST | |
exception_code = = ErrorCodes : : UNKNOWN_TYPE_OF_AST_NODE | |
exception_code = = ErrorCodes : : TOO_DEEP_AST | |
exception_code = = ErrorCodes : : TOO_BIG_AST | |
exception_code = = ErrorCodes : : UNEXPECTED_AST_STRUCTURE | |
exception_code = = ErrorCodes : : SYNTAX_ERROR | |
exception_code = = ErrorCodes : : INCORRECT_DATA | |
exception_code = = ErrorCodes : : TYPE_MISMATCH )
2020-05-03 19:47:07 +00:00
{
2019-08-02 05:07:10 +00:00
return HTTPResponse : : HTTP_BAD_REQUEST ;
2020-05-03 19:47:07 +00:00
}
2020-11-11 12:34:28 +00:00
else if ( exception_code = = ErrorCodes : : UNKNOWN_TABLE | |
exception_code = = ErrorCodes : : UNKNOWN_FUNCTION | |
exception_code = = ErrorCodes : : UNKNOWN_IDENTIFIER | |
exception_code = = ErrorCodes : : UNKNOWN_TYPE | |
exception_code = = ErrorCodes : : UNKNOWN_STORAGE | |
exception_code = = ErrorCodes : : UNKNOWN_DATABASE | |
exception_code = = ErrorCodes : : UNKNOWN_SETTING | |
exception_code = = ErrorCodes : : UNKNOWN_DIRECTION_OF_SORTING | |
exception_code = = ErrorCodes : : UNKNOWN_AGGREGATE_FUNCTION | |
exception_code = = ErrorCodes : : UNKNOWN_FORMAT | |
exception_code = = ErrorCodes : : UNKNOWN_DATABASE_ENGINE | |
exception_code = = ErrorCodes : : UNKNOWN_TYPE_OF_QUERY )
2020-05-03 19:47:07 +00:00
{
2019-08-02 05:07:10 +00:00
return HTTPResponse : : HTTP_NOT_FOUND ;
2020-05-03 19:47:07 +00:00
}
2019-08-02 05:07:10 +00:00
else if ( exception_code = = ErrorCodes : : QUERY_IS_TOO_LARGE )
2020-05-03 19:47:07 +00:00
{
2019-08-02 05:07:10 +00:00
return HTTPResponse : : HTTP_REQUESTENTITYTOOLARGE ;
2020-05-03 19:47:07 +00:00
}
2019-08-02 05:07:10 +00:00
else if ( exception_code = = ErrorCodes : : NOT_IMPLEMENTED )
2020-05-03 19:47:07 +00:00
{
2019-08-02 05:07:10 +00:00
return HTTPResponse : : HTTP_NOT_IMPLEMENTED ;
2020-05-03 19:47:07 +00:00
}
2020-11-11 12:34:28 +00:00
else if ( exception_code = = ErrorCodes : : SOCKET_TIMEOUT | |
exception_code = = ErrorCodes : : CANNOT_OPEN_FILE )
2020-05-03 19:47:07 +00:00
{
2019-08-02 05:07:10 +00:00
return HTTPResponse : : HTTP_SERVICE_UNAVAILABLE ;
2020-05-03 19:47:07 +00:00
}
2019-08-02 05:07:10 +00:00
else if ( exception_code = = ErrorCodes : : HTTP_LENGTH_REQUIRED )
2020-05-03 19:47:07 +00:00
{
2019-08-02 05:07:10 +00:00
return HTTPResponse : : HTTP_LENGTH_REQUIRED ;
2020-05-03 19:47:07 +00:00
}
2019-08-02 05:07:10 +00:00
return HTTPResponse : : HTTP_INTERNAL_SERVER_ERROR ;
2017-02-06 08:23:02 +00:00
}
2017-06-02 18:48:33 +00:00
2020-11-11 12:34:28 +00:00
static std : : chrono : : steady_clock : : duration parseSessionTimeout (
const Poco : : Util : : AbstractConfiguration & config ,
const HTMLForm & params )
2017-06-02 18:48:33 +00:00
{
2020-03-06 18:14:33 +00:00
unsigned session_timeout = config . getInt ( " default_session_timeout " , 60 ) ;
2017-06-02 18:48:33 +00:00
if ( params . has ( " session_timeout " ) )
{
2020-03-06 18:14:33 +00:00
unsigned max_session_timeout = config . getUInt ( " max_session_timeout " , 3600 ) ;
2017-06-02 21:01:17 +00:00
std : : string session_timeout_str = params . get ( " session_timeout " ) ;
2017-06-02 18:48:33 +00:00
2017-06-27 15:58:33 +00:00
ReadBufferFromString buf ( session_timeout_str ) ;
if ( ! tryReadIntText ( session_timeout , buf ) | | ! buf . eof ( ) )
throw Exception ( " Invalid session timeout: ' " + session_timeout_str + " ' " , ErrorCodes : : INVALID_SESSION_TIMEOUT ) ;
2017-06-02 18:48:33 +00:00
2017-06-02 21:01:17 +00:00
if ( session_timeout > max_session_timeout )
2020-11-11 12:34:28 +00:00
throw Exception ( " Session timeout ' " + session_timeout_str + " ' is larger than max_session_timeout: " + toString ( max_session_timeout )
+ " . Maximum session timeout could be modified in configuration file. " ,
2017-06-02 18:48:33 +00:00
ErrorCodes : : INVALID_SESSION_TIMEOUT ) ;
}
2020-03-06 18:14:33 +00:00
return std : : chrono : : seconds ( session_timeout ) ;
2017-06-02 18:48:33 +00:00
}
2017-02-28 14:15:13 +00:00
void HTTPHandler : : pushDelayedResults ( Output & used_output )
{
2017-04-01 07:20:54 +00:00
std : : vector < WriteBufferPtr > write_buffers ;
std : : vector < ReadBufferPtr > read_buffers ;
std : : vector < ReadBuffer * > read_buffers_raw_ptr ;
2020-05-18 08:08:55 +00:00
auto * cascade_buffer = typeid_cast < CascadeWriteBuffer * > ( used_output . out_maybe_delayed_and_compressed . get ( ) ) ;
2017-04-01 07:20:54 +00:00
if ( ! cascade_buffer )
throw Exception ( " Expected CascadeWriteBuffer " , ErrorCodes : : LOGICAL_ERROR ) ;
cascade_buffer - > getResultBuffers ( write_buffers ) ;
if ( write_buffers . empty ( ) )
throw Exception ( " At least one buffer is expected to overwrite result into HTTP response " , ErrorCodes : : LOGICAL_ERROR ) ;
for ( auto & write_buf : write_buffers )
{
IReadableWriteBuffer * write_buf_concrete ;
ReadBufferPtr reread_buf ;
2020-11-11 12:34:28 +00:00
if ( write_buf
& & ( write_buf_concrete = dynamic_cast < IReadableWriteBuffer * > ( write_buf . get ( ) ) )
2017-04-01 07:20:54 +00:00
& & ( reread_buf = write_buf_concrete - > tryGetReadBuffer ( ) ) )
{
read_buffers . emplace_back ( reread_buf ) ;
read_buffers_raw_ptr . emplace_back ( reread_buf . get ( ) ) ;
}
}
2021-02-04 15:59:54 +00:00
if ( ! read_buffers_raw_ptr . empty ( ) )
{
ConcatReadBuffer concat_read_buffer ( read_buffers_raw_ptr ) ;
copyData ( concat_read_buffer , * used_output . out_maybe_compressed ) ;
}
2017-02-28 14:15:13 +00:00
}
2020-11-11 12:34:28 +00:00
HTTPHandler : : HTTPHandler ( IServer & server_ , const std : : string & name )
: server ( server_ )
, log ( & Poco : : Logger : : get ( name ) )
2017-02-06 08:23:02 +00:00
{
2018-03-08 07:36:58 +00:00
server_display_name = server . config ( ) . getString ( " display_name " , getFQDNOrHostName ( ) ) ;
2017-02-06 08:23:02 +00:00
}
2016-01-11 21:46:36 +00:00
2017-06-02 21:01:17 +00:00
2016-06-25 07:22:12 +00:00
void HTTPHandler : : processQuery (
2020-06-21 13:55:18 +00:00
Context & context ,
2017-04-01 07:20:54 +00:00
Poco : : Net : : HTTPServerRequest & request ,
HTMLForm & params ,
Poco : : Net : : HTTPServerResponse & response ,
2020-10-16 19:22:14 +00:00
Output & used_output ,
std : : optional < CurrentThread : : QueryScope > & query_scope )
2012-03-09 03:06:09 +00:00
{
2020-05-23 22:24:01 +00:00
LOG_TRACE ( log , " Request URI: {} " , request . getURI ( ) ) ;
2017-04-01 07:20:54 +00:00
std : : istream & istr = request . stream ( ) ;
2017-09-21 11:16:18 +00:00
/// The user and password can be passed by headers (similar to X-Auth-*),
/// which is used by load balancers to pass authentication information.
std : : string user = request . get ( " X-ClickHouse-User " , " " ) ;
std : : string password = request . get ( " X-ClickHouse-Key " , " " ) ;
std : : string quota_key = request . get ( " X-ClickHouse-Quota " , " " ) ;
2017-04-01 07:20:54 +00:00
2017-09-21 11:16:18 +00:00
if ( user . empty ( ) & & password . empty ( ) & & quota_key . empty ( ) )
2017-04-01 07:20:54 +00:00
{
2017-09-21 11:16:18 +00:00
/// User name and password can be passed using query parameters
/// or using HTTP Basic auth (both methods are insecure).
if ( request . hasCredentials ( ) )
{
Poco : : Net : : HTTPBasicCredentials credentials ( request ) ;
user = credentials . getUsername ( ) ;
password = credentials . getPassword ( ) ;
2017-09-23 23:18:48 +00:00
}
else
{
2017-09-21 11:16:18 +00:00
user = params . get ( " user " , " default " ) ;
password = params . get ( " password " , " " ) ;
}
2017-04-01 07:20:54 +00:00
2017-09-21 11:16:18 +00:00
quota_key = params . get ( " quota_key " , " " ) ;
}
else
{
/// It is prohibited to mix different authorization schemes.
2020-11-11 12:34:28 +00:00
if ( request . hasCredentials ( )
| | params . has ( " user " )
| | params . has ( " password " )
| | params . has ( " quota_key " ) )
2017-09-21 11:16:18 +00:00
{
2020-11-11 12:34:28 +00:00
throw Exception ( " Invalid authentication: it is not allowed to use X-ClickHouse HTTP headers and other authentication methods simultaneously " , ErrorCodes : : REQUIRED_PASSWORD ) ;
2017-09-21 11:16:18 +00:00
}
2017-04-01 07:20:54 +00:00
}
2020-12-02 19:20:47 +00:00
/// Set client info. It will be used for quota accounting parameters in 'setUser' method.
ClientInfo & client_info = context . getClientInfo ( ) ;
client_info . query_kind = ClientInfo : : QueryKind : : INITIAL_QUERY ;
client_info . interface = ClientInfo : : Interface : : HTTP ;
ClientInfo : : HTTPMethod http_method = ClientInfo : : HTTPMethod : : UNKNOWN ;
if ( request . getMethod ( ) = = Poco : : Net : : HTTPServerRequest : : HTTP_GET )
http_method = ClientInfo : : HTTPMethod : : GET ;
else if ( request . getMethod ( ) = = Poco : : Net : : HTTPServerRequest : : HTTP_POST )
http_method = ClientInfo : : HTTPMethod : : POST ;
client_info . http_method = http_method ;
client_info . http_user_agent = request . get ( " User-Agent " , " " ) ;
2021-01-21 22:55:45 +00:00
client_info . http_referer = request . get ( " Referer " , " " ) ;
2020-12-02 19:20:47 +00:00
client_info . forwarded_for = request . get ( " X-Forwarded-For " , " " ) ;
/// This will also set client_info.current_user and current_address
2020-04-15 01:58:10 +00:00
context . setUser ( user , password , request . clientAddress ( ) ) ;
if ( ! quota_key . empty ( ) )
context . setQuotaKey ( quota_key ) ;
2017-04-01 07:20:54 +00:00
2020-12-02 19:20:47 +00:00
/// Query sent through HTTP interface is initial.
client_info . initial_user = client_info . current_user ;
client_info . initial_address = client_info . current_address ;
2017-06-02 21:01:17 +00:00
/// The user could specify session identifier and session timeout.
/// It allows to modify settings, create temporary tables and reuse them in subsequent requests.
2020-03-05 19:23:39 +00:00
std : : shared_ptr < NamedSession > session ;
2017-06-02 18:48:33 +00:00
String session_id ;
2020-03-06 18:14:33 +00:00
std : : chrono : : steady_clock : : duration session_timeout ;
2017-06-02 18:48:33 +00:00
bool session_is_set = params . has ( " session_id " ) ;
2018-06-19 20:30:35 +00:00
const auto & config = server . config ( ) ;
2017-06-02 18:48:33 +00:00
if ( session_is_set )
{
session_id = params . get ( " session_id " ) ;
2017-09-08 11:57:43 +00:00
session_timeout = parseSessionTimeout ( config , params ) ;
2017-06-02 18:48:33 +00:00
std : : string session_check = params . get ( " session_check " , " " ) ;
2020-03-05 19:23:39 +00:00
session = context . acquireNamedSession ( session_id , session_timeout , session_check = = " 1 " ) ;
2017-06-02 18:48:33 +00:00
2020-03-05 03:57:31 +00:00
context = session - > context ;
context . setSessionContext ( session - > context ) ;
2017-06-02 18:48:33 +00:00
}
2020-03-05 04:10:48 +00:00
SCOPE_EXIT ( {
if ( session )
session - > release ( ) ;
} ) ;
2020-10-19 21:26:10 +00:00
// Parse the OpenTelemetry traceparent header.
2020-10-20 08:13:21 +00:00
// Disable in Arcadia -- it interferes with the
// test_clickhouse.TestTracing.test_tracing_via_http_proxy[traceparent] test.
# if !defined(ARCADIA_BUILD)
2020-08-28 01:21:08 +00:00
if ( request . has ( " traceparent " ) )
{
std : : string opentelemetry_traceparent = request . get ( " traceparent " ) ;
std : : string error ;
2020-11-18 17:43:18 +00:00
if ( ! context . getClientInfo ( ) . client_trace_context . parseTraceparentHeader (
2020-11-11 12:34:28 +00:00
opentelemetry_traceparent , error ) )
2020-08-28 01:21:08 +00:00
{
2020-11-11 12:34:28 +00:00
throw Exception ( ErrorCodes : : BAD_REQUEST_PARAMETER ,
2020-08-28 01:21:08 +00:00
" Failed to parse OpenTelemetry traceparent header '{}': {} " ,
2020-11-11 12:34:28 +00:00
opentelemetry_traceparent , error ) ;
2020-08-28 01:21:08 +00:00
}
2020-11-18 17:43:18 +00:00
context . getClientInfo ( ) . client_trace_context . tracestate = request . get ( " tracestate " , " " ) ;
2020-08-28 01:21:08 +00:00
}
2020-10-20 08:13:21 +00:00
# endif
2020-08-28 01:21:08 +00:00
2020-10-20 11:35:13 +00:00
// Set the query id supplied by the user, if any, and also update the
// OpenTelemetry fields.
2020-11-11 12:34:28 +00:00
context . setCurrentQueryId ( params . get ( " query_id " ,
request . get ( " X-ClickHouse-Query-Id " , " " ) ) ) ;
2020-10-20 11:35:13 +00:00
2020-12-02 19:20:47 +00:00
client_info . initial_query_id = client_info . current_query_id ;
2017-04-01 07:20:54 +00:00
/// The client can pass a HTTP header indicating supported compression method (gzip or deflate).
String http_response_compression_methods = request . get ( " Accept-Encoding " , " " ) ;
2020-01-04 22:59:08 +00:00
CompressionMethod http_response_compression_method = CompressionMethod : : None ;
2017-04-01 07:20:54 +00:00
if ( ! http_response_compression_methods . empty ( ) )
{
2020-01-05 01:51:59 +00:00
/// If client supports brotli - it's preferred.
2017-04-01 07:20:54 +00:00
/// Both gzip and deflate are supported. If the client supports both, gzip is preferred.
/// NOTE parsing of the list of methods is slightly incorrect.
2020-01-05 01:51:59 +00:00
2020-01-05 01:54:58 +00:00
if ( std : : string : : npos ! = http_response_compression_methods . find ( " br " ) )
2020-01-05 01:51:59 +00:00
http_response_compression_method = CompressionMethod : : Brotli ;
else if ( std : : string : : npos ! = http_response_compression_methods . find ( " gzip " ) )
2019-02-13 20:54:12 +00:00
http_response_compression_method = CompressionMethod : : Gzip ;
2017-04-01 07:20:54 +00:00
else if ( std : : string : : npos ! = http_response_compression_methods . find ( " deflate " ) )
2019-02-13 20:54:12 +00:00
http_response_compression_method = CompressionMethod : : Zlib ;
2020-11-11 01:50:56 +00:00
else if ( std : : string : : npos ! = http_response_compression_methods . find ( " xz " ) )
http_response_compression_method = CompressionMethod : : Xz ;
2020-11-29 12:59:56 +00:00
else if ( std : : string : : npos ! = http_response_compression_methods . find ( " zstd " ) )
http_response_compression_method = CompressionMethod : : Zstd ;
2017-04-01 07:20:54 +00:00
}
2020-01-04 22:59:08 +00:00
bool client_supports_http_compression = http_response_compression_method ! = CompressionMethod : : None ;
2017-04-01 07:20:54 +00:00
/// Client can pass a 'compress' flag in the query string. In this case the query result is
/// compressed using internal algorithm. This is not reflected in HTTP headers.
bool internal_compression = params . getParsed < bool > ( " compress " , false ) ;
/// At least, we should postpone sending of first buffer_size result bytes
2020-11-11 12:34:28 +00:00
size_t buffer_size_total = std : : max (
params . getParsed < size_t > ( " buffer_size " , DBMS_DEFAULT_BUFFER_SIZE ) , static_cast < size_t > ( DBMS_DEFAULT_BUFFER_SIZE ) ) ;
2017-04-01 07:20:54 +00:00
/// If it is specified, the whole result will be buffered.
/// First ~buffer_size bytes will be buffered in memory, the remaining bytes will be stored in temporary file.
bool buffer_until_eof = params . getParsed < bool > ( " wait_end_of_query " , false ) ;
size_t buffer_size_http = DBMS_DEFAULT_BUFFER_SIZE ;
size_t buffer_size_memory = ( buffer_size_total > buffer_size_http ) ? buffer_size_total : 0 ;
2017-09-08 16:41:35 +00:00
unsigned keep_alive_timeout = config . getUInt ( " keep_alive_timeout " , 10 ) ;
2017-09-08 11:57:43 +00:00
2017-04-01 07:20:54 +00:00
used_output . out = std : : make_shared < WriteBufferFromHTTPServerResponse > (
2020-01-04 22:59:08 +00:00
request , response , keep_alive_timeout , client_supports_http_compression , http_response_compression_method ) ;
2017-04-01 07:20:54 +00:00
if ( internal_compression )
used_output . out_maybe_compressed = std : : make_shared < CompressedWriteBuffer > ( * used_output . out ) ;
else
used_output . out_maybe_compressed = used_output . out ;
if ( buffer_size_memory > 0 | | buffer_until_eof )
{
CascadeWriteBuffer : : WriteBufferPtrs cascade_buffer1 ;
CascadeWriteBuffer : : WriteBufferConstructors cascade_buffer2 ;
if ( buffer_size_memory > 0 )
cascade_buffer1 . emplace_back ( std : : make_shared < MemoryWriteBuffer > ( buffer_size_memory ) ) ;
if ( buffer_until_eof )
{
2020-05-28 05:38:55 +00:00
const std : : string tmp_path ( context . getTemporaryVolume ( ) - > getDisk ( ) - > getPath ( ) ) ;
2020-01-19 14:26:28 +00:00
const std : : string tmp_path_template ( tmp_path + " http_buffers/ " ) ;
2017-04-01 07:20:54 +00:00
2020-11-11 12:34:28 +00:00
auto create_tmp_disk_buffer = [ tmp_path_template ] ( const WriteBufferPtr & )
{
return WriteBufferFromTemporaryFile : : create ( tmp_path_template ) ;
} ;
2017-04-01 07:20:54 +00:00
cascade_buffer2 . emplace_back ( std : : move ( create_tmp_disk_buffer ) ) ;
}
else
{
2020-11-11 12:34:28 +00:00
auto push_memory_buffer_and_continue = [ next_buffer = used_output . out_maybe_compressed ] ( const WriteBufferPtr & prev_buf )
{
2020-05-18 08:08:55 +00:00
auto * prev_memory_buffer = typeid_cast < MemoryWriteBuffer * > ( prev_buf . get ( ) ) ;
2017-04-01 07:20:54 +00:00
if ( ! prev_memory_buffer )
throw Exception ( " Expected MemoryWriteBuffer " , ErrorCodes : : LOGICAL_ERROR ) ;
auto rdbuf = prev_memory_buffer - > tryGetReadBuffer ( ) ;
2020-11-11 12:34:28 +00:00
copyData ( * rdbuf , * next_buffer ) ;
2017-04-01 07:20:54 +00:00
return next_buffer ;
} ;
cascade_buffer2 . emplace_back ( push_memory_buffer_and_continue ) ;
}
2020-11-11 12:34:28 +00:00
used_output . out_maybe_delayed_and_compressed = std : : make_shared < CascadeWriteBuffer > (
std : : move ( cascade_buffer1 ) , std : : move ( cascade_buffer2 ) ) ;
2017-04-01 07:20:54 +00:00
}
else
{
used_output . out_maybe_delayed_and_compressed = used_output . out_maybe_compressed ;
}
/// Request body can be compressed using algorithm specified in the Content-Encoding header.
String http_request_compression_method_str = request . get ( " Content-Encoding " , " " ) ;
2020-01-04 07:31:00 +00:00
std : : unique_ptr < ReadBuffer > in_post = wrapReadBufferWithCompressionMethod (
std : : make_unique < ReadBufferFromIStream > ( istr ) , chooseCompressionMethod ( { } , http_request_compression_method_str ) ) ;
2017-04-01 07:20:54 +00:00
/// The data can also be compressed using incompatible internal algorithm. This is indicated by
/// 'decompress' query parameter.
std : : unique_ptr < ReadBuffer > in_post_maybe_compressed ;
bool in_post_compressed = false ;
if ( params . getParsed < bool > ( " decompress " , false ) )
{
in_post_maybe_compressed = std : : make_unique < CompressedReadBuffer > ( * in_post ) ;
in_post_compressed = true ;
}
else
in_post_maybe_compressed = std : : move ( in_post ) ;
std : : unique_ptr < ReadBuffer > in ;
2020-11-11 12:34:28 +00:00
static const NameSet reserved_param_names { " compress " , " decompress " , " user " , " password " , " quota_key " , " query_id " , " stacktrace " ,
" buffer_size " , " wait_end_of_query " , " session_id " , " session_timeout " , " session_check " } ;
2018-06-01 15:32:27 +00:00
Names reserved_param_suffixes ;
2020-11-11 12:34:28 +00:00
auto param_could_be_skipped = [ & ] ( const String & name )
{
2020-06-14 00:56:13 +00:00
/// Empty parameter appears when URL like ?&a=b or a=b&&c=d. Just skip them for user's convenience.
if ( name . empty ( ) )
return true ;
2018-06-01 15:32:27 +00:00
if ( reserved_param_names . count ( name ) )
return true ;
for ( const String & suffix : reserved_param_suffixes )
{
if ( endsWith ( name , suffix ) )
return true ;
}
return false ;
} ;
2017-04-01 07:20:54 +00:00
/// Settings can be overridden in the query.
/// Some parameters (database, default_format, everything used in the code above) do not
/// belong to the Settings class.
/// 'readonly' setting values mean:
/// readonly = 0 - any query is allowed, client can change any setting.
/// readonly = 1 - only readonly queries are allowed, client can't change settings.
/// readonly = 2 - only readonly queries are allowed, client can change any setting except 'readonly'.
/// In theory if initially readonly = 0, the client can change any setting and then set readonly
/// to some other value.
2020-03-13 14:50:26 +00:00
const auto & settings = context . getSettingsRef ( ) ;
2017-04-01 07:20:54 +00:00
/// Only readonly queries are allowed for HTTP GET requests.
if ( request . getMethod ( ) = = Poco : : Net : : HTTPServerRequest : : HTTP_GET )
{
2018-03-11 00:15:26 +00:00
if ( settings . readonly = = 0 )
2020-03-13 14:50:26 +00:00
context . setSetting ( " readonly " , 2 ) ;
2017-04-01 07:20:54 +00:00
}
2020-03-09 03:38:43 +00:00
bool has_external_data = startsWith ( request . getContentType ( ) , " multipart/form-data " ) ;
2019-05-28 21:31:19 +00:00
2019-06-15 18:25:27 +00:00
if ( has_external_data )
2019-05-28 21:31:19 +00:00
{
/// Skip unneeded parameters to avoid confusing them later with context settings or query parameters.
reserved_param_suffixes . reserve ( 3 ) ;
/// It is a bug and ambiguity with `date_time_input_format` and `low_cardinality_allow_in_native_format` formats/settings.
reserved_param_suffixes . emplace_back ( " _format " ) ;
reserved_param_suffixes . emplace_back ( " _types " ) ;
reserved_param_suffixes . emplace_back ( " _structure " ) ;
}
2020-07-28 09:58:12 +00:00
std : : string database = request . get ( " X-ClickHouse-Database " , " " ) ;
std : : string default_format = request . get ( " X-ClickHouse-Format " , " " ) ;
2019-04-18 23:29:32 +00:00
SettingsChanges settings_changes ;
2019-05-29 00:23:47 +00:00
for ( const auto & [ key , value ] : params )
2017-04-01 07:20:54 +00:00
{
2019-05-28 21:31:19 +00:00
if ( key = = " database " )
2017-04-01 07:20:54 +00:00
{
2020-07-28 09:58:12 +00:00
if ( database . empty ( ) )
database = value ;
2017-04-01 07:20:54 +00:00
}
2019-05-28 21:31:19 +00:00
else if ( key = = " default_format " )
2017-04-01 07:20:54 +00:00
{
2020-07-28 09:58:12 +00:00
if ( default_format . empty ( ) )
default_format = value ;
2017-04-01 07:20:54 +00:00
}
2019-05-28 21:31:19 +00:00
else if ( param_could_be_skipped ( key ) )
2017-04-01 07:20:54 +00:00
{
}
else
{
2020-04-04 08:57:16 +00:00
/// Other than query parameters are treated as settings.
if ( ! customizeQueryParam ( context , key , value ) )
settings_changes . push_back ( { key , value } ) ;
2017-04-01 07:20:54 +00:00
}
}
2020-07-28 09:58:12 +00:00
if ( ! database . empty ( ) )
context . setCurrentDatabase ( database ) ;
if ( ! default_format . empty ( ) )
context . setDefaultFormat ( default_format ) ;
2019-05-28 21:31:19 +00:00
/// For external data we also want settings
2019-04-18 23:29:32 +00:00
context . checkSettingsConstraints ( settings_changes ) ;
context . applySettingsChanges ( settings_changes ) ;
2020-04-04 08:57:16 +00:00
const auto & query = getQuery ( request , params , context ) ;
std : : unique_ptr < ReadBuffer > in_param = std : : make_unique < ReadBufferFromString > ( query ) ;
in = has_external_data ? std : : move ( in_param ) : std : : make_unique < ConcatReadBuffer > ( * in_param , * in_post_maybe_compressed ) ;
2019-05-28 21:31:19 +00:00
2017-04-01 07:20:54 +00:00
/// HTTP response compression is turned on only if the client signalled that they support it
/// (using Accept-Encoding header) and 'enable_http_compression' setting is turned on.
used_output . out - > setCompression ( client_supports_http_compression & & settings . enable_http_compression ) ;
if ( client_supports_http_compression )
used_output . out - > setCompressionLevel ( settings . http_zlib_compression_level ) ;
used_output . out - > setSendProgressInterval ( settings . http_headers_progress_interval_ms ) ;
/// If 'http_native_compression_disable_checksumming_on_decompress' setting is turned on,
/// checksums of client data compressed with internal algorithm are not checked.
if ( in_post_compressed & & settings . http_native_compression_disable_checksumming_on_decompress )
static_cast < CompressedReadBuffer & > ( * in_post_maybe_compressed ) . disableChecksumming ( ) ;
/// Add CORS header if 'add_http_cors_header' setting is turned on and the client passed
/// Origin header.
used_output . out - > addHeaderCORS ( settings . add_http_cors_header & & ! request . get ( " Origin " , " " ) . empty ( ) ) ;
2020-11-11 12:34:28 +00:00
auto append_callback = [ & context ] ( ProgressCallback callback )
{
2019-02-01 01:48:25 +00:00
auto prev = context . getProgressCallback ( ) ;
2020-11-11 12:34:28 +00:00
context . setProgressCallback ( [ prev , callback ] ( const Progress & progress )
{
2019-02-01 01:48:25 +00:00
if ( prev )
prev ( progress ) ;
callback ( progress ) ;
} ) ;
} ;
2017-04-01 07:20:54 +00:00
/// While still no data has been sent, we will report about query execution progress by sending HTTP headers.
if ( settings . send_progress_in_http_headers )
2020-11-11 12:34:28 +00:00
append_callback ( [ & used_output ] ( const Progress & progress ) { used_output . out - > onProgress ( progress ) ; } ) ;
2019-02-01 01:48:25 +00:00
if ( settings . readonly > 0 & & settings . cancel_http_readonly_queries_on_client_close )
{
Poco : : Net : : StreamSocket & socket = dynamic_cast < Poco : : Net : : HTTPServerRequestImpl & > ( request ) . socket ( ) ;
2020-11-11 12:34:28 +00:00
append_callback ( [ & context , & socket ] ( const Progress & )
{
2019-02-01 01:48:25 +00:00
/// Assume that at the point this method is called no one is reading data from the socket any more.
/// True for read-only queries.
2019-02-03 20:40:34 +00:00
try
{
2019-02-01 01:48:25 +00:00
char b ;
int status = socket . receiveBytes ( & b , 1 , MSG_DONTWAIT | MSG_PEEK ) ;
if ( status = = 0 )
context . killCurrentQuery ( ) ;
}
2019-02-03 20:40:34 +00:00
catch ( Poco : : TimeoutException & )
{
}
2019-02-01 01:48:25 +00:00
catch ( . . . )
{
context . killCurrentQuery ( ) ;
}
} ) ;
}
2017-04-01 07:20:54 +00:00
2020-04-21 11:30:45 +00:00
customizeContext ( request , context ) ;
2019-03-06 16:41:35 +00:00
2020-10-16 19:22:14 +00:00
query_scope . emplace ( context ) ;
2020-11-11 12:34:28 +00:00
executeQuery ( * in , * used_output . out_maybe_delayed_and_compressed , /* allow_into_outfile = */ false , context ,
[ & response ] ( const String & current_query_id , const String & content_type , const String & format , const String & timezone )
{
2020-01-21 19:03:33 +00:00
response . setContentType ( content_type ) ;
2020-03-03 15:32:41 +00:00
response . add ( " X-ClickHouse-Query-Id " , current_query_id ) ;
2020-01-21 19:03:33 +00:00
response . add ( " X-ClickHouse-Format " , format ) ;
2020-03-03 15:32:41 +00:00
response . add ( " X-ClickHouse-Timezone " , timezone ) ;
2020-11-11 12:34:28 +00:00
}
) ;
2017-04-01 07:20:54 +00:00
if ( used_output . hasDelayed ( ) )
{
/// TODO: set Content-Length if possible
pushDelayedResults ( used_output ) ;
}
/// Send HTTP headers with code 200 if no exception happened and the data is still not sent to
/// the client.
used_output . out - > finalize ( ) ;
2012-03-09 03:06:09 +00:00
}
2020-11-11 12:34:28 +00:00
void HTTPHandler : : trySendExceptionToClient ( const std : : string & s , int exception_code ,
Poco : : Net : : HTTPServerRequest & request , Poco : : Net : : HTTPServerResponse & response ,
2017-04-01 07:20:54 +00:00
Output & used_output )
2012-03-09 03:06:09 +00:00
{
2017-04-01 07:20:54 +00:00
try
{
2020-01-22 16:32:18 +00:00
response . set ( " X-ClickHouse-Exception-Code " , toString < int > ( exception_code ) ) ;
2020-01-23 13:40:16 +00:00
2017-04-01 07:20:54 +00:00
/// If HTTP method is POST and Keep-Alive is turned on, we should read the whole request body
/// to avoid reading part of the current request body in the next request.
2020-11-11 12:34:28 +00:00
if ( request . getMethod ( ) = = Poco : : Net : : HTTPRequest : : HTTP_POST
& & response . getKeepAlive ( )
& & ! request . stream ( ) . eof ( )
2017-09-25 19:45:15 +00:00
& & exception_code ! = ErrorCodes : : HTTP_LENGTH_REQUIRED )
2017-04-01 07:20:54 +00:00
{
request . stream ( ) . ignore ( std : : numeric_limits < std : : streamsize > : : max ( ) ) ;
}
2020-11-11 12:34:28 +00:00
bool auth_fail = exception_code = = ErrorCodes : : UNKNOWN_USER | |
exception_code = = ErrorCodes : : WRONG_PASSWORD | |
exception_code = = ErrorCodes : : REQUIRED_PASSWORD ;
2017-04-01 07:20:54 +00:00
if ( auth_fail )
{
response . requireAuthentication ( " ClickHouse server HTTP API " ) ;
}
else
{
response . setStatusAndReason ( exceptionCodeToHTTPStatus ( exception_code ) ) ;
}
if ( ! response . sent ( ) & & ! used_output . out_maybe_compressed )
{
/// If nothing was sent yet and we don't even know if we must compress the response.
response . send ( ) < < s < < std : : endl ;
}
else if ( used_output . out_maybe_compressed )
{
/// Destroy CascadeBuffer to actualize buffers' positions and reset extra references
if ( used_output . hasDelayed ( ) )
used_output . out_maybe_delayed_and_compressed . reset ( ) ;
/// Send the error message into already used (and possibly compressed) stream.
/// Note that the error message will possibly be sent after some data.
/// Also HTTP code 200 could have already been sent.
/// If buffer has data, and that data wasn't sent yet, then no need to send that data
bool data_sent = used_output . out - > count ( ) ! = used_output . out - > offset ( ) ;
if ( ! data_sent )
{
used_output . out_maybe_compressed - > position ( ) = used_output . out_maybe_compressed - > buffer ( ) . begin ( ) ;
used_output . out - > position ( ) = used_output . out - > buffer ( ) . begin ( ) ;
}
writeString ( s , * used_output . out_maybe_compressed ) ;
writeChar ( ' \n ' , * used_output . out_maybe_compressed ) ;
used_output . out_maybe_compressed - > next ( ) ;
used_output . out - > finalize ( ) ;
}
}
catch ( . . . )
{
tryLogCurrentException ( log , " Cannot send exception to client " ) ;
}
2014-07-14 19:46:06 +00:00
}
2012-06-25 05:07:34 +00:00
2017-01-17 17:21:48 +00:00
2014-07-14 19:46:06 +00:00
void HTTPHandler : : handleRequest ( Poco : : Net : : HTTPServerRequest & request , Poco : : Net : : HTTPServerResponse & response )
{
2018-08-31 00:59:48 +00:00
setThreadName ( " HTTPHandler " ) ;
2019-01-15 18:39:54 +00:00
ThreadStatus thread_status ;
2018-08-31 00:59:48 +00:00
2020-06-21 13:55:18 +00:00
/// Should be initialized before anything,
/// For correct memory accounting.
Context context = server . context ( ) ;
2020-10-16 19:22:14 +00:00
/// Cannot be set here, since query_id is unknown.
std : : optional < CurrentThread : : QueryScope > query_scope ;
2020-06-21 13:55:18 +00:00
2017-04-01 07:20:54 +00:00
Output used_output ;
2014-08-04 19:48:50 +00:00
2017-04-01 07:20:54 +00:00
/// In case of exception, send stack trace to client.
bool with_stacktrace = false ;
2016-06-25 07:22:12 +00:00
2017-04-01 07:20:54 +00:00
try
{
response . setContentType ( " text/plain; charset=UTF-8 " ) ;
2018-03-08 07:36:58 +00:00
response . set ( " X-ClickHouse-Server-Display-Name " , server_display_name ) ;
2017-04-01 07:20:54 +00:00
/// For keep-alive to work.
if ( request . getVersion ( ) = = Poco : : Net : : HTTPServerRequest : : HTTP_1_1 )
response . setChunkedTransferEncoding ( true ) ;
2014-07-14 19:46:06 +00:00
2017-04-01 07:20:54 +00:00
HTMLForm params ( request ) ;
with_stacktrace = params . getParsed < bool > ( " stacktrace " , false ) ;
2016-06-25 07:22:12 +00:00
2017-09-25 19:45:15 +00:00
/// Workaround. Poco does not detect 411 Length Required case.
2020-11-11 12:34:28 +00:00
if ( request . getMethod ( ) = = Poco : : Net : : HTTPRequest : : HTTP_POST & & ! request . getChunkedTransferEncoding ( ) & &
! request . hasContentLength ( ) )
2017-09-25 19:45:15 +00:00
{
2020-11-11 12:34:28 +00:00
throw Exception ( " The Transfer-Encoding is not chunked and there is no Content-Length header for POST request " , ErrorCodes : : HTTP_LENGTH_REQUIRED ) ;
2017-09-25 19:45:15 +00:00
}
2020-10-16 19:22:14 +00:00
processQuery ( context , request , params , response , used_output , query_scope ) ;
2020-10-10 17:47:34 +00:00
LOG_DEBUG ( log , " Done processing query " ) ;
2017-04-01 07:20:54 +00:00
}
catch ( . . . )
{
tryLogCurrentException ( log ) ;
2016-06-25 07:22:12 +00:00
2017-04-01 07:20:54 +00:00
/** If exception is received from remote server, then stack trace is embedded in message.
* If exception is thrown on local server , then stack trace is in separate field .
*/
std : : string exception_message = getCurrentExceptionMessage ( with_stacktrace , true ) ;
int exception_code = getCurrentExceptionCode ( ) ;
2016-06-25 07:22:12 +00:00
2017-04-01 07:20:54 +00:00
trySendExceptionToClient ( exception_message , exception_code , request , response , used_output ) ;
}
2012-03-09 03:06:09 +00:00
}
2020-04-04 08:57:16 +00:00
DynamicQueryHandler : : DynamicQueryHandler ( IServer & server_ , const std : : string & param_name_ )
: HTTPHandler ( server_ , " DynamicQueryHandler " ) , param_name ( param_name_ )
2019-11-07 10:24:30 +00:00
{
2020-04-04 08:57:16 +00:00
}
2019-11-07 10:24:30 +00:00
2020-04-04 08:57:16 +00:00
bool DynamicQueryHandler : : customizeQueryParam ( Context & context , const std : : string & key , const std : : string & value )
{
if ( key = = param_name )
2020-11-11 12:34:28 +00:00
return true ; /// do nothing
2020-04-04 08:57:16 +00:00
if ( startsWith ( key , " param_ " ) )
2019-11-07 10:24:30 +00:00
{
2020-04-04 08:57:16 +00:00
/// Save name and values of substitution in dictionary.
const String parameter_name = key . substr ( strlen ( " param_ " ) ) ;
context . setQueryParameter ( parameter_name , value ) ;
return true ;
2017-04-01 07:20:54 +00:00
}
2019-11-07 10:24:30 +00:00
2020-04-04 08:57:16 +00:00
return false ;
2012-03-09 03:06:09 +00:00
}
2020-04-04 08:57:16 +00:00
std : : string DynamicQueryHandler : : getQuery ( Poco : : Net : : HTTPServerRequest & request , HTMLForm & params , Context & context )
{
if ( likely ( ! startsWith ( request . getContentType ( ) , " multipart/form-data " ) ) )
{
/// Part of the query can be passed in the 'query' parameter and the rest in the request body
/// (http method need not necessarily be POST). In this case the entire query consists of the
/// contents of the 'query' parameter, a line break and the request body.
std : : string query_param = params . get ( param_name , " " ) ;
return query_param . empty ( ) ? query_param : query_param + " \n " ;
}
/// Support for "external data for query processing".
/// Used in case of POST request with form-data, but it isn't expected to be deleted after that scope.
ExternalTablesHandler handler ( context , params ) ;
params . load ( request , request . stream ( ) , handler ) ;
std : : string full_query ;
/// Params are of both form params POST and uri (GET params)
for ( const auto & it : params )
if ( it . first = = param_name )
full_query + = it . second ;
return full_query ;
}
2020-04-27 20:51:39 +00:00
PredefinedQueryHandler : : PredefinedQueryHandler (
2020-11-11 12:34:28 +00:00
IServer & server_ , const NameSet & receive_params_ , const std : : string & predefined_query_
, const CompiledRegexPtr & url_regex_ , const std : : unordered_map < String , CompiledRegexPtr > & header_name_with_regex_ )
: HTTPHandler ( server_ , " PredefinedQueryHandler " ) , receive_params ( receive_params_ ) , predefined_query ( predefined_query_ )
, url_regex ( url_regex_ ) , header_name_with_capture_regex ( header_name_with_regex_ )
2020-04-04 08:57:16 +00:00
{
}
2020-04-27 20:51:39 +00:00
bool PredefinedQueryHandler : : customizeQueryParam ( Context & context , const std : : string & key , const std : : string & value )
2020-04-04 08:57:16 +00:00
{
if ( receive_params . count ( key ) )
{
context . setQueryParameter ( key , value ) ;
return true ;
}
return false ;
}
2020-04-27 20:51:39 +00:00
void PredefinedQueryHandler : : customizeContext ( Poco : : Net : : HTTPServerRequest & request , DB : : Context & context )
2020-04-21 11:30:45 +00:00
{
/// If in the configuration file, the handler's header is regex and contains named capture group
/// We will extract regex named capture groups as query parameters
2020-11-11 12:34:28 +00:00
const auto & set_query_params = [ & ] ( const char * begin , const char * end , const CompiledRegexPtr & compiled_regex )
{
2020-04-21 11:30:45 +00:00
int num_captures = compiled_regex - > NumberOfCapturingGroups ( ) + 1 ;
2020-04-28 07:19:37 +00:00
re2 : : StringPiece matches [ num_captures ] ;
re2 : : StringPiece input ( begin , end - begin ) ;
if ( compiled_regex - > Match ( input , 0 , end - begin , re2 : : RE2 : : Anchor : : ANCHOR_BOTH , matches , num_captures ) )
2020-04-21 11:30:45 +00:00
{
for ( const auto & [ capturing_name , capturing_index ] : compiled_regex - > NamedCapturingGroups ( ) )
{
const auto & capturing_value = matches [ capturing_index ] ;
if ( capturing_value . data ( ) )
context . setQueryParameter ( capturing_name , String ( capturing_value . data ( ) , capturing_value . size ( ) ) ) ;
}
}
} ;
2020-04-26 11:06:14 +00:00
if ( url_regex )
{
const auto & uri = request . getURI ( ) ;
2020-04-27 22:19:24 +00:00
set_query_params ( uri . data ( ) , find_first_symbols < ' ? ' > ( uri . data ( ) , uri . data ( ) + uri . size ( ) ) , url_regex ) ;
2020-04-26 11:06:14 +00:00
}
2020-04-21 11:30:45 +00:00
for ( const auto & [ header_name , regex ] : header_name_with_capture_regex )
{
2020-04-26 11:06:14 +00:00
const auto & header_value = request . get ( header_name ) ;
set_query_params ( header_value . data ( ) , header_value . data ( ) + header_value . size ( ) , regex ) ;
2020-04-21 11:30:45 +00:00
}
}
2020-04-27 20:51:39 +00:00
std : : string PredefinedQueryHandler : : getQuery ( Poco : : Net : : HTTPServerRequest & request , HTMLForm & params , Context & context )
2020-04-04 08:57:16 +00:00
{
if ( unlikely ( startsWith ( request . getContentType ( ) , " multipart/form-data " ) ) )
{
/// Support for "external data for query processing".
ExternalTablesHandler handler ( context , params ) ;
params . load ( request , request . stream ( ) , handler ) ;
}
2020-04-27 20:51:39 +00:00
return predefined_query ;
2020-04-04 08:57:16 +00:00
}
Poco : : Net : : HTTPRequestHandlerFactory * createDynamicHandlerFactory ( IServer & server , const std : : string & config_prefix )
{
2020-04-21 11:30:45 +00:00
std : : string query_param_name = server . config ( ) . getString ( config_prefix + " .handler.query_param_name " , " query " ) ;
2020-11-11 12:34:28 +00:00
return addFiltersFromConfig ( new HandlingRuleHTTPHandlerFactory < DynamicQueryHandler > ( server , std : : move ( query_param_name ) ) , server . config ( ) , config_prefix ) ;
2020-04-04 08:57:16 +00:00
}
2020-04-27 22:19:24 +00:00
static inline bool capturingNamedQueryParam ( NameSet receive_params , const CompiledRegexPtr & compiled_regex )
2020-04-26 11:06:14 +00:00
{
const auto & capturing_names = compiled_regex - > NamedCapturingGroups ( ) ;
2020-11-11 12:34:28 +00:00
return std : : count_if ( capturing_names . begin ( ) , capturing_names . end ( ) , [ & ] ( const auto & iterator )
{
return std : : count_if ( receive_params . begin ( ) , receive_params . end ( ) ,
[ & ] ( const auto & param_name ) { return param_name = = iterator . first ; } ) ;
2020-04-26 11:06:14 +00:00
} ) ;
}
2020-04-04 08:57:16 +00:00
2020-04-27 22:19:24 +00:00
static inline CompiledRegexPtr getCompiledRegex ( const std : : string & expression )
{
2020-04-28 07:19:37 +00:00
auto compiled_regex = std : : make_shared < const re2 : : RE2 > ( expression ) ;
2020-04-27 22:19:24 +00:00
if ( ! compiled_regex - > ok ( ) )
2020-11-11 12:34:28 +00:00
throw Exception ( " Cannot compile re2: " + expression + " for http handling rule, error: " +
compiled_regex - > error ( ) + " . Look at https://github.com/google/re2/wiki/Syntax for reference. " , ErrorCodes : : CANNOT_COMPILE_REGEXP ) ;
2020-04-27 22:19:24 +00:00
return compiled_regex ;
}
2020-04-27 20:51:39 +00:00
Poco : : Net : : HTTPRequestHandlerFactory * createPredefinedHandlerFactory ( IServer & server , const std : : string & config_prefix )
2020-04-04 08:57:16 +00:00
{
2020-04-26 11:06:14 +00:00
Poco : : Util : : AbstractConfiguration & configuration = server . config ( ) ;
if ( ! configuration . has ( config_prefix + " .handler.query " ) )
2020-11-11 12:34:28 +00:00
throw Exception ( " There is no path ' " + config_prefix + " .handler.query " + " ' in configuration file. " , ErrorCodes : : NO_ELEMENTS_IN_CONFIG ) ;
2020-04-04 08:57:16 +00:00
2020-04-27 20:51:39 +00:00
std : : string predefined_query = configuration . getString ( config_prefix + " .handler.query " ) ;
NameSet analyze_receive_params = analyzeReceiveQueryParams ( predefined_query ) ;
2020-04-21 11:30:45 +00:00
2020-04-27 22:19:24 +00:00
std : : unordered_map < String , CompiledRegexPtr > headers_name_with_regex ;
2020-04-26 11:06:14 +00:00
Poco : : Util : : AbstractConfiguration : : Keys headers_name ;
configuration . keys ( config_prefix + " .headers " , headers_name ) ;
2020-04-21 11:30:45 +00:00
2020-04-26 11:06:14 +00:00
for ( const auto & header_name : headers_name )
2020-04-21 11:30:45 +00:00
{
2020-04-26 11:06:14 +00:00
auto expression = configuration . getString ( config_prefix + " .headers. " + header_name ) ;
2020-04-04 08:57:16 +00:00
2020-04-26 11:06:14 +00:00
if ( ! startsWith ( expression , " regex: " ) )
continue ;
expression = expression . substr ( 6 ) ;
2020-04-27 22:19:24 +00:00
auto regex = getCompiledRegex ( expression ) ;
if ( capturingNamedQueryParam ( analyze_receive_params , regex ) )
headers_name_with_regex . emplace ( std : : make_pair ( header_name , regex ) ) ;
2020-04-21 11:30:45 +00:00
}
2020-04-26 11:06:14 +00:00
if ( configuration . has ( config_prefix + " .url " ) )
{
auto url_expression = configuration . getString ( config_prefix + " .url " ) ;
if ( startsWith ( url_expression , " regex: " ) )
url_expression = url_expression . substr ( 6 ) ;
2020-04-27 22:19:24 +00:00
auto regex = getCompiledRegex ( url_expression ) ;
if ( capturingNamedQueryParam ( analyze_receive_params , regex ) )
2020-11-11 12:34:28 +00:00
return addFiltersFromConfig ( new HandlingRuleHTTPHandlerFactory < PredefinedQueryHandler > (
server , std : : move ( analyze_receive_params ) , std : : move ( predefined_query ) , std : : move ( regex ) ,
std : : move ( headers_name_with_regex ) ) , configuration , config_prefix ) ;
}
return addFiltersFromConfig ( new HandlingRuleHTTPHandlerFactory < PredefinedQueryHandler > (
server , std : : move ( analyze_receive_params ) , std : : move ( predefined_query ) , CompiledRegexPtr { } , std : : move ( headers_name_with_regex ) ) ,
configuration , config_prefix ) ;
2020-04-26 11:06:14 +00:00
}
2012-03-09 03:06:09 +00:00
}