Merge pull request #11628 from zhang2014/feature/ISSUES-7572

ISSUES-7572 support config default http handler
This commit is contained in:
tavplubix 2020-06-15 20:37:04 +03:00 committed by GitHub
commit 86e2cfe999
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 161 additions and 61 deletions

View File

@ -1,6 +1,5 @@
#include "HTTPHandlerFactory.h"
#include <re2/stringpiece.h>
#include <Poco/Util/LayeredConfiguration.h>
#include "HTTPHandler.h"
@ -21,6 +20,9 @@ namespace ErrorCodes
extern const int INVALID_CONFIG_PARAMETER;
}
static void addCommonDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server);
static void addDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server, AsynchronousMetrics & async_metrics);
HTTPRequestHandlerFactoryMain::HTTPRequestHandlerFactoryMain(const std::string & name_)
: log(&Poco::Logger::get(name_)), name(name_)
{
@ -65,7 +67,8 @@ HTTPRequestHandlerFactoryMain::TThis * HTTPRequestHandlerFactoryMain::addHandler
return this;
}
static inline auto createHandlersFactoryFromConfig(IServer & server, const std::string & name, const String & prefix)
static inline auto createHandlersFactoryFromConfig(
IServer & server, const std::string & name, const String & prefix, AsynchronousMetrics & async_metrics)
{
auto main_handler_factory = std::make_unique<HTTPRequestHandlerFactoryMain>(name);
@ -74,66 +77,46 @@ static inline auto createHandlersFactoryFromConfig(IServer & server, const std::
for (const auto & key : keys)
{
if (!startsWith(key, "rule"))
throw Exception("Unknown element in config: " + prefix + "." + key + ", must be 'rule'", ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
if (key == "defaults")
addDefaultHandlersFactory(*main_handler_factory, server, async_metrics);
else if (startsWith(key, "rule"))
{
const auto & handler_type = server.config().getString(prefix + "." + key + ".handler.type", "");
const auto & handler_type = server.config().getString(prefix + "." + key + ".handler.type", "");
if (handler_type.empty())
throw Exception("Handler type in config is not specified here: " + prefix + "." + key + ".handler.type",
ErrorCodes::INVALID_CONFIG_PARAMETER);
if (handler_type == "static")
main_handler_factory->addHandler(createStaticHandlerFactory(server, prefix + "." + key));
else if (handler_type == "dynamic_query_handler")
main_handler_factory->addHandler(createDynamicHandlerFactory(server, prefix + "." + key));
else if (handler_type == "predefined_query_handler")
main_handler_factory->addHandler(createPredefinedHandlerFactory(server, prefix + "." + key));
else if (handler_type.empty())
throw Exception("Handler type in config is not specified here: " +
prefix + "." + key + ".handler.type", ErrorCodes::INVALID_CONFIG_PARAMETER);
if (handler_type == "static")
main_handler_factory->addHandler(createStaticHandlerFactory(server, prefix + "." + key));
else if (handler_type == "dynamic_query_handler")
main_handler_factory->addHandler(createDynamicHandlerFactory(server, prefix + "." + key));
else if (handler_type == "predefined_query_handler")
main_handler_factory->addHandler(createPredefinedHandlerFactory(server, prefix + "." + key));
else if (handler_type == "prometheus")
main_handler_factory->addHandler(createPrometheusHandlerFactory(server, async_metrics, prefix + "." + key));
else if (handler_type == "replicas_status")
main_handler_factory->addHandler(createReplicasStatusHandlerFactory(server, prefix + "." + key));
else
throw Exception("Unknown handler type '" + handler_type + "' in config here: " + prefix + "." + key + ".handler.type",
ErrorCodes::INVALID_CONFIG_PARAMETER);
}
else
throw Exception("Unknown handler type '" + handler_type +"' in config here: " +
prefix + "." + key + ".handler.type",ErrorCodes::INVALID_CONFIG_PARAMETER);
throw Exception("Unknown element in config: " + prefix + "." + key + ", must be 'rule' or 'defaults'",
ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
}
return main_handler_factory.release();
}
static const auto ping_response_expression = "Ok.\n";
static const auto root_response_expression = "config://http_server_default_response";
static inline Poco::Net::HTTPRequestHandlerFactory * createHTTPHandlerFactory(
IServer & server, const std::string & name, AsynchronousMetrics & async_metrics)
static inline Poco::Net::HTTPRequestHandlerFactory * createHTTPHandlerFactory(IServer & server, const std::string & name, AsynchronousMetrics & async_metrics)
{
if (server.config().has("http_handlers"))
return createHandlersFactoryFromConfig(server, name, "http_handlers");
return createHandlersFactoryFromConfig(server, name, "http_handlers", async_metrics);
else
{
auto factory = std::make_unique<HTTPRequestHandlerFactoryMain>(name);
auto root_handler = std::make_unique<HandlingRuleHTTPHandlerFactory<StaticRequestHandler>>(server, root_response_expression);
root_handler->attachStrictPath("/")->allowGetAndHeadRequest();
factory->addHandler(root_handler.release());
auto ping_handler = std::make_unique<HandlingRuleHTTPHandlerFactory<StaticRequestHandler>>(server, ping_response_expression);
ping_handler->attachStrictPath("/ping")->allowGetAndHeadRequest();
factory->addHandler(ping_handler.release());
auto replicas_status_handler = std::make_unique<HandlingRuleHTTPHandlerFactory<ReplicasStatusHandler>>(server);
replicas_status_handler->attachNonStrictPath("/replicas_status")->allowGetAndHeadRequest();
factory->addHandler(replicas_status_handler.release());
auto query_handler = std::make_unique<HandlingRuleHTTPHandlerFactory<DynamicQueryHandler>>(server, "query");
query_handler->allowPostAndGetParamsRequest();
factory->addHandler(query_handler.release());
/// We check that prometheus handler will be served on current (default) port.
/// Otherwise it will be created separately, see below.
if (server.config().has("prometheus") && server.config().getInt("prometheus.port", 0) == 0)
{
auto prometheus_handler = std::make_unique<HandlingRuleHTTPHandlerFactory<PrometheusRequestHandler>>(
server, PrometheusMetricsWriter(server.config(), "prometheus", async_metrics));
prometheus_handler->attachStrictPath(server.config().getString("prometheus.endpoint", "/metrics"))->allowGetAndHeadRequest();
factory->addHandler(prometheus_handler.release());
}
addDefaultHandlersFactory(*factory, server, async_metrics);
return factory.release();
}
}
@ -141,18 +124,7 @@ static inline Poco::Net::HTTPRequestHandlerFactory * createHTTPHandlerFactory(
static inline Poco::Net::HTTPRequestHandlerFactory * createInterserverHTTPHandlerFactory(IServer & server, const std::string & name)
{
auto factory = std::make_unique<HTTPRequestHandlerFactoryMain>(name);
auto root_handler = std::make_unique<HandlingRuleHTTPHandlerFactory<StaticRequestHandler>>(server, root_response_expression);
root_handler->attachStrictPath("/")->allowGetAndHeadRequest();
factory->addHandler(root_handler.release());
auto ping_handler = std::make_unique<HandlingRuleHTTPHandlerFactory<StaticRequestHandler>>(server, ping_response_expression);
ping_handler->attachStrictPath("/ping")->allowGetAndHeadRequest();
factory->addHandler(ping_handler.release());
auto replicas_status_handler = std::make_unique<HandlingRuleHTTPHandlerFactory<ReplicasStatusHandler>>(server);
replicas_status_handler->attachNonStrictPath("/replicas_status")->allowGetAndHeadRequest();
factory->addHandler(replicas_status_handler.release());
addCommonDefaultHandlersFactory(*factory, server);
auto main_handler = std::make_unique<HandlingRuleHTTPHandlerFactory<InterserverIOHTTPHandler>>(server);
main_handler->allowPostAndGetParamsRequest();
@ -180,4 +152,41 @@ Poco::Net::HTTPRequestHandlerFactory * createHandlerFactory(IServer & server, As
throw Exception("LOGICAL ERROR: Unknown HTTP handler factory name.", ErrorCodes::LOGICAL_ERROR);
}
static const auto ping_response_expression = "Ok.\n";
static const auto root_response_expression = "config://http_server_default_response";
void addCommonDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server)
{
auto root_handler = std::make_unique<HandlingRuleHTTPHandlerFactory<StaticRequestHandler>>(server, root_response_expression);
root_handler->attachStrictPath("/")->allowGetAndHeadRequest();
factory.addHandler(root_handler.release());
auto ping_handler = std::make_unique<HandlingRuleHTTPHandlerFactory<StaticRequestHandler>>(server, ping_response_expression);
ping_handler->attachStrictPath("/ping")->allowGetAndHeadRequest();
factory.addHandler(ping_handler.release());
auto replicas_status_handler = std::make_unique<HandlingRuleHTTPHandlerFactory<ReplicasStatusHandler>>(server);
replicas_status_handler->attachNonStrictPath("/replicas_status")->allowGetAndHeadRequest();
factory.addHandler(replicas_status_handler.release());
}
void addDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server, AsynchronousMetrics & async_metrics)
{
addCommonDefaultHandlersFactory(factory, server);
auto query_handler = std::make_unique<HandlingRuleHTTPHandlerFactory<DynamicQueryHandler>>(server, "query");
query_handler->allowPostAndGetParamsRequest();
factory.addHandler(query_handler.release());
/// We check that prometheus handler will be served on current (default) port.
/// Otherwise it will be created separately, see createHandlerFactory(...).
if (server.config().has("prometheus") && server.config().getInt("prometheus.port", 0) == 0)
{
auto prometheus_handler = std::make_unique<HandlingRuleHTTPHandlerFactory<PrometheusRequestHandler>>(
server, PrometheusMetricsWriter(server.config(), "prometheus", async_metrics));
prometheus_handler->attachStrictPath(server.config().getString("prometheus.endpoint", "/metrics"))->allowGetAndHeadRequest();
factory.addHandler(prometheus_handler.release());
}
}
}

View File

@ -109,6 +109,10 @@ Poco::Net::HTTPRequestHandlerFactory * createDynamicHandlerFactory(IServer & ser
Poco::Net::HTTPRequestHandlerFactory * createPredefinedHandlerFactory(IServer & server, const std::string & config_prefix);
Poco::Net::HTTPRequestHandlerFactory * createReplicasStatusHandlerFactory(IServer & server, const std::string & config_prefix);
Poco::Net::HTTPRequestHandlerFactory * createPrometheusHandlerFactory(IServer & server, AsynchronousMetrics & async_metrics, const std::string & config_prefix);
Poco::Net::HTTPRequestHandlerFactory * createHandlerFactory(IServer & server, AsynchronousMetrics & async_metrics, const std::string & name);
}

View File

@ -12,6 +12,7 @@
#include <Common/CurrentMetrics.h>
#include <IO/WriteBufferFromHTTPServerResponse.h>
#include <Server/HTTPHandlerRequestFilter.h>
namespace DB
@ -40,4 +41,10 @@ void PrometheusRequestHandler::handleRequest(
}
}
Poco::Net::HTTPRequestHandlerFactory * createPrometheusHandlerFactory(IServer & server, AsynchronousMetrics & async_metrics, const std::string & config_prefix)
{
return addFiltersFromConfig(new HandlingRuleHTTPHandlerFactory<PrometheusRequestHandler>(
server, PrometheusMetricsWriter(server.config(), config_prefix + ".handler", async_metrics)), server.config(), config_prefix);
}
}

View File

@ -7,8 +7,11 @@
#include <Databases/IDatabase.h>
#include <IO/HTTPCommon.h>
#include <Poco/Net/HTTPRequestHandlerFactory.h>
#include <Poco/Net/HTTPServerRequest.h>
#include <Poco/Net/HTTPServerResponse.h>
#include <Server/HTTPHandlerFactory.h>
#include <Server/HTTPHandlerRequestFilter.h>
namespace DB
@ -104,5 +107,9 @@ void ReplicasStatusHandler::handleRequest(Poco::Net::HTTPServerRequest & request
}
}
Poco::Net::HTTPRequestHandlerFactory * createReplicasStatusHandlerFactory(IServer & server, const std::string & config_prefix)
{
return addFiltersFromConfig(new HandlingRuleHTTPHandlerFactory<ReplicasStatusHandler>(server), server.config(), config_prefix);
}
}

View File

@ -113,3 +113,38 @@ def test_relative_path_static_handler():
assert 'text/html; charset=UTF-8' == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', headers={'XXX': 'xxx'}).headers['Content-Type']
assert '<html><body>Relative Path File</body></html>\n' == cluster.instance.http_request('test_get_relative_path_static_handler', method='GET', headers={'XXX': 'xxx'}).content
def test_defaults_http_handlers():
with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "defaults_handlers", "test_defaults_handlers")) as cluster:
assert 200 == cluster.instance.http_request('', method='GET').status_code
assert 'Default server response' == cluster.instance.http_request('', method='GET').content
assert 200 == cluster.instance.http_request('ping', method='GET').status_code
assert 'Ok.\n' == cluster.instance.http_request('ping', method='GET').content
assert 200 == cluster.instance.http_request('replicas_status', method='GET').status_code
assert 'Ok.\n' == cluster.instance.http_request('replicas_status', method='GET').content
assert 200 == cluster.instance.http_request('?query=SELECT+1', method='GET').status_code
assert '1\n' == cluster.instance.http_request('?query=SELECT+1', method='GET').content
def test_prometheus_handler():
with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "prometheus_handler", "test_prometheus_handler")) as cluster:
assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code
assert 404 == cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'bad'}).status_code
assert 404 == cluster.instance.http_request('test_prometheus', method='POST', headers={'XXX': 'xxx'}).status_code
assert 200 == cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'xxx'}).status_code
assert 'ClickHouseProfileEvents_Query' in cluster.instance.http_request('test_prometheus', method='GET', headers={'XXX': 'xxx'}).content
def test_replicas_status_handler():
with contextlib.closing(SimpleCluster(ClickHouseCluster(__file__), "replicas_status_handler", "test_replicas_status_handler")) as cluster:
assert 404 == cluster.instance.http_request('', method='GET', headers={'XXX': 'xxx'}).status_code
assert 404 == cluster.instance.http_request('test_replicas_status', method='GET', headers={'XXX': 'bad'}).status_code
assert 404 == cluster.instance.http_request('test_replicas_status', method='POST', headers={'XXX': 'xxx'}).status_code
assert 200 == cluster.instance.http_request('test_replicas_status', method='GET', headers={'XXX': 'xxx'}).status_code
assert 'Ok.\n' == cluster.instance.http_request('test_replicas_status', method='GET', headers={'XXX': 'xxx'}).content

View File

@ -0,0 +1,9 @@
<?xml version="1.0"?>
<yandex>
<http_server_default_response>Default server response</http_server_default_response>
<http_handlers>
<defaults/>
</http_handlers>
</yandex>

View File

@ -0,0 +1,17 @@
<?xml version="1.0"?>
<yandex>
<http_handlers>
<rule>
<methods>GET</methods>
<headers><XXX>xxx</XXX></headers>
<url>/test_prometheus</url>
<handler>
<type>prometheus</type>
<events>true</events>
<metrics>true</metrics>
<asynchronous_metrics>true</asynchronous_metrics>
</handler>
</rule>
</http_handlers>
</yandex>

View File

@ -0,0 +1,12 @@
<?xml version="1.0"?>
<yandex>
<http_handlers>
<rule>
<methods>GET</methods>
<headers><XXX>xxx</XXX></headers>
<url>/test_replicas_status</url>
<handler><type>replicas_status</type></handler>
</rule>
</http_handlers>
</yandex>