thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 01:22:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

CLICKHOUSE-4112: Add keepalive for http connections

Browse Source
This commit is contained in:
alesapin 2018-11-20 16:15:44 +03:00
parent e7fb653d0c
commit 01e2137364
7 changed files with 30 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dbms/src/Common/ErrorCodes.cpp
View File

@ -396,6 +396,7 @@ namespace ErrorCodes
extern const int MULTIPLE_ASSIGNMENTS_TO_COLUMN = 419; extern const int MULTIPLE_ASSIGNMENTS_TO_COLUMN = 419;
extern const int CANNOT_UPDATE_COLUMN = 420; extern const int CANNOT_UPDATE_COLUMN = 420;
extern const int CANNOT_ADD_DIFFERENT_AGGREGATE_STATES = 421; extern const int CANNOT_ADD_DIFFERENT_AGGREGATE_STATES = 421;
extern const int UNKNOWN_PROTOCOL = 422;
extern const int KEEPER_EXCEPTION = 999; extern const int KEEPER_EXCEPTION = 999;
extern const int POCO_EXCEPTION = 1000; extern const int POCO_EXCEPTION = 1000;

2
dbms/src/Core/Defines.h
View File

@ -62,6 +62,8 @@
#define DEFAULT_HTTP_READ_BUFFER_TIMEOUT 1800 #define DEFAULT_HTTP_READ_BUFFER_TIMEOUT 1800
#define DEFAULT_HTTP_READ_BUFFER_CONNECTION_TIMEOUT 1 #define DEFAULT_HTTP_READ_BUFFER_CONNECTION_TIMEOUT 1
/// Maximum namber of http-connections between two endpoints
/// the number is unmotivated
#define DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT 15 #define DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT 15
// more aliases: https://mailman.videolan.org/pipermail/x264-devel/2014-May/010660.html // more aliases: https://mailman.videolan.org/pipermail/x264-devel/2014-May/010660.html

15
dbms/src/IO/HTTPCommon.cpp
View File

@ -52,23 +52,24 @@ namespace
#endif #endif
} }
HTTPSessionPtr makeHTTPSessionImpl(const std::string & host, UInt16 port, bool https) HTTPSessionPtr makeHTTPSessionImpl(const std::string & host, UInt16 port, bool https, bool keep_alive)
{ {
HTTPSessionPtr session; HTTPSessionPtr session;
if (https) if (https)
#if USE_POCO_NETSSL #if USE_POCO_NETSSL
session = std::make_unique<Poco::Net::HTTPSClientSession>(); session = std::make_shared<Poco::Net::HTTPSClientSession>();
#else #else
throw Exception("ClickHouse was built without HTTPS support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME); throw Exception("ClickHouse was built without HTTPS support", ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME);
#endif #endif
else else
session = std::make_unique<Poco::Net::HTTPClientSession>(); session = std::make_shared<Poco::Net::HTTPClientSession>();
ProfileEvents::increment(ProfileEvents::CreatedHTTPConnections); ProfileEvents::increment(ProfileEvents::CreatedHTTPConnections);
session->setHost(DNSResolver::instance().resolveHost(host).toString()); session->setHost(DNSResolver::instance().resolveHost(host).toString());
session->setPort(port); session->setPort(port);
session->setKeepAlive(keep_alive);
return session; return session;
} }
@ -83,7 +84,7 @@ namespace
ObjectPtr allocObject() override ObjectPtr allocObject() override
{ {
return makeHTTPSessionImpl(host, port, https); return makeHTTPSessionImpl(host, port, https, true);
} }
public: public:
@ -133,7 +134,9 @@ namespace
std::tie(pool_ptr, std::ignore) = endpoints_pool.emplace( std::tie(pool_ptr, std::ignore) = endpoints_pool.emplace(
key, std::make_shared<SingleEndpointHTTPSessionPool>(host, port, https, max_connections_per_endpoint)); key, std::make_shared<SingleEndpointHTTPSessionPool>(host, port, https, max_connections_per_endpoint));
auto session = pool_ptr->second->get(-1); auto retry_timeout = timeouts.connection_timeout.totalMicroseconds();
auto session = pool_ptr->second->get(retry_timeout);
setTimeouts(*session, timeouts); setTimeouts(*session, timeouts);
return session; return session;
} }
@ -156,7 +159,7 @@ HTTPSessionPtr makeHTTPSession(const Poco::URI & uri, const ConnectionTimeouts &
UInt16 port = uri.getPort(); UInt16 port = uri.getPort();
bool https = (uri.getScheme() == "https"); bool https = (uri.getScheme() == "https");
auto session = makeHTTPSessionImpl(host, port, https); auto session = makeHTTPSessionImpl(host, port, https, false);
setTimeouts(*session, timeouts); setTimeouts(*session, timeouts);
return session; return session;
} }

2
dbms/src/IO/HTTPCommon.h
View File

@ -40,7 +40,7 @@ public:
SingleEndpointHTTPSessionPool(const std::string & host_, UInt16 port_, bool https_, size_t max_pool_size_); SingleEndpointHTTPSessionPool(const std::string & host_, UInt16 port_, bool https_, size_t max_pool_size_);
}; };
using PooledHTTPSessionPtr = SingleEndpointHTTPSessionPool::Entry; using PooledHTTPSessionPtr = SingleEndpointHTTPSessionPool::Entry;
using HTTPSessionPtr = std::unique_ptr<Poco::Net::HTTPClientSession>; using HTTPSessionPtr = std::shared_ptr<Poco::Net::HTTPClientSession>;
void setResponseDefaultHeaders(Poco::Net::HTTPServerResponse & response, unsigned keep_alive_timeout); void setResponseDefaultHeaders(Poco::Net::HTTPServerResponse & response, unsigned keep_alive_timeout);

2
dbms/src/IO/ReadWriteBufferFromHTTP.h
View File

@ -50,7 +50,7 @@ namespace detail
: ReadBuffer(nullptr, 0) : ReadBuffer(nullptr, 0)
, uri {uri} , uri {uri}
, method {!method.empty() ? method : out_stream_callback ? Poco::Net::HTTPRequest::HTTP_POST : Poco::Net::HTTPRequest::HTTP_GET} , method {!method.empty() ? method : out_stream_callback ? Poco::Net::HTTPRequest::HTTP_POST : Poco::Net::HTTPRequest::HTTP_GET}
, session {std::move(session_)} , session {session_}
{ {
// With empty path poco will send "POST HTTP/1.1" its bug. // With empty path poco will send "POST HTTP/1.1" its bug.
if (uri.getPath().empty()) if (uri.getPath().empty())

11
dbms/tests/integration/test_max_http_connections_for_replication/configs/log_conf.xml Normal file
View File

@ -0,0 +1,11 @@
<yandex>
<logger>
<level>trace</level>
<log>/var/log/clickhouse-server/log.log</log>
<errorlog>/var/log/clickhouse-server/log.err.log</errorlog>
<size>1000M</size>
<count>10</count>
<stderr>/var/log/clickhouse-server/stderr.log</stderr>
<stdout>/var/log/clickhouse-server/stdout.log</stdout>
</logger>
</yandex>

10
dbms/tests/integration/test_max_http_connections_for_replication/test.py
View File

@ -22,8 +22,8 @@ def _fill_nodes(nodes, shard, connections_count):
'''.format(shard=shard, replica=node.name, connections=connections_count)) '''.format(shard=shard, replica=node.name, connections=connections_count))
cluster = ClickHouseCluster(__file__) cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', config_dir="configs", main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node1 = cluster.add_instance('node1', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True)
node2 = cluster.add_instance('node2', config_dir="configs", main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node2 = cluster.add_instance('node2', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def start_small_cluster(): def start_small_cluster():
@ -52,9 +52,9 @@ def test_single_endpoint_connections_count(start_small_cluster):
assert node2.query("SELECT value FROM system.events where event='CreatedHTTPConnections'") == '1\n' assert node2.query("SELECT value FROM system.events where event='CreatedHTTPConnections'") == '1\n'
node3 = cluster.add_instance('node3', config_dir="configs", main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node3 = cluster.add_instance('node3', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True)
node4 = cluster.add_instance('node4', config_dir="configs", main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node4 = cluster.add_instance('node4', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True)
node5 = cluster.add_instance('node5', config_dir="configs", main_configs=['configs/remote_servers.xml'], with_zookeeper=True) node5 = cluster.add_instance('node5', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/log_conf.xml'], with_zookeeper=True)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def start_big_cluster(): def start_big_cluster():