diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index c4b0c77a026..39c9fe5fc97 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -58,6 +58,7 @@ namespace ErrorCodes extern const int NO_ELEMENTS_IN_CONFIG; extern const int SUPPORT_IS_DISABLED; extern const int ARGUMENT_OUT_OF_BOUND; + extern const int EXCESSIVE_ELEMENT_IN_CONFIG; } @@ -209,25 +210,38 @@ int Server::main(const std::vector & /*args*/) Poco::File(user_files_path).createDirectories(); } - if (config().has("interserver_http_port")) + if (config().has("interserver_http_port") && config().has("interserver_https_port")) + throw Exception("Both http and https interserver ports are specified", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); + + static const auto interserver_tags = { - String this_host = config().getString("interserver_http_host", ""); + std::make_tuple("interserver_http_host", "interserver_http_port", "http"), + std::make_tuple("interserver_https_host", "interserver_https_port", "https") + }; - if (this_host.empty()) + for (auto [host_tag, port_tag, scheme] : interserver_tags) + { + if (config().has(port_tag)) { - this_host = getFQDNOrHostName(); - LOG_DEBUG(log, - "Configuration parameter 'interserver_http_host' doesn't exist or exists and empty. Will use '" + this_host - + "' as replica host."); + String this_host = config().getString(host_tag, ""); + + if (this_host.empty()) + { + this_host = getFQDNOrHostName(); + LOG_DEBUG(log, + "Configuration parameter '" + String(host_tag) + "' doesn't exist or exists and empty. Will use '" + this_host + + "' as replica host."); + } + + String port_str = config().getString(port_tag); + int port = parse(port_str); + + if (port < 0 || port > 0xFFFF) + throw Exception("Out of range '" + String(port_tag) + "': " + toString(port), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + global_context->setInterserverIOAddress(this_host, port); + global_context->setInterserverScheme(scheme); } - - String port_str = config().getString("interserver_http_port"); - int port = parse(port_str); - - if (port < 0 || port > 0xFFFF) - throw Exception("Out of range 'interserver_http_port': " + toString(port), ErrorCodes::ARGUMENT_OUT_OF_BOUND); - - global_context->setInterserverIOAddress(this_host, port); } if (config().has("interserver_http_credentials")) @@ -238,7 +252,7 @@ int Server::main(const std::vector & /*args*/) if (user.empty()) throw Exception("Configuration parameter interserver_http_credentials user can't be empty", ErrorCodes::NO_ELEMENTS_IN_CONFIG); - global_context->setInterverserCredentials(user, password); + global_context->setInterserverCredentials(user, password); } if (config().has("macros")) @@ -516,6 +530,27 @@ int Server::main(const std::vector & /*args*/) LOG_INFO(log, "Listening interserver http: " + address.toString()); } + + if (config().has("interserver_https_port")) + { +#if USE_POCO_NETSSL + initSSL(); + Poco::Net::SecureServerSocket socket; + auto address = socket_bind_listen(socket, listen_host, config().getInt("interserver_https_port"), /* secure = */ true); + socket.setReceiveTimeout(settings.http_receive_timeout); + socket.setSendTimeout(settings.http_send_timeout); + servers.emplace_back(new Poco::Net::HTTPServer( + new InterserverIOHTTPHandlerFactory(*this, "InterserverIOHTTPHandler-factory"), + server_pool, + socket, + http_params)); + + LOG_INFO(log, "Listening interserver https: " + address.toString()); +#else + throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.", + ErrorCodes::SUPPORT_IS_DISABLED}; +#endif + } } catch (const Poco::Net::NetException & e) { diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 0561c2f11c2..bdf2c264b97 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -111,6 +111,7 @@ struct ContextShared UInt16 interserver_io_port = 0; /// and port. String interserver_io_user; String interserver_io_password; + String interserver_scheme; /// http or https String path; /// Path to the data directory, with a slash at the end. String tmp_path; /// The path to the temporary files that occur when processing the request. @@ -1380,7 +1381,16 @@ void Context::setInterserverIOAddress(const String & host, UInt16 port) shared->interserver_io_port = port; } -void Context::setInterverserCredentials(const String & user, const String & password) +std::pair Context::getInterserverIOAddress() const +{ + if (shared->interserver_io_host.empty() || shared->interserver_io_port == 0) + throw Exception("Parameter 'interserver_http(s)_port' required for replication is not specified in configuration file.", + ErrorCodes::NO_ELEMENTS_IN_CONFIG); + + return { shared->interserver_io_host, shared->interserver_io_port }; +} + +void Context::setInterserverCredentials(const String & user, const String & password) { shared->interserver_io_user = user; shared->interserver_io_password = password; @@ -1391,14 +1401,14 @@ std::pair Context::getInterserverCredentials() const return { shared->interserver_io_user, shared->interserver_io_password }; } - -std::pair Context::getInterserverIOAddress() const +void Context::setInterserverScheme(const String & scheme) { - if (shared->interserver_io_host.empty() || shared->interserver_io_port == 0) - throw Exception("Parameter 'interserver_http_port' required for replication is not specified in configuration file.", - ErrorCodes::NO_ELEMENTS_IN_CONFIG); + shared->interserver_scheme = scheme; +} - return { shared->interserver_io_host, shared->interserver_io_port }; +String Context::getInterserverScheme() const +{ + return shared->interserver_scheme; } UInt16 Context::getTCPPort() const diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 38a0e7cb4bc..fe6483affe5 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -250,10 +250,14 @@ public: void setInterserverIOAddress(const String & host, UInt16 port); std::pair getInterserverIOAddress() const; - // Credentials which server will use to communicate with others - void setInterverserCredentials(const String & user, const String & password); + /// Credentials which server will use to communicate with others + void setInterserverCredentials(const String & user, const String & password); std::pair getInterserverCredentials() const; + /// Interserver requests scheme (http or https) + void setInterserverScheme(const String & scheme); + String getInterserverScheme() const; + /// The port that the server listens for executing SQL queries. UInt16 getTCPPort() const; diff --git a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp index 39db6142605..bffbb8a11a6 100644 --- a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp @@ -163,11 +163,12 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( const ConnectionTimeouts & timeouts, const String & user, const String & password, + const String & interserver_scheme, bool to_detached, const String & tmp_prefix_) { Poco::URI uri; - uri.setScheme("http"); + uri.setScheme(interserver_scheme); uri.setHost(host); uri.setPort(port); uri.setQueryParameters( diff --git a/dbms/src/Storages/MergeTree/DataPartsExchange.h b/dbms/src/Storages/MergeTree/DataPartsExchange.h index 32eb80e96ca..d97687da886 100644 --- a/dbms/src/Storages/MergeTree/DataPartsExchange.h +++ b/dbms/src/Storages/MergeTree/DataPartsExchange.h @@ -56,6 +56,7 @@ public: const ConnectionTimeouts & timeouts, const String & user, const String & password, + const String & interserver_scheme, bool to_detached = false, const String & tmp_prefix_ = ""); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index d4ab1281000..cf4a2321721 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -1972,9 +1972,10 @@ bool StorageReplicatedMergeTree::executeReplaceRange(const LogEntry & entry) ReplicatedMergeTreeAddress address(getZooKeeper()->get(replica_path + "/host")); auto timeouts = ConnectionTimeouts::getHTTPTimeouts(context.getSettingsRef()); auto [user, password] = context.getInterserverCredentials(); + String interserver_scheme = context.getInterserverScheme(); part_desc->res_part = fetcher.fetchPart(part_desc->found_new_part_name, replica_path, - address.host, address.replication_port, timeouts, user, password, false, TMP_PREFIX + "fetch_"); + address.host, address.replication_port, timeouts, user, password, interserver_scheme, false, TMP_PREFIX + "fetch_"); /// TODO: check columns_version of fetched part @@ -2704,10 +2705,11 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin ReplicatedMergeTreeAddress address(getZooKeeper()->get(replica_path + "/host")); auto timeouts = ConnectionTimeouts::getHTTPTimeouts(context.getSettingsRef()); auto [user, password] = context.getInterserverCredentials(); + String interserver_scheme = context.getInterserverScheme(); try { - part = fetcher.fetchPart(part_name, replica_path, address.host, address.replication_port, timeouts, user, password, to_detached); + part = fetcher.fetchPart(part_name, replica_path, address.host, address.replication_port, timeouts, user, password, interserver_scheme, to_detached); if (!to_detached) { diff --git a/dbms/tests/integration/test_https_replication/__init__.py b/dbms/tests/integration/test_https_replication/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_https_replication/configs/config.xml b/dbms/tests/integration/test_https_replication/configs/config.xml new file mode 100644 index 00000000000..35a43b2fc54 --- /dev/null +++ b/dbms/tests/integration/test_https_replication/configs/config.xml @@ -0,0 +1,364 @@ + + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + + 8123 + 9000 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 4096 + 3 + + + 100 + + + + + + 8589934592 + + + 5368709120 + + + + /var/lib/clickhouse/ + + + /var/lib/clickhouse/tmp/ + + + /var/lib/clickhouse/user_files/ + + + users.xml + + + default + + + + + + default + + + + + + + + + + + + + + localhost + 9000 + + + + + + + localhost + 9440 + 1 + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + system + query_log
+ + toYYYYMM(event_date) + + 7500 +
+ + + + + + + + + + + + + + + + *_dictionary.xml + + + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + +
diff --git a/dbms/tests/integration/test_https_replication/configs/no_ssl_conf.xml b/dbms/tests/integration/test_https_replication/configs/no_ssl_conf.xml new file mode 100644 index 00000000000..db43fd59e99 --- /dev/null +++ b/dbms/tests/integration/test_https_replication/configs/no_ssl_conf.xml @@ -0,0 +1,3 @@ + + 9010 + diff --git a/dbms/tests/integration/test_https_replication/configs/remote_servers.xml b/dbms/tests/integration/test_https_replication/configs/remote_servers.xml new file mode 100644 index 00000000000..ce36da06e9a --- /dev/null +++ b/dbms/tests/integration/test_https_replication/configs/remote_servers.xml @@ -0,0 +1,49 @@ + + + + + true + + test + node1 + 9000 + + + test + node2 + 9000 + + + + + + true + + test + node3 + 9000 + + + test + node4 + 9000 + + + + + + true + + test + node5 + 9000 + + + test + node6 + 9000 + + + + + diff --git a/dbms/tests/integration/test_https_replication/configs/server.crt b/dbms/tests/integration/test_https_replication/configs/server.crt new file mode 100644 index 00000000000..7ade2d96273 --- /dev/null +++ b/dbms/tests/integration/test_https_replication/configs/server.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV +BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow +FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI +4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T +4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU +7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj +sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg +pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC +kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw +DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j +4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr +85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C +L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD ++UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L +P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp +0g== +-----END CERTIFICATE----- diff --git a/dbms/tests/integration/test_https_replication/configs/server.key b/dbms/tests/integration/test_https_replication/configs/server.key new file mode 100644 index 00000000000..f0fb61ac443 --- /dev/null +++ b/dbms/tests/integration/test_https_replication/configs/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn +DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L +OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus +TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7 +JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F +KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6 +ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP +TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f +7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN +MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5 +212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu +plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr +/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu +Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa +CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g +1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5 +bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq +cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez +on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K +GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54 +nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU +2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh +vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa +dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr +vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1 +KCBtmIrQLqzMpnBpLNuSY+Q= +-----END PRIVATE KEY----- diff --git a/dbms/tests/integration/test_https_replication/configs/ssl_conf.xml b/dbms/tests/integration/test_https_replication/configs/ssl_conf.xml new file mode 100644 index 00000000000..237bbc6af1c --- /dev/null +++ b/dbms/tests/integration/test_https_replication/configs/ssl_conf.xml @@ -0,0 +1,18 @@ + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + none + true + + + true + none + + AcceptCertificateHandler + + + + 9010 + diff --git a/dbms/tests/integration/test_https_replication/test.py b/dbms/tests/integration/test_https_replication/test.py new file mode 100644 index 00000000000..ba0a4de9164 --- /dev/null +++ b/dbms/tests/integration/test_https_replication/test.py @@ -0,0 +1,103 @@ +import time +import pytest + +from helpers.cluster import ClickHouseCluster + +""" +Both ssl_conf.xml and no_ssl_conf.xml have the same port +""" + +def _fill_nodes(nodes, shard): + for node in nodes: + node.query( + ''' + CREATE DATABASE test; + + CREATE TABLE test_table(date Date, id UInt32, dummy UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/test{shard}/replicated', '{replica}', date, id, 8192); + '''.format(shard=shard, replica=node.name)) + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml'], with_zookeeper=True) +node2 = cluster.add_instance('node2', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml'], with_zookeeper=True) + +@pytest.fixture(scope="module") +def both_https_cluster(): + try: + cluster.start() + + _fill_nodes([node1, node2], 1) + + yield cluster + + finally: + cluster.shutdown() + +def test_both_https(both_https_cluster): + node1.query("insert into test_table values ('2017-06-16', 111, 0)") + time.sleep(1) + + assert node1.query("SELECT id FROM test_table order by id") == '111\n' + assert node2.query("SELECT id FROM test_table order by id") == '111\n' + + node2.query("insert into test_table values ('2017-06-17', 222, 1)") + time.sleep(1) + + assert node1.query("SELECT id FROM test_table order by id") == '111\n222\n' + assert node2.query("SELECT id FROM test_table order by id") == '111\n222\n' + +node3 = cluster.add_instance('node3', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], with_zookeeper=True) +node4 = cluster.add_instance('node4', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], with_zookeeper=True) + +@pytest.fixture(scope="module") +def both_http_cluster(): + try: + cluster.start() + + _fill_nodes([node3, node4], 2) + + yield cluster + + finally: + cluster.shutdown() + +def test_both_http(both_http_cluster): + node3.query("insert into test_table values ('2017-06-16', 111, 0)") + time.sleep(1) + + assert node3.query("SELECT id FROM test_table order by id") == '111\n' + assert node4.query("SELECT id FROM test_table order by id") == '111\n' + + node4.query("insert into test_table values ('2017-06-17', 222, 1)") + time.sleep(1) + + assert node3.query("SELECT id FROM test_table order by id") == '111\n222\n' + assert node4.query("SELECT id FROM test_table order by id") == '111\n222\n' + +node5 = cluster.add_instance('node5', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/ssl_conf.xml'], with_zookeeper=True) +node6 = cluster.add_instance('node6', config_dir="configs", main_configs=['configs/remote_servers.xml', 'configs/no_ssl_conf.xml'], with_zookeeper=True) + +@pytest.fixture(scope="module") +def mixed_protocol_cluster(): + try: + cluster.start() + + _fill_nodes([node5, node6], 3) + + yield cluster + + finally: + cluster.shutdown() + +def test_mixed_protocol(mixed_protocol_cluster): + node5.query("insert into test_table values ('2017-06-16', 111, 0)") + time.sleep(1) + + assert node5.query("SELECT id FROM test_table order by id") == '111\n' + assert node6.query("SELECT id FROM test_table order by id") == '' + + node6.query("insert into test_table values ('2017-06-17', 222, 1)") + time.sleep(1) + + assert node5.query("SELECT id FROM test_table order by id") == '111\n' + assert node6.query("SELECT id FROM test_table order by id") == '222\n'