diff --git a/contrib/NuRaft b/contrib/NuRaft index c2b0811f164..c11f7fce687 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit c2b0811f164a7948208489562dab4f186eb305ce +Subproject commit c11f7fce68737cdc67a1d61678b2717d617ebb5a diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index bfe50288a40..c72f57c49d2 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -54,11 +54,10 @@ namespace DB namespace ErrorCodes { extern const int RAFT_ERROR; - extern const int NO_ELEMENTS_IN_CONFIG; extern const int SUPPORT_IS_DISABLED; extern const int LOGICAL_ERROR; extern const int INVALID_CONFIG_PARAMETER; - extern const int BAD_ARGUMENTS; + extern const int OPENSSL_ERROR; } using namespace std::chrono_literals; @@ -68,47 +67,38 @@ namespace #if USE_SSL -int callSetCertificate(SSL * ssl, void * arg) +auto getSslContextProvider(const Poco::Util::AbstractConfiguration & config, std::string_view key) { - if (!arg) - return -1; - - const CertificateReloader::Data * data = reinterpret_cast(arg); - return setCertificateCallback(ssl, data, getLogger("SSLContext")); -} - -void setSSLParams(nuraft::asio_service::options & asio_opts) -{ - const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config(); - String certificate_file_property = "openSSL.server.certificateFile"; - String private_key_file_property = "openSSL.server.privateKeyFile"; - String root_ca_file_property = "openSSL.server.caConfig"; - - if (!config.has(certificate_file_property)) - throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Server certificate file is not set."); - - if (!config.has(private_key_file_property)) - throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Server private key file is not set."); + String load_default_ca_file_property = fmt::format("openSSL.{}.loadDefaultCAFile", key); + String verification_mode_property = fmt::format("openSSL.{}.verificationMode", key); + String root_ca_file_property = fmt::format("openSSL.{}.caConfig", key); + String private_key_passphrase_property = fmt::format("openSSL.{}.privateKeyPassphraseHandler.options.password", key); Poco::Net::Context::Params params; - params.certificateFile = config.getString(certificate_file_property); - if (params.certificateFile.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Server certificate file in config '{}' is empty", certificate_file_property); + String certificate_file_property = fmt::format("openSSL.{}.certificateFile", key); + String private_key_file_property = fmt::format("openSSL.{}.privateKeyFile", key); + if (config.has(certificate_file_property)) + params.certificateFile = config.getString(certificate_file_property); - params.privateKeyFile = config.getString(private_key_file_property); - if (params.privateKeyFile.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Server key file in config '{}' is empty", private_key_file_property); + if (config.has(private_key_file_property)) + params.privateKeyFile = config.getString(private_key_file_property); - auto pass_phrase = config.getString("openSSL.server.privateKeyPassphraseHandler.options.password", ""); - auto certificate_data = std::make_shared(params.certificateFile, params.privateKeyFile, pass_phrase); + std::shared_ptr certificate_data; + if (config.has(private_key_passphrase_property)) + { + certificate_data = std::make_shared( + params.certificateFile, params.privateKeyFile, config.getString(private_key_passphrase_property)); + params.certificateFile.clear(); + params.privateKeyFile.clear(); + } if (config.has(root_ca_file_property)) params.caLocation = config.getString(root_ca_file_property); - params.loadDefaultCAs = config.getBool("openSSL.server.loadDefaultCAFile", false); - params.verificationMode = Poco::Net::Utility::convertVerificationMode(config.getString("openSSL.server.verificationMode", "none")); + params.loadDefaultCAs = config.getBool(load_default_ca_file_property, false); + params.verificationMode = Poco::Net::Utility::convertVerificationMode(config.getString(verification_mode_property, "none")); - std::string disabled_protocols_list = config.getString("openSSL.server.disableProtocols", ""); + std::string disabled_protocols_list = config.getString(fmt::format("openSSL.{}.disableProtocols", key), ""); Poco::StringTokenizer dp_tok(disabled_protocols_list, ";,", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY); int disabled_protocols = 0; for (const auto & token : dp_tok) @@ -125,21 +115,54 @@ void setSSLParams(nuraft::asio_service::options & asio_opts) disabled_protocols |= Poco::Net::Context::PROTO_TLSV1_2; } - asio_opts.ssl_context_provider_server_ = [params, certificate_data, disabled_protocols] + auto prefer_server_cypher = config.getBool(fmt::format("openSSL.{}.preferServerCiphers", key), false); + auto cache_sessions = config.getBool(fmt::format("openSSL.{}.cache_sessions", key), false); + return [params, disabled_protocols, prefer_server_cypher, cache_sessions, is_server = key == "server", certificate_data] { - Poco::Net::Context context(Poco::Net::Context::Usage::TLSV1_2_SERVER_USE, params); + Poco::Net::Context context(is_server ? Poco::Net::Context::Usage::SERVER_USE : Poco::Net::Context::Usage::CLIENT_USE, params); context.disableProtocols(disabled_protocols); - SSL_CTX * ssl_ctx = context.takeSslContext(); - SSL_CTX_set_cert_cb(ssl_ctx, callSetCertificate, reinterpret_cast(certificate_data.get())); - return ssl_ctx; - }; - asio_opts.ssl_context_provider_client_ = [ctx_params = std::move(params)] - { - Poco::Net::Context context(Poco::Net::Context::Usage::TLSV1_2_CLIENT_USE, ctx_params); + if (prefer_server_cypher) + context.preferServerCiphers(); + + if (cache_sessions) + context.enableSessionCache(); + + auto * ssl_ctx = context.sslContext(); + if (certificate_data) + { + if (auto err = SSL_CTX_clear_chain_certs(ssl_ctx); err != 1) + throw Exception(ErrorCodes::OPENSSL_ERROR, "Clear certificates {}", Poco::Net::Utility::getLastError()); + + if (auto err = SSL_CTX_use_certificate(ssl_ctx, const_cast(certificate_data->certs_chain[0].certificate())); err != 1) + throw Exception(ErrorCodes::OPENSSL_ERROR, "Use certificate {}", Poco::Net::Utility::getLastError()); + + for (auto cert = certificate_data->certs_chain.begin() + 1; cert != certificate_data->certs_chain.end(); cert++) + { + if (auto err = SSL_CTX_add1_chain_cert(ssl_ctx, const_cast(cert->certificate())); err != 1) + throw Exception(ErrorCodes::OPENSSL_ERROR, "Add certificate to chain {}", Poco::Net::Utility::getLastError()); + } + + if (auto err = SSL_CTX_use_PrivateKey(ssl_ctx, const_cast(static_cast(certificate_data->key))); err != 1) + throw Exception(ErrorCodes::OPENSSL_ERROR, "Use private key {}", Poco::Net::Utility::getLastError()); + + if (auto err = SSL_CTX_check_private_key(ssl_ctx); err != 1) + throw Exception(ErrorCodes::OPENSSL_ERROR, "Unusable key-pair {}", Poco::Net::Utility::getLastError()); + } + + return context.takeSslContext(); }; } + +void setSSLParams(nuraft::asio_service::options & asio_opts) +{ + asio_opts.enable_ssl_ = true; + + const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config(); + asio_opts.ssl_context_provider_server_ = getSslContextProvider(config, "server"); + asio_opts.ssl_context_provider_client_ = getSslContextProvider(config, "client"); +} #endif std::string checkAndGetSuperdigest(const String & user_and_digest) @@ -242,7 +265,7 @@ struct KeeperServer::KeeperRaftServer : public nuraft::raft_server } const size_t voting_members = get_num_voting_members(); - const auto not_responding_peers = get_not_responding_peers(); + const auto not_responding_peers = get_not_responding_peers_count(); const auto quorum_size = voting_members / 2 + 1; const auto max_not_responding_peers = voting_members - quorum_size; @@ -279,6 +302,11 @@ struct KeeperServer::KeeperRaftServer : public nuraft::raft_server return std::unique_lock(lock_); } + std::unique_lock lockCommit() + { + return std::unique_lock(commit_lock_); + } + bool isCommitInProgress() const { return sm_commit_exec_in_progress_; @@ -454,7 +482,6 @@ void KeeperServer::launchRaftServer(const Poco::Util::AbstractConfiguration & co throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSL support for NuRaft is disabled because ClickHouse was built without SSL support."); #endif } - if (is_recovering) enterRecoveryMode(params); @@ -1212,6 +1239,7 @@ Keeper4LWInfo KeeperServer::getPartiallyFilled4LWInfo() const uint64_t KeeperServer::createSnapshot() { + auto commit_lock = raft_instance->lockCommit(); uint64_t log_idx = raft_instance->create_snapshot(); if (log_idx != 0) LOG_INFO(log, "Snapshot creation scheduled with last committed log index {}.", log_idx); diff --git a/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.crt b/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.crt index cabc53fa809..05ce3ef8ee4 100644 --- a/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.crt +++ b/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.crt @@ -1,20 +1,22 @@ -----BEGIN CERTIFICATE----- -MIIDPDCCAiQCFBXNOvsLA+dqmX/TkYG9JXdD5m72MA0GCSqGSIb3DQEBCwUAMFox -CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl -cm5ldCBXaWRnaXRzIFB0eSBMdGQxEzARBgNVBAMMCmNsaWNraG91c2UwIBcNMjIw -NDIxMTAzNDU1WhgPMjEyMjAzMjgxMDM0NTVaMFkxCzAJBgNVBAYTAkFVMRMwEQYD -VQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBM -dGQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC -AQoCggEBAKaXz596N4NC2zZdIqdwZbSYAtNdBCsBVPt5YT9F640aF5zOogPZyxGP -ENyOZwABi/7HhwFbH657xyRvi8lTau8dZL+0tbakyoIn1Tw6j+/3GXTjLduJSy6C -mOf4OzsrFC8mYgU+7p5ijvWVlO9h5NMbLdAPSIB5WSHhmSORH5LgjoK6oMOYdRod -GmfHqSbwPVwy3Li5SXlniCQmJsM0zl64LFbJ/NU+13qETmhBiDgmh0Svi+wzSzqZ -q1PIX92T3k44IXNZbvF7lKbUOS9Xb3BoxA4cDqRcTx4x73xRDwodSmqiuQOC99HI -A0C/tZJ25VNAGjLKukPSHqYscq2PAsUCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEA -IDQwjf/ja3TfOXrz+Gn1eErSKnWS3asjRT9rYWQsy3tzVUkMIcszrG+FqTR16g5H -ZWyuEOi6KIRmda3SYKdLKmtQLrgx6/d/jvH5TQ0LTFZrp6vh0lo3pV+L6fLo1ZRD -V1i8jW/7HHNyqJamUXOjwA0DpPOMkdtwuyV+rJ+2bTG1ZSK33O4Ae2CY5+dad6zy -YI6b1c9flWfDznuNEMH7jDDjKgXwjZGeU53FiuuhHiNyRchsr/B9eIBsom8oykiD -kch4cnAxx2E+m3hLYzupkXHOVQ5CNpVk8PGUCIGcyqDxPt+fOj1WbDQ9laEcfhmV -kR+vHmzOqWZnHU4QeMqDig== +MIIDtjCCAp6gAwIBAgIUdOfco+b8/fQZQOafHgghkEYL3YkwDQYJKoZIhvcNAQEL +BQAwazELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5 +MRAwDgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRwwGgYDVQQDDBNUZXN0 +Y2x1c3RlciBSb290IENBMB4XDTI0MTIwMjE0MjkyNloXDTM0MTEzMDE0MjkyNlow +ajELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5MRAw +DgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRswGQYDVQQDDBJjbGlja2hv +dXNlLWtlZXBlcjEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDUXmnn +Cv7sY9lbrS1Q3c5q7ok9R4XEPq/jWBFkIEnJR7vSjCEnOjLxg+1MdUItjbqODf9N +5vFbHiiqWQVkGrmg8/CTSme0qyNr7FcmG1hO4bzK/dvIyK1R7YISqZpXoCTVzEnU +IjU7f+PkX2uAiSypxM4zpNyC7++j6ah8xYNRfR9AS5c7e1dvNKBNMmNipYxVgaEo +pIke40m12ezIzLOtkL/rGlsnM2Tv/0Wv1xQE+OjHByyQE08vuliatFfweTXLF48m +4S4NdOq5dh2WX8xLPr8BxRLjXzs08wgKVFpWkIOR2uEInjuVQAGMuZeOqUuQGdar +GMH4M/3tDl0eJ7mbAgMBAAGjUzBRMB0GA1UdDgQWBBTr9ldBtTB0vatq2yhQgYtt +zMNhJDAfBgNVHSMEGDAWgBTGSqv6LHbQlKrpPWtYEVoX+/c5cTAPBgNVHRMBAf8E +BTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBYc03AV8n0D43xm07MxpgqDvNEZC9u +Q2LnP89UBnBmXD5FwMz4XhA/iupyAeYItZ8R17caIpLHgwOUrh3oHxVW5V144Q0p +hBBp/im8WQ8NnS3z52CusxE1Zu5AMjoZtxY8FTvgs6vuJZYds/dgtUg5bBawR2LX +A5FsPLyYpwCjoPTM622uXkuPfRMc5SC5edwHa1RyoG8Poz8B6Y63iKQydOXin9Q1 +9rQ7mqM7D2dCURx4gVoN9y+fLkXgQEzTMBT4wuVJl+CXnxcmKsoROAy7g2mL0RMw +P3cl+Bod3NrabhjAqG01nHsQzy0uJ/aJHbqoR3OtYo8DdsoKrBRramiG -----END CERTIFICATE----- diff --git a/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.key b/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.key index 1e29a4c8fa1..ac20648bb8b 100644 --- a/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.key +++ b/tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.key @@ -1,30 +1,28 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: AES-256-CBC,4E14FF586022476CD22AAFB662BB0E40 - -dpJZKq5k+fMuC7XECfTSRjPeOEl9wNuVtZkcjEWaHN8ky4umND7ARyRyuU1Nk7cy -fpCFlFKOqDfCkT5zVK/fB6pF32wqAI7sqeSuYPfQY0+L77yRdwM6L46WslzVKZYE -lXD1AmqKT/LgF3+eBY5slkAAJo10zYDgKEwnoQVBp31YW2/+6oAGaY/O6x3p7aTG -dw9CP+SFc0o8lPl1lsSovdNXDUiVCftvClog7hwyDv8AhHyGgynw3UJXX8UlyWu+ -Zz5zpgrvB2gvDLeoZZ6qjMGvtpEwlYBh4de9ZOsvQUpXEEfkQFtJV0j613OCQune -LoTxKpYV1V/mZX4HPaJ1oC0OJ7XcliAOSS9K49YobTuz7Kg5Wg3bVGo9xRfYDjch -rVeMP4u5RXSGuHL23FPMfK6EqcldrFyRwLaY/IV1Yl6UNUMKAphn/WMtWVuT3TiT -QMCI2VRt7ItwZwwFn5RgyDweWdFf5v3AmN/lOhATDBqosahkPxDZ1VZ6OBPoJLPM -UrDWH/lqrByeEjtAOwr5UsWKwLuJ8qUxQ4TchHwFKOwy6VsrRwMQ3ZWi2govPF9I -W0sfLj5Ulfjx6zHdqnF48a1Elit4JH6inBEWFuj7bmlOotq+PHoeT61zAwW+gnrG -3JTo3XnaE2WwRDpqvKYHWLv/J218rq8PtIaq9gjr55odPfIt8lkJ1XzF4WQ21rIJ -GNWZ3xz4fxpvrKnQyAKGu0ZcdjA1nqs16oiVr+UnJoXmkM5yBCic4fZYwPTSQHYS -ZxwaTzEjfeGxrSeLrN9CgoweucvogOvUjJOBcW/py80du8vWz0YyzMhg3o0YeGME -C+Kts/YWxmyfw4DaWt8RtWCKl85hEmz8RODvkMLGtLzvVoSyLQWqp1NhGIlFtzXs -7sPLulUeyD2avTC/RB/Pu9Nk80c0368BxCoeYbiFWZpaN70SJmCUE5H59J2d0olw -5v2RVjLBi8wqnzoa0+2L8wnG7IQGadS97dj0eBR+JXNtoJhVrurS80RJ6B0bNxdu -gX8otfnJYsZyK5hbEhcQqLdnyGhDEE8YHe7Hv9stWwLAFOfOMzyzC06lFS1eNiw4 -FJyXJUhDieb8EqetouAC8dNVXz4Q1zOTlGuAbGoKm5v0U5IhCQap9GUSW5QiUgOQ -AEMs9aGfd91R+IcDf19mZptsQLYA6MGBN6fm+3O2iZImKIbF+ZZo0S6liFFmn6lm -M+diTzaoiqgEkiXOuRhdQUMaiGV8BMZxv8qUH6/vyC3gSueoTio0f9PfASDYfvXD -A3GuI87P6LF1it2UlN6ssFoXTZdfQQZwRmNuqOqw+BJOJHrR6trcXOCZOQ77Qnvd -M5a348gIzluVUkExAPGCsySQWMx4Of5NBF28jEC3+TAwkRqBV2ZHmfGLWnvwaB+A -YUeKtpWblfG1lsrDAdwL2dilU95oby+35sExX7M2dCrL9Y2P5oTCW3u12//ZSLeL -Yhi1Rzol6LAuesZCVF0Zv/YYDhzAckJfT/qXK5B5pz9saswxCUBEpiKlLpVsjOFJ -2bHm8NgOMD5b3cdh1kvts4wZe+giry7LHsn46f+9VqN+gA6XxeVsPyb4uO1KW3SN ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDUXmnnCv7sY9lb +rS1Q3c5q7ok9R4XEPq/jWBFkIEnJR7vSjCEnOjLxg+1MdUItjbqODf9N5vFbHiiq +WQVkGrmg8/CTSme0qyNr7FcmG1hO4bzK/dvIyK1R7YISqZpXoCTVzEnUIjU7f+Pk +X2uAiSypxM4zpNyC7++j6ah8xYNRfR9AS5c7e1dvNKBNMmNipYxVgaEopIke40m1 +2ezIzLOtkL/rGlsnM2Tv/0Wv1xQE+OjHByyQE08vuliatFfweTXLF48m4S4NdOq5 +dh2WX8xLPr8BxRLjXzs08wgKVFpWkIOR2uEInjuVQAGMuZeOqUuQGdarGMH4M/3t +Dl0eJ7mbAgMBAAECggEAJwzxZlXESI2Xw17VzV/r/Ae+3rDPLSXly+U+1W2Gg+eX +5wBzfDYcdgKvWPba42uDWWnDf3yu9vVVvvU9o4myhqE0pLDy3ur1SXwdDlnK5D5o +K9+AUaxtCnqlB29+fQxqmZHGJabgqP88VZsiNnGC7/jLff2buswKAdcOb1sWaZ5F +p5YBL6TmUzpg3Pdbs2N/OVsZGh5Y+d6m6hJgjsbdcWC4LrFurj9UyQoCGIRruelN +Ra3ft4QAC3biD0/hEc4WyjzZdlMvvEqaeQUmqK2TJLELwyH33W1Ek2FsLm6LTP9i +7kW2+9684GWBXo5ge87BXASVwAXnFE+pnXV5QGLMqQKBgQDpo2ZM/R/lWLR8Hpww +WKZZFfWPpsv0d6DbwVWowdRoojPue62nIby6+LBk08j9UwqkC0TK8pWCjL4hhSfU +JjLuGuzl+RFfstpt42qh2zgi8aedLtGHjFyjHp1jE3rb9l95YXUpNm7syEMbLR8V +NR09na41ftFCBPBNYFsdxrzKPwKBgQDosd6qwhrTL+ndtiJIeonRvunjo4yXn22/ +qWqRy3WJmZpDKWpsGWOmlJ4G7+10Q3zOMpb+nUOEjJNI3EdnLrVTeJo8WNuNXHyl +axgWV3TR8JT6GIG+zavGEI51JjTH3X1eGzm9T4Di8mj34FzyK1af6atiiRj5sIuk +NG71CUjhpQKBgQCDJ90n3vjm0LMQ8kYPxdQsMm2VZLcd14IPmyqw/45z5opsmDVV +m1TNSQoMr+8mdlWE3WaS3zcbAFNDkfJX39G7ZJYUS4t7Q3XnNkEH934975Z+YGfz +RdJDJ86GbcsMa/QQuasBpbMDbTBusxe92gE+M6Q2F6j0/LzBUxQTVRtqFQKBgBFZ +IXatnf5cthzXdVrd9+RxTVKxYMv1EOOXJ+DSwGKP1xZmwg5pHirPLbDqtlNSrL1a +vDMjWmNJb7mg4pnou5ALj8QsA8JYQNq8T0FrJ8R3IUQ8C4BEKShNF7HYNVspQi1/ +7iAVC1DgLb89NPDBFmY5r5NbEUecR+zoE9Wk6ZAZAoGBAIHTifNzf7/qDtHI2+Cc +YGiudMlWWwNqTUr1BjPQx1au1VImpDAB6eaz3DV0oIS0fpREte6SRrcwvtrRqp4M +AT4uCjiVOaXW/MwybfS6BIivTvuTkyPuNCBIWTH1JBQ3CEcEIIV5YcpFDQs5FQ0M +GPtHFxbKMUmLmJVW5nbKUUhr +-----END PRIVATE KEY----- diff --git a/tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.crt b/tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.crt index dfa32da5444..686cd645554 100644 --- a/tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.crt +++ b/tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.crt @@ -1,19 +1,22 @@ -----BEGIN CERTIFICATE----- -MIIDETCCAfkCFHL+gKBQnU0P73/nrFrGaVPauTPmMA0GCSqGSIb3DQEBCwUAMEUx -CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl -cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjEwNDEyMTE0NzI5WhcNMjEwNTEyMTE0 -NzI5WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE -CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEA1iPeYn1Vy4QnQi6uNVqQnFLr0u3qdrMjGEBNAOuGmtIdhIn8 -rMCzaehNr3y2YTMRbZAqmv28P/wOXpzR1uQaFlQzTOjmsn/HOZ9JX2hv5sBUv7SU -UiPJS7UtptKDPbLv3N/v1dOXbY+vVyzo8U1Q9OS1J5yhYW6KtxP++hfSrOsFu669 -d1pqWFWaNBsmf0zF+ETvi6lywhyTFA1/PazcStP5GntcDL7eDvGq+DDsRC40oRpy -S4xRQRSteCTtGGmWpx+Jmt+90wFnLgruUbWT0veCoLxLvz0tJUk3ueUVnMkrxBQG -Fz+IWm+SQppNU5LlAcBcu9wJfo3h34BXp0NFNQIDAQABMA0GCSqGSIb3DQEBCwUA -A4IBAQCUnvQsv+GsPwGnIWqH9iiFVhgDx5QbSTW94Fyqk8dcIJBzWAiCshmLBWPJ -pfy4y2nxJbzovFsd9DA49pxqqILeLjue99yma2DVKeo+XDLDN3OX5faIMTBd7AnL -0MKqW7gUSLRUZrNOvFciAY8xRezgBQQBo4mcmmMbAbk5wKndGY6ZZOcY+JwXlqGB -5hyi6ishO8ciiZi3GMFNWWk9ViSfo27IqjKdSkQq1pr3FULvepd6SkdX+NvfZTAH -rG+CSoFGiJcOBbhDkvpY32cAJEnJOA1vHpFxfnGP8/1haeVZHqSwH1cySD78HVtF -fBs000wGHzBYWNI2KkwjNtYf06P4 +MIIDtjCCAp6gAwIBAgIUP0g0uMpZSD2OOtjFXz/anI4EU+swDQYJKoZIhvcNAQEL +BQAwazELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5 +MRAwDgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRwwGgYDVQQDDBNUZXN0 +Y2x1c3RlciBSb290IENBMB4XDTI0MTIwMjE0MjkyNloXDTM0MTEzMDE0MjkyNlow +ajELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5MRAw +DgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRswGQYDVQQDDBJjbGlja2hv +dXNlLWtlZXBlcjEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCgflyz +Kg1deXEXFJIzoyLIAfRPs8MpOsKt06DPVvyZp2ct+g2GCcZlwV4L/GunIV2sugeX +ZHcJ+B06gKSgouxOMFjTnBEdlygLegMeyrJI6TKREiiWMYYxfUVabpC0DtKeZxc/ +D9BY4qLjngxbdRwS7l4eKv74jV9dowDfCNZxXLtzP3uj+AFlLuWk0LP6qFmJMUii +tM7f3oLzxURxIddBASjz12dyQGdm/6v6UcVWnqSDXApozb9LPmapUiJM9axcEvjM +C/Qr14021OEgLVGGEeAAA4JHWZPCqQjbgaDHm5xa61KAMnwDxk/GbMX/TFSwgV4x +pDChT9GKzVMHNf7PAgMBAAGjUzBRMB0GA1UdDgQWBBRZ8QY0I9WoGyFlwyGOs4ZY +cf5tEDAfBgNVHSMEGDAWgBTGSqv6LHbQlKrpPWtYEVoX+/c5cTAPBgNVHRMBAf8E +BTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQB6hUw6IrDGBvGN3AIVatO/6xZX5LZM +Lp5B4uL5rz+6BXf+hZFsj3o4uvyxaEW12m+/bPPOA4EBdShtUfydfMoDHJsnrE8k +D6aVq04f7vjffGeFzvQhfAEnK5/rutWDyq9rXlqcKcPFLhl2Pozk7ty3V+Wz7i3+ +0n2uDTxAfcdlkeSlzPpXP/JOMFN6BwmzrgsyLHyPeIhjfv/lFMoAOblpF6tDFvlY +sXTk0P3Eh9zQ9vT2HI3ZVkqXUe3qQZhUOkKezy0J/OK/6wlvRoO3GXr8/gJO+lJp +ATwurpIc/za08toWbziOOL4xhY4RA+7S9uK3Uz+2a8AoRyUurMP4AHpx -----END CERTIFICATE----- diff --git a/tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.key b/tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.key index 7e57c8b6b34..c4e956e3177 100644 --- a/tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.key +++ b/tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.key @@ -1,27 +1,28 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEA1iPeYn1Vy4QnQi6uNVqQnFLr0u3qdrMjGEBNAOuGmtIdhIn8 -rMCzaehNr3y2YTMRbZAqmv28P/wOXpzR1uQaFlQzTOjmsn/HOZ9JX2hv5sBUv7SU -UiPJS7UtptKDPbLv3N/v1dOXbY+vVyzo8U1Q9OS1J5yhYW6KtxP++hfSrOsFu669 -d1pqWFWaNBsmf0zF+ETvi6lywhyTFA1/PazcStP5GntcDL7eDvGq+DDsRC40oRpy -S4xRQRSteCTtGGmWpx+Jmt+90wFnLgruUbWT0veCoLxLvz0tJUk3ueUVnMkrxBQG -Fz+IWm+SQppNU5LlAcBcu9wJfo3h34BXp0NFNQIDAQABAoIBAHYDso2o8V2F6XTp -8QxqawQcFudaQztDonW9CjMVmks8vRPMUDqMwNP/OMEcBA8xa8tsBm8Ao3zH1suB -tYuujkn8AYHDYVDCZvN0u6UfE3yiRpKYXJ2gJ1HX+d7UaYvZT6P0rmKzh+LTqxhq -Ib7Kk3FDkirQgYgGueAH3x/JfUvaAGvFrq+HvvlhHOs7M7iFU4nJA8jNfBolpTnG -v5MMI+f8/GHGreVICJUoclE+4V/4LDHUlrc3l1kQk0keeD6ECw/pl48TNL6ncXKu -baez1rfKbMPjhLUy2q5UZa93oW+olchEOXs1nUNKUhIOOr0f0YweYhUHNTineVM9 -yTecMIkCgYEA7CFQMyeLVeBA6C9AHBe8Zf/k64cMPyr0lUz6548ulil580PNPbvW -kd2vIKfUMgCO5lMA47ArL4bXZ7cjTvJmPYE1Yv8z+F0Tk03fnTrudHOSBEiGXAu3 -MPTxCDU7Se5Dwj0Fq81aFRtCHl8Rrss+WiBD8eRoxb/vwXKFc6VUAWMCgYEA6CjZ -XrZz11lySBhjkyVXcdLj89hDZ+bPxA7b3VB7TfCxsn5xVck7U3TFkg5Z9XwEQ7Ob -XFAPuwT9GKm7QPp6L8T2RltoJ3ys40UH1RtcNLz2aIo/xSP7lopPdAfWHef5r4y9 -kRw+Gh4NP/l5wefXsRz/D0jY3+t+QnwnhuCKbocCgYEAiR6bPOlkvzyXVH1DxEyA -Sdb8b00f7nqaRyzJsrfxvJ9fQsWHpKa0ZkYOUW9ECLlMQjHHHXEK0vGBmqe9qDWY -63RhtRgvbLVYDb018k7rc9I846Hd7AudmJ9UbIjE4hyrWlsnNOntur32ej6IvTEn -Bx0fd5NEyDi6GGLRXiOOkbMCgYAressLE/yqDlR68CZl/o5cAPU0TAKDyRSMUYQX -9OTC+hstpMSxHlkADlSaQBnVAf8CdvbX2R65FfwYzGEHkGGl5KuDDcd57b2rathG -rzMbpXA4r/u1fkG2Nf0fbABL5ZA7so4mSTXQSmSM4LpO+I7K2vVh9XC4rzAcX4g/ -mHoUrQKBgBf3rxp5h9P3HWoZYjzBDo2FqXUjKLLjE9ed5e/VqecqfHIkmueuNHlN -xifHr7lpsYu6IXkTnlK14pvLoPuwP59dCIOUYwAFz8RlH4MSUGNhYeGA8cqRrhmJ -tYk3OKExuN/+O12kUPVTy6BMH1hBdRJP+7y7lapWsRhZt18y+8Za ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCgflyzKg1deXEX +FJIzoyLIAfRPs8MpOsKt06DPVvyZp2ct+g2GCcZlwV4L/GunIV2sugeXZHcJ+B06 +gKSgouxOMFjTnBEdlygLegMeyrJI6TKREiiWMYYxfUVabpC0DtKeZxc/D9BY4qLj +ngxbdRwS7l4eKv74jV9dowDfCNZxXLtzP3uj+AFlLuWk0LP6qFmJMUiitM7f3oLz +xURxIddBASjz12dyQGdm/6v6UcVWnqSDXApozb9LPmapUiJM9axcEvjMC/Qr1402 +1OEgLVGGEeAAA4JHWZPCqQjbgaDHm5xa61KAMnwDxk/GbMX/TFSwgV4xpDChT9GK +zVMHNf7PAgMBAAECgf9+Cl08oHSJPWifSeoBfvgCfJKel5pj60fT5kcO10Ghy6Vt +IMKisi8dKET8wz/IGcFe8RtmpR4UVK4NXB6YguDhALWEtwwntwfURKiokiWJ+HVD +8s4Fbyht/m1UTqUk23MG2xPgcorkBlWc/pqdaXOWjwpnUXVNXAJketvrKBHTbgq+ +XBvWlYNm+8ThGxm6Ryf3v6K04m5bVAzdQljGBUXRixqEZ71qkQ1TwEBaphbEWPLZ +7nod0yKxQ38ydYrrqqjoINvNLr4OBrlsjuYFAXbwwJmY0L5EleleVDjuiKmbCLUX +CKFaT+YlvHKY5m313ohVFldjRqThjuNt2VtuOFECgYEA0cyCH+sBTaMrYCgi8+l2 +nB+w9zVUqI70naT9zpBlei3DWgklMlNoGRbiIrvFyS2eH3GMq+WZub2/Ci0AU0RG +QYcvp4dfJth6IoKif8Un+RSMW1rN4pPDA88YIr+BKlnkjwfj+71ldBuX75UHXut1 +8z8ThxrmpMCOFql6S5odkSsCgYEAw9ZANtjTBpo2Dff8uPE+Ml3rilRWk3RjX+iU +UbKrsNWwU0QXQu4RbxHaCmJEPFP7bL6W/DkWSvzrxxnnmtQ2UXvXu1jf5D8asliz ++HMIbX3beiKec+C0xbt5xbOYkO4pfurS8V9fYsJTFyDDiMk9cDlHwKfH71yO2Qgp +Zw4Szu0CgYEAtb8awxfMyzsdanGaxf5r+BgkMCQNMPCWzLKQBRBmOI/IegkOJijH +N2TxhfFxCDTylH7DxG5k29ma0+/kJj4xNrcr+090iKxkMd1FdLaRSGAar1Fcpnon +KPeRCxknhk7Vh2rof761Uv5MgwpxljMYvR7ZheMyB2ugK9Wp0jCyiH0CgYAus9B6 +g/jHUU1kxWgKftWTU1yRj41Z+t6cB64fUZmqQTucj9dwSa/0qfAym76kGG8UPtto +6QBM/8YGpEHcZZFSm1MWRZqXJwlp0MeSj3RKEEKf/NOG1OanZQ8kO7E9lt5kewG1 +OEZaGfeQw2p+G2fAdJiM9DY0+gDC9zRQdEW9/QKBgQCIlZ6ToPdBJPuhKTtV0+c/ +e3k0+bKy3OIoW+laaRS3JMMyvveQECaLKMsVMYCppJfKCV/NOUnTIw/g1vXuom6I +QZHGK10aWB8bunblKFrSFxfBStgDCPeZklb4ECQP4+QfLb7Xi7fAvfSi0hFP4LNo +Ea+Ttp/shik8d0sy217IEQ== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper1.xml b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper1.xml index dabf280bc36..bc3d2a44ce2 100644 --- a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper1.xml +++ b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper1.xml @@ -11,6 +11,7 @@ 10000 75 trace + 1000 diff --git a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper2.xml b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper2.xml index 21d8f1e0eb3..39eec44d094 100644 --- a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper2.xml +++ b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper2.xml @@ -11,6 +11,7 @@ 10000 75 trace + 1000 diff --git a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper3.xml b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper3.xml index 5d8cfb8b3e7..88a3d367433 100644 --- a/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper3.xml +++ b/tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper3.xml @@ -11,6 +11,7 @@ 10000 75 trace + 1000 diff --git a/tests/integration/test_keeper_internal_secure/configs/rootCA.pem b/tests/integration/test_keeper_internal_secure/configs/rootCA.pem index ec16533d98a..208259f8f85 100644 --- a/tests/integration/test_keeper_internal_secure/configs/rootCA.pem +++ b/tests/integration/test_keeper_internal_secure/configs/rootCA.pem @@ -1,21 +1,22 @@ -----BEGIN CERTIFICATE----- -MIIDazCCAlOgAwIBAgIUUiyhAav08YhTLfUIXLN/0Ln09n4wDQYJKoZIhvcNAQEL -BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMTA0MTIxMTQ1MjBaFw0yMTA1 -MTIxMTQ1MjBaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw -HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB -AQUAA4IBDwAwggEKAoIBAQDK0Ww4voPlkePBPS2MsEi7e1ePS+CDxTdDuOwWWEA7 -JiOyqIGqdyL6AE2EqjL3sSdVFVxytpGQWDuM6JHXdb01AnMngBuql9Jkiln7i267 -v54HtMWdm8o3rik/b/mB+kkn/sP715tI49Ybh/RobtvtK16ZgHr1ombkq6rXiom2 -8GmSmpYFwZtZsXtm2JwbZVayupQpWwdu3KrTXKBtVyKVvvWdgkf47DWYtWDS3vqE -cShM1H97G4DvI+4RX1WtQevQ0yCx1aFTg4xMHFkpUxlP8iW6mQaQPqy9rnI57e3L -RHc2I/B56xa43R3GmQ2S7bE4hvm1SrZDtVgrZLf4nvwNAgMBAAGjUzBRMB0GA1Ud -DgQWBBQ4+o0x1FzK7nRbcnm2pNLwaywCdzAfBgNVHSMEGDAWgBQ4+o0x1FzK7nRb -cnm2pNLwaywCdzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQDE -YmM8MH6RKcaqMqCBefWLj0LTcZ/Wm4G/eCFC51PkAIsf7thnzViemBHRXUSF8wzc -1MBPD6II6OB1F0i7ntGjtlhnL2WcPYbo2Np59p7fo9SMbYwF49OZ40twsuKeeoAp -pfow+y/EBZqa99MY2q6FU6FDA3Rpv0Sdk+/5PHdsSP6cgeMszFBUS0tCQEvEl83n -FJUb0vjEX4x3J64XO/0DKXyCxFyF77OwHG2ZV5BeCpIhGXu+d/e221LJkGI2orKR -kgsaUwrkS8HQt3Hd0gYpLI1Opx/JlRpB0VLYLzRGj7kDpbAcTj3SMEUp/FAZmlXR -Iiebt73eE3rOWVFgyY9f +MIIDtzCCAp+gAwIBAgIUeJXILNkZb1FYvV7YnFYDB1OUrB4wDQYJKoZIhvcNAQEL +BQAwazELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5 +MRAwDgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRwwGgYDVQQDDBNUZXN0 +Y2x1c3RlciBSb290IENBMB4XDTI0MTIwMjE0MjkyNloXDTM0MTEzMDE0MjkyNlow +azELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5MRAw +DgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRwwGgYDVQQDDBNUZXN0Y2x1 +c3RlciBSb290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvhBQ +1W4Swyw4g8VROLzDCYieR+6tyvUwkP/KyH9UapuCnQtZCaNhz6uCCouWONIV8LEx +Mk6YnHJKkBfsWx2C2dKQo3PFyroDa+9J08eFglZCCUcYqYJSzHne07fniIug37w2 +hekFWPbl8dYaNrnRNVUqkAHkFcxJA7JHnEPx+N0V58+2OJrq8bucTVA35oCq6Cjj +wBDJI9/puwtRpwTa3dcZ6bGFKArRKTKO5Nd6gufQKd2MrwXOOGFCltrPDbAUCbKU +UpphEmZIB7rPhCl3qkUgiFM8obgVGgw7E6UD1BKkCS42SFlONAdxnVKNTghN7RK3 +dmAmUYvtT1O7tj4BPQIDAQABo1MwUTAdBgNVHQ4EFgQUxkqr+ix20JSq6T1rWBFa +F/v3OXEwHwYDVR0jBBgwFoAUxkqr+ix20JSq6T1rWBFaF/v3OXEwDwYDVR0TAQH/ +BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAEyO49djQOI9qaHc0tvuAtio+qRqT +zQN5H8FJS4T7RDZSc3sXaUElY8hD2ecGZPDtxmFZy+IMtSGZcMfKlEr3pugrYwRh +571dFp1+o8wEmyOU0NHsGmSxCLZOk9nMxZEhLvc722B6oKHTIm3rvxkQqKpdfliE +oFW2QZVBteZ04A4AKOs0mkZptycZKMLiht5I2s1gzlbK9084huGmnayW9a3pKWyV +zpgnugfxEiwjKh7HpF9Mc2M4Z3f483bj/f6+G8Z7668dORQUFUwv/ohxN8w8zfgA +pfrQNknzYfihuFam6/CFzOsT9Nndtuz14N/LKI9csd2ixWTknHPLyMGrVA== -----END CERTIFICATE----- diff --git a/tests/integration/test_keeper_internal_secure/configs/ssl_conf.yml b/tests/integration/test_keeper_internal_secure/configs/ssl_conf.yml index a444122b09d..0a0a8d87507 100644 --- a/tests/integration/test_keeper_internal_secure/configs/ssl_conf.yml +++ b/tests/integration/test_keeper_internal_secure/configs/ssl_conf.yml @@ -8,3 +8,11 @@ openSSL: cacheSessions: true disableProtocols: 'sslv2,sslv3' preferServerCiphers: true + client: + certificateFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.crt' + caConfig: '/etc/clickhouse-server/config.d/rootCA.pem' + loadDefaultCAFile: true + verificationMode: 'none' + cacheSessions: true + disableProtocols: 'sslv2,sslv3' + preferServerCiphers: true diff --git a/tests/integration/test_keeper_internal_secure/configs/ssl_conf_password.yml b/tests/integration/test_keeper_internal_secure/configs/ssl_conf_password.yml index 51b65c5253a..d55d729b346 100644 --- a/tests/integration/test_keeper_internal_secure/configs/ssl_conf_password.yml +++ b/tests/integration/test_keeper_internal_secure/configs/ssl_conf_password.yml @@ -2,10 +2,18 @@ openSSL: server: certificateFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.crt' privateKeyFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.key' + caConfig: '/etc/clickhouse-server/config.d/rootCA.pem' privateKeyPassphraseHandler: name: KeyFileHandler options: password: 'PASSWORD' + loadDefaultCAFile: true + verificationMode: 'none' + cacheSessions: true + disableProtocols: 'sslv2,sslv3' + preferServerCiphers: true + client: + certificateFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.crt' caConfig: '/etc/clickhouse-server/config.d/rootCA.pem' loadDefaultCAFile: true verificationMode: 'none' diff --git a/tests/integration/test_keeper_internal_secure/test.py b/tests/integration/test_keeper_internal_secure/test.py index 8cab03b6e2d..c08df4db856 100644 --- a/tests/integration/test_keeper_internal_secure/test.py +++ b/tests/integration/test_keeper_internal_secure/test.py @@ -158,6 +158,9 @@ def check_valid_configuration(filename, password): for node in nodes: setupSsl(node, filename, password) start_all_clickhouse() + nodes[0].wait_for_log_line( + "Raft ASIO listener initiated on :::9234, SSL enabled", look_behind_lines=1000 + ) run_test() @@ -166,10 +169,11 @@ def check_invalid_configuration(filename, password): for node in nodes: setupSsl(node, filename, password) - nodes[0].start_clickhouse(expected_to_fail=True) + nodes[0].start_clickhouse() nodes[0].wait_for_log_line( - "OpenSSLException: EVPKey::loadKey.*error:0480006C:PEM routines::no start line", + "Raft ASIO listener initiated on :::9234, SSL enabled", look_behind_lines=1000 ) + nodes[0].wait_for_log_line("failed to connect to peer.*Connection refused") def test_secure_raft_works(started_cluster):