Merge pull request #72831 from ClickHouse/backport/24.9/72730

Backport #72730 to 24.9: Fix advanced SSL configuration for Keeper's internal communication

contrib/NuRaft

@@ -1 +1 @@
-Subproject commit c2b0811f164a7948208489562dab4f186eb305ce
+Subproject commit c11f7fce68737cdc67a1d61678b2717d617ebb5a

src/Coordination/KeeperServer.cpp

@@ -54,11 +54,10 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int RAFT_ERROR;
-    extern const int NO_ELEMENTS_IN_CONFIG;
     extern const int SUPPORT_IS_DISABLED;
     extern const int LOGICAL_ERROR;
     extern const int INVALID_CONFIG_PARAMETER;
-    extern const int BAD_ARGUMENTS;
+    extern const int OPENSSL_ERROR;
 }
 
 using namespace std::chrono_literals;
@@ -68,47 +67,38 @@ namespace
 
 #if USE_SSL
 
-int callSetCertificate(SSL * ssl, void * arg)
+auto getSslContextProvider(const Poco::Util::AbstractConfiguration & config, std::string_view key)
 {
-    if (!arg)
-        return -1;
-
-    const CertificateReloader::Data * data = reinterpret_cast<CertificateReloader::Data *>(arg);
-    return setCertificateCallback(ssl, data, getLogger("SSLContext"));
-}
-
-void setSSLParams(nuraft::asio_service::options & asio_opts)
-{
-    const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config();
-    String certificate_file_property = "openSSL.server.certificateFile";
-    String private_key_file_property = "openSSL.server.privateKeyFile";
-    String root_ca_file_property = "openSSL.server.caConfig";
-
-    if (!config.has(certificate_file_property))
-        throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Server certificate file is not set.");
-
-    if (!config.has(private_key_file_property))
-        throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Server private key file is not set.");
+    String load_default_ca_file_property = fmt::format("openSSL.{}.loadDefaultCAFile", key);
+    String verification_mode_property = fmt::format("openSSL.{}.verificationMode", key);
+    String root_ca_file_property = fmt::format("openSSL.{}.caConfig", key);
+    String private_key_passphrase_property = fmt::format("openSSL.{}.privateKeyPassphraseHandler.options.password", key);
 
     Poco::Net::Context::Params params;
+    String certificate_file_property = fmt::format("openSSL.{}.certificateFile", key);
+    String private_key_file_property = fmt::format("openSSL.{}.privateKeyFile", key);
+    if (config.has(certificate_file_property))
         params.certificateFile = config.getString(certificate_file_property);
-    if (params.certificateFile.empty())
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Server certificate file in config '{}' is empty", certificate_file_property);
 
+    if (config.has(private_key_file_property))
         params.privateKeyFile = config.getString(private_key_file_property);
-    if (params.privateKeyFile.empty())
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Server key file in config '{}' is empty", private_key_file_property);
 
-    auto pass_phrase = config.getString("openSSL.server.privateKeyPassphraseHandler.options.password", "");
-    auto certificate_data = std::make_shared<CertificateReloader::Data>(params.certificateFile, params.privateKeyFile, pass_phrase);
+    std::shared_ptr<CertificateReloader::Data> certificate_data;
+    if (config.has(private_key_passphrase_property))
+    {
+        certificate_data = std::make_shared<CertificateReloader::Data>(
+            params.certificateFile, params.privateKeyFile, config.getString(private_key_passphrase_property));
+        params.certificateFile.clear();
+        params.privateKeyFile.clear();
+    }
 
     if (config.has(root_ca_file_property))
         params.caLocation = config.getString(root_ca_file_property);
 
-    params.loadDefaultCAs = config.getBool("openSSL.server.loadDefaultCAFile", false);
-    params.verificationMode = Poco::Net::Utility::convertVerificationMode(config.getString("openSSL.server.verificationMode", "none"));
+    params.loadDefaultCAs = config.getBool(load_default_ca_file_property, false);
+    params.verificationMode = Poco::Net::Utility::convertVerificationMode(config.getString(verification_mode_property, "none"));
 
-    std::string disabled_protocols_list = config.getString("openSSL.server.disableProtocols", "");
+    std::string disabled_protocols_list = config.getString(fmt::format("openSSL.{}.disableProtocols", key), "");
     Poco::StringTokenizer dp_tok(disabled_protocols_list, ";,", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY);
     int disabled_protocols = 0;
     for (const auto & token : dp_tok)
@@ -125,21 +115,54 @@ void setSSLParams(nuraft::asio_service::options & asio_opts)
             disabled_protocols |= Poco::Net::Context::PROTO_TLSV1_2;
     }
 
-    asio_opts.ssl_context_provider_server_ = [params, certificate_data, disabled_protocols]
+    auto prefer_server_cypher = config.getBool(fmt::format("openSSL.{}.preferServerCiphers", key), false);
+    auto cache_sessions = config.getBool(fmt::format("openSSL.{}.cache_sessions", key), false);
+    return [params, disabled_protocols, prefer_server_cypher, cache_sessions, is_server = key == "server", certificate_data]
     {
-        Poco::Net::Context context(Poco::Net::Context::Usage::TLSV1_2_SERVER_USE, params);
+        Poco::Net::Context context(is_server ? Poco::Net::Context::Usage::SERVER_USE : Poco::Net::Context::Usage::CLIENT_USE, params);
         context.disableProtocols(disabled_protocols);
-        SSL_CTX * ssl_ctx = context.takeSslContext();
-        SSL_CTX_set_cert_cb(ssl_ctx, callSetCertificate, reinterpret_cast<void *>(certificate_data.get()));
-        return ssl_ctx;
-    };
 
-    asio_opts.ssl_context_provider_client_ = [ctx_params = std::move(params)]
+        if (prefer_server_cypher)
+            context.preferServerCiphers();
+
+        if (cache_sessions)
+            context.enableSessionCache();
+
+        auto * ssl_ctx = context.sslContext();
+        if (certificate_data)
         {
-        Poco::Net::Context context(Poco::Net::Context::Usage::TLSV1_2_CLIENT_USE, ctx_params);
+            if (auto err = SSL_CTX_clear_chain_certs(ssl_ctx); err != 1)
+                throw Exception(ErrorCodes::OPENSSL_ERROR, "Clear certificates {}", Poco::Net::Utility::getLastError());
+
+            if (auto err = SSL_CTX_use_certificate(ssl_ctx, const_cast<X509 *>(certificate_data->certs_chain[0].certificate())); err != 1)
+                throw Exception(ErrorCodes::OPENSSL_ERROR, "Use certificate {}", Poco::Net::Utility::getLastError());
+
+            for (auto cert = certificate_data->certs_chain.begin() + 1; cert != certificate_data->certs_chain.end(); cert++)
+            {
+                if (auto err = SSL_CTX_add1_chain_cert(ssl_ctx, const_cast<X509 *>(cert->certificate())); err != 1)
+                    throw Exception(ErrorCodes::OPENSSL_ERROR, "Add certificate to chain {}", Poco::Net::Utility::getLastError());
+            }
+
+            if (auto err = SSL_CTX_use_PrivateKey(ssl_ctx, const_cast<EVP_PKEY *>(static_cast<const EVP_PKEY *>(certificate_data->key))); err != 1)
+                throw Exception(ErrorCodes::OPENSSL_ERROR, "Use private key {}", Poco::Net::Utility::getLastError());
+
+            if (auto err = SSL_CTX_check_private_key(ssl_ctx); err != 1)
+                throw Exception(ErrorCodes::OPENSSL_ERROR, "Unusable key-pair {}", Poco::Net::Utility::getLastError());
+        }
+
+
         return context.takeSslContext();
     };
 }
+
+void setSSLParams(nuraft::asio_service::options & asio_opts)
+{
+    asio_opts.enable_ssl_ = true;
+
+    const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config();
+    asio_opts.ssl_context_provider_server_ = getSslContextProvider(config, "server");
+    asio_opts.ssl_context_provider_client_ = getSslContextProvider(config, "client");
+}
 #endif
 
 std::string checkAndGetSuperdigest(const String & user_and_digest)
@@ -242,7 +265,7 @@ struct KeeperServer::KeeperRaftServer : public nuraft::raft_server
         }
 
         const size_t voting_members = get_num_voting_members();
-        const auto not_responding_peers = get_not_responding_peers();
+        const auto not_responding_peers = get_not_responding_peers_count();
         const auto quorum_size = voting_members / 2 + 1;
         const auto max_not_responding_peers = voting_members - quorum_size;
 
@@ -279,6 +302,11 @@ struct KeeperServer::KeeperRaftServer : public nuraft::raft_server
         return std::unique_lock(lock_);
     }
 
+    std::unique_lock<std::mutex> lockCommit()
+    {
+        return std::unique_lock(commit_lock_);
+    }
+
     bool isCommitInProgress() const
     {
         return sm_commit_exec_in_progress_;
@@ -454,7 +482,6 @@ void KeeperServer::launchRaftServer(const Poco::Util::AbstractConfiguration & co
         throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSL support for NuRaft is disabled because ClickHouse was built without SSL support.");
 #endif
     }
-
     if (is_recovering)
         enterRecoveryMode(params);
 
@@ -1212,6 +1239,7 @@ Keeper4LWInfo KeeperServer::getPartiallyFilled4LWInfo() const
 
 uint64_t KeeperServer::createSnapshot()
 {
+    auto commit_lock = raft_instance->lockCommit();
     uint64_t log_idx = raft_instance->create_snapshot();
     if (log_idx != 0)
         LOG_INFO(log, "Snapshot creation scheduled with last committed log index {}.", log_idx);

tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.crt

@@ -1,20 +1,22 @@
 -----BEGIN CERTIFICATE-----
-MIIDPDCCAiQCFBXNOvsLA+dqmX/TkYG9JXdD5m72MA0GCSqGSIb3DQEBCwUAMFox
-CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
-cm5ldCBXaWRnaXRzIFB0eSBMdGQxEzARBgNVBAMMCmNsaWNraG91c2UwIBcNMjIw
-NDIxMTAzNDU1WhgPMjEyMjAzMjgxMDM0NTVaMFkxCzAJBgNVBAYTAkFVMRMwEQYD
-VQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBM
-dGQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
-AQoCggEBAKaXz596N4NC2zZdIqdwZbSYAtNdBCsBVPt5YT9F640aF5zOogPZyxGP
-ENyOZwABi/7HhwFbH657xyRvi8lTau8dZL+0tbakyoIn1Tw6j+/3GXTjLduJSy6C
-mOf4OzsrFC8mYgU+7p5ijvWVlO9h5NMbLdAPSIB5WSHhmSORH5LgjoK6oMOYdRod
-GmfHqSbwPVwy3Li5SXlniCQmJsM0zl64LFbJ/NU+13qETmhBiDgmh0Svi+wzSzqZ
-q1PIX92T3k44IXNZbvF7lKbUOS9Xb3BoxA4cDqRcTx4x73xRDwodSmqiuQOC99HI
-A0C/tZJ25VNAGjLKukPSHqYscq2PAsUCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEA
-IDQwjf/ja3TfOXrz+Gn1eErSKnWS3asjRT9rYWQsy3tzVUkMIcszrG+FqTR16g5H
-ZWyuEOi6KIRmda3SYKdLKmtQLrgx6/d/jvH5TQ0LTFZrp6vh0lo3pV+L6fLo1ZRD
-V1i8jW/7HHNyqJamUXOjwA0DpPOMkdtwuyV+rJ+2bTG1ZSK33O4Ae2CY5+dad6zy
-YI6b1c9flWfDznuNEMH7jDDjKgXwjZGeU53FiuuhHiNyRchsr/B9eIBsom8oykiD
-kch4cnAxx2E+m3hLYzupkXHOVQ5CNpVk8PGUCIGcyqDxPt+fOj1WbDQ9laEcfhmV
-kR+vHmzOqWZnHU4QeMqDig==
+MIIDtjCCAp6gAwIBAgIUdOfco+b8/fQZQOafHgghkEYL3YkwDQYJKoZIhvcNAQEL
+BQAwazELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5
+MRAwDgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRwwGgYDVQQDDBNUZXN0
+Y2x1c3RlciBSb290IENBMB4XDTI0MTIwMjE0MjkyNloXDTM0MTEzMDE0MjkyNlow
+ajELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5MRAw
+DgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRswGQYDVQQDDBJjbGlja2hv
+dXNlLWtlZXBlcjEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDUXmnn
+Cv7sY9lbrS1Q3c5q7ok9R4XEPq/jWBFkIEnJR7vSjCEnOjLxg+1MdUItjbqODf9N
+5vFbHiiqWQVkGrmg8/CTSme0qyNr7FcmG1hO4bzK/dvIyK1R7YISqZpXoCTVzEnU
+IjU7f+PkX2uAiSypxM4zpNyC7++j6ah8xYNRfR9AS5c7e1dvNKBNMmNipYxVgaEo
+pIke40m12ezIzLOtkL/rGlsnM2Tv/0Wv1xQE+OjHByyQE08vuliatFfweTXLF48m
+4S4NdOq5dh2WX8xLPr8BxRLjXzs08wgKVFpWkIOR2uEInjuVQAGMuZeOqUuQGdar
+GMH4M/3tDl0eJ7mbAgMBAAGjUzBRMB0GA1UdDgQWBBTr9ldBtTB0vatq2yhQgYtt
+zMNhJDAfBgNVHSMEGDAWgBTGSqv6LHbQlKrpPWtYEVoX+/c5cTAPBgNVHRMBAf8E
+BTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBYc03AV8n0D43xm07MxpgqDvNEZC9u
+Q2LnP89UBnBmXD5FwMz4XhA/iupyAeYItZ8R17caIpLHgwOUrh3oHxVW5V144Q0p
+hBBp/im8WQ8NnS3z52CusxE1Zu5AMjoZtxY8FTvgs6vuJZYds/dgtUg5bBawR2LX
+A5FsPLyYpwCjoPTM622uXkuPfRMc5SC5edwHa1RyoG8Poz8B6Y63iKQydOXin9Q1
+9rQ7mqM7D2dCURx4gVoN9y+fLkXgQEzTMBT4wuVJl+CXnxcmKsoROAy7g2mL0RMw
+P3cl+Bod3NrabhjAqG01nHsQzy0uJ/aJHbqoR3OtYo8DdsoKrBRramiG
 -----END CERTIFICATE-----

tests/integration/test_keeper_internal_secure/configs/WithPassPhrase.key

@@ -1,30 +1,28 @@
------BEGIN RSA PRIVATE KEY-----
-Proc-Type: 4,ENCRYPTED
-DEK-Info: AES-256-CBC,4E14FF586022476CD22AAFB662BB0E40
-
-dpJZKq5k+fMuC7XECfTSRjPeOEl9wNuVtZkcjEWaHN8ky4umND7ARyRyuU1Nk7cy
-fpCFlFKOqDfCkT5zVK/fB6pF32wqAI7sqeSuYPfQY0+L77yRdwM6L46WslzVKZYE
-lXD1AmqKT/LgF3+eBY5slkAAJo10zYDgKEwnoQVBp31YW2/+6oAGaY/O6x3p7aTG
-dw9CP+SFc0o8lPl1lsSovdNXDUiVCftvClog7hwyDv8AhHyGgynw3UJXX8UlyWu+
-Zz5zpgrvB2gvDLeoZZ6qjMGvtpEwlYBh4de9ZOsvQUpXEEfkQFtJV0j613OCQune
-LoTxKpYV1V/mZX4HPaJ1oC0OJ7XcliAOSS9K49YobTuz7Kg5Wg3bVGo9xRfYDjch
-rVeMP4u5RXSGuHL23FPMfK6EqcldrFyRwLaY/IV1Yl6UNUMKAphn/WMtWVuT3TiT
-QMCI2VRt7ItwZwwFn5RgyDweWdFf5v3AmN/lOhATDBqosahkPxDZ1VZ6OBPoJLPM
-UrDWH/lqrByeEjtAOwr5UsWKwLuJ8qUxQ4TchHwFKOwy6VsrRwMQ3ZWi2govPF9I
-W0sfLj5Ulfjx6zHdqnF48a1Elit4JH6inBEWFuj7bmlOotq+PHoeT61zAwW+gnrG
-3JTo3XnaE2WwRDpqvKYHWLv/J218rq8PtIaq9gjr55odPfIt8lkJ1XzF4WQ21rIJ
-GNWZ3xz4fxpvrKnQyAKGu0ZcdjA1nqs16oiVr+UnJoXmkM5yBCic4fZYwPTSQHYS
-ZxwaTzEjfeGxrSeLrN9CgoweucvogOvUjJOBcW/py80du8vWz0YyzMhg3o0YeGME
-C+Kts/YWxmyfw4DaWt8RtWCKl85hEmz8RODvkMLGtLzvVoSyLQWqp1NhGIlFtzXs
-7sPLulUeyD2avTC/RB/Pu9Nk80c0368BxCoeYbiFWZpaN70SJmCUE5H59J2d0olw
-5v2RVjLBi8wqnzoa0+2L8wnG7IQGadS97dj0eBR+JXNtoJhVrurS80RJ6B0bNxdu
-gX8otfnJYsZyK5hbEhcQqLdnyGhDEE8YHe7Hv9stWwLAFOfOMzyzC06lFS1eNiw4
-FJyXJUhDieb8EqetouAC8dNVXz4Q1zOTlGuAbGoKm5v0U5IhCQap9GUSW5QiUgOQ
-AEMs9aGfd91R+IcDf19mZptsQLYA6MGBN6fm+3O2iZImKIbF+ZZo0S6liFFmn6lm
-M+diTzaoiqgEkiXOuRhdQUMaiGV8BMZxv8qUH6/vyC3gSueoTio0f9PfASDYfvXD
-A3GuI87P6LF1it2UlN6ssFoXTZdfQQZwRmNuqOqw+BJOJHrR6trcXOCZOQ77Qnvd
-M5a348gIzluVUkExAPGCsySQWMx4Of5NBF28jEC3+TAwkRqBV2ZHmfGLWnvwaB+A
-YUeKtpWblfG1lsrDAdwL2dilU95oby+35sExX7M2dCrL9Y2P5oTCW3u12//ZSLeL
-Yhi1Rzol6LAuesZCVF0Zv/YYDhzAckJfT/qXK5B5pz9saswxCUBEpiKlLpVsjOFJ
-2bHm8NgOMD5b3cdh1kvts4wZe+giry7LHsn46f+9VqN+gA6XxeVsPyb4uO1KW3SN
------END RSA PRIVATE KEY-----
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDUXmnnCv7sY9lb
+rS1Q3c5q7ok9R4XEPq/jWBFkIEnJR7vSjCEnOjLxg+1MdUItjbqODf9N5vFbHiiq
+WQVkGrmg8/CTSme0qyNr7FcmG1hO4bzK/dvIyK1R7YISqZpXoCTVzEnUIjU7f+Pk
+X2uAiSypxM4zpNyC7++j6ah8xYNRfR9AS5c7e1dvNKBNMmNipYxVgaEopIke40m1
+2ezIzLOtkL/rGlsnM2Tv/0Wv1xQE+OjHByyQE08vuliatFfweTXLF48m4S4NdOq5
+dh2WX8xLPr8BxRLjXzs08wgKVFpWkIOR2uEInjuVQAGMuZeOqUuQGdarGMH4M/3t
+Dl0eJ7mbAgMBAAECggEAJwzxZlXESI2Xw17VzV/r/Ae+3rDPLSXly+U+1W2Gg+eX
+5wBzfDYcdgKvWPba42uDWWnDf3yu9vVVvvU9o4myhqE0pLDy3ur1SXwdDlnK5D5o
+K9+AUaxtCnqlB29+fQxqmZHGJabgqP88VZsiNnGC7/jLff2buswKAdcOb1sWaZ5F
+p5YBL6TmUzpg3Pdbs2N/OVsZGh5Y+d6m6hJgjsbdcWC4LrFurj9UyQoCGIRruelN
+Ra3ft4QAC3biD0/hEc4WyjzZdlMvvEqaeQUmqK2TJLELwyH33W1Ek2FsLm6LTP9i
+7kW2+9684GWBXo5ge87BXASVwAXnFE+pnXV5QGLMqQKBgQDpo2ZM/R/lWLR8Hpww
+WKZZFfWPpsv0d6DbwVWowdRoojPue62nIby6+LBk08j9UwqkC0TK8pWCjL4hhSfU
+JjLuGuzl+RFfstpt42qh2zgi8aedLtGHjFyjHp1jE3rb9l95YXUpNm7syEMbLR8V
+NR09na41ftFCBPBNYFsdxrzKPwKBgQDosd6qwhrTL+ndtiJIeonRvunjo4yXn22/
+qWqRy3WJmZpDKWpsGWOmlJ4G7+10Q3zOMpb+nUOEjJNI3EdnLrVTeJo8WNuNXHyl
+axgWV3TR8JT6GIG+zavGEI51JjTH3X1eGzm9T4Di8mj34FzyK1af6atiiRj5sIuk
+NG71CUjhpQKBgQCDJ90n3vjm0LMQ8kYPxdQsMm2VZLcd14IPmyqw/45z5opsmDVV
+m1TNSQoMr+8mdlWE3WaS3zcbAFNDkfJX39G7ZJYUS4t7Q3XnNkEH934975Z+YGfz
+RdJDJ86GbcsMa/QQuasBpbMDbTBusxe92gE+M6Q2F6j0/LzBUxQTVRtqFQKBgBFZ
+IXatnf5cthzXdVrd9+RxTVKxYMv1EOOXJ+DSwGKP1xZmwg5pHirPLbDqtlNSrL1a
+vDMjWmNJb7mg4pnou5ALj8QsA8JYQNq8T0FrJ8R3IUQ8C4BEKShNF7HYNVspQi1/
+7iAVC1DgLb89NPDBFmY5r5NbEUecR+zoE9Wk6ZAZAoGBAIHTifNzf7/qDtHI2+Cc
+YGiudMlWWwNqTUr1BjPQx1au1VImpDAB6eaz3DV0oIS0fpREte6SRrcwvtrRqp4M
+AT4uCjiVOaXW/MwybfS6BIivTvuTkyPuNCBIWTH1JBQ3CEcEIIV5YcpFDQs5FQ0M
+GPtHFxbKMUmLmJVW5nbKUUhr
+-----END PRIVATE KEY-----

tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.crt

@@ -1,19 +1,22 @@
 -----BEGIN CERTIFICATE-----
-MIIDETCCAfkCFHL+gKBQnU0P73/nrFrGaVPauTPmMA0GCSqGSIb3DQEBCwUAMEUx
-CzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
-cm5ldCBXaWRnaXRzIFB0eSBMdGQwHhcNMjEwNDEyMTE0NzI5WhcNMjEwNTEyMTE0
-NzI5WjBFMQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UE
-CgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC
-AQ8AMIIBCgKCAQEA1iPeYn1Vy4QnQi6uNVqQnFLr0u3qdrMjGEBNAOuGmtIdhIn8
-rMCzaehNr3y2YTMRbZAqmv28P/wOXpzR1uQaFlQzTOjmsn/HOZ9JX2hv5sBUv7SU
-UiPJS7UtptKDPbLv3N/v1dOXbY+vVyzo8U1Q9OS1J5yhYW6KtxP++hfSrOsFu669
-d1pqWFWaNBsmf0zF+ETvi6lywhyTFA1/PazcStP5GntcDL7eDvGq+DDsRC40oRpy
-S4xRQRSteCTtGGmWpx+Jmt+90wFnLgruUbWT0veCoLxLvz0tJUk3ueUVnMkrxBQG
-Fz+IWm+SQppNU5LlAcBcu9wJfo3h34BXp0NFNQIDAQABMA0GCSqGSIb3DQEBCwUA
-A4IBAQCUnvQsv+GsPwGnIWqH9iiFVhgDx5QbSTW94Fyqk8dcIJBzWAiCshmLBWPJ
-pfy4y2nxJbzovFsd9DA49pxqqILeLjue99yma2DVKeo+XDLDN3OX5faIMTBd7AnL
-0MKqW7gUSLRUZrNOvFciAY8xRezgBQQBo4mcmmMbAbk5wKndGY6ZZOcY+JwXlqGB
-5hyi6ishO8ciiZi3GMFNWWk9ViSfo27IqjKdSkQq1pr3FULvepd6SkdX+NvfZTAH
-rG+CSoFGiJcOBbhDkvpY32cAJEnJOA1vHpFxfnGP8/1haeVZHqSwH1cySD78HVtF
-fBs000wGHzBYWNI2KkwjNtYf06P4
+MIIDtjCCAp6gAwIBAgIUP0g0uMpZSD2OOtjFXz/anI4EU+swDQYJKoZIhvcNAQEL
+BQAwazELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5
+MRAwDgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRwwGgYDVQQDDBNUZXN0
+Y2x1c3RlciBSb290IENBMB4XDTI0MTIwMjE0MjkyNloXDTM0MTEzMDE0MjkyNlow
+ajELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5MRAw
+DgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRswGQYDVQQDDBJjbGlja2hv
+dXNlLWtlZXBlcjEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCgflyz
+Kg1deXEXFJIzoyLIAfRPs8MpOsKt06DPVvyZp2ct+g2GCcZlwV4L/GunIV2sugeX
+ZHcJ+B06gKSgouxOMFjTnBEdlygLegMeyrJI6TKREiiWMYYxfUVabpC0DtKeZxc/
+D9BY4qLjngxbdRwS7l4eKv74jV9dowDfCNZxXLtzP3uj+AFlLuWk0LP6qFmJMUii
+tM7f3oLzxURxIddBASjz12dyQGdm/6v6UcVWnqSDXApozb9LPmapUiJM9axcEvjM
+C/Qr14021OEgLVGGEeAAA4JHWZPCqQjbgaDHm5xa61KAMnwDxk/GbMX/TFSwgV4x
+pDChT9GKzVMHNf7PAgMBAAGjUzBRMB0GA1UdDgQWBBRZ8QY0I9WoGyFlwyGOs4ZY
+cf5tEDAfBgNVHSMEGDAWgBTGSqv6LHbQlKrpPWtYEVoX+/c5cTAPBgNVHRMBAf8E
+BTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQB6hUw6IrDGBvGN3AIVatO/6xZX5LZM
+Lp5B4uL5rz+6BXf+hZFsj3o4uvyxaEW12m+/bPPOA4EBdShtUfydfMoDHJsnrE8k
+D6aVq04f7vjffGeFzvQhfAEnK5/rutWDyq9rXlqcKcPFLhl2Pozk7ty3V+Wz7i3+
+0n2uDTxAfcdlkeSlzPpXP/JOMFN6BwmzrgsyLHyPeIhjfv/lFMoAOblpF6tDFvlY
+sXTk0P3Eh9zQ9vT2HI3ZVkqXUe3qQZhUOkKezy0J/OK/6wlvRoO3GXr8/gJO+lJp
+ATwurpIc/za08toWbziOOL4xhY4RA+7S9uK3Uz+2a8AoRyUurMP4AHpx
 -----END CERTIFICATE-----

tests/integration/test_keeper_internal_secure/configs/WithoutPassPhrase.key

@@ -1,27 +1,28 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEowIBAAKCAQEA1iPeYn1Vy4QnQi6uNVqQnFLr0u3qdrMjGEBNAOuGmtIdhIn8
-rMCzaehNr3y2YTMRbZAqmv28P/wOXpzR1uQaFlQzTOjmsn/HOZ9JX2hv5sBUv7SU
-UiPJS7UtptKDPbLv3N/v1dOXbY+vVyzo8U1Q9OS1J5yhYW6KtxP++hfSrOsFu669
-d1pqWFWaNBsmf0zF+ETvi6lywhyTFA1/PazcStP5GntcDL7eDvGq+DDsRC40oRpy
-S4xRQRSteCTtGGmWpx+Jmt+90wFnLgruUbWT0veCoLxLvz0tJUk3ueUVnMkrxBQG
-Fz+IWm+SQppNU5LlAcBcu9wJfo3h34BXp0NFNQIDAQABAoIBAHYDso2o8V2F6XTp
-8QxqawQcFudaQztDonW9CjMVmks8vRPMUDqMwNP/OMEcBA8xa8tsBm8Ao3zH1suB
-tYuujkn8AYHDYVDCZvN0u6UfE3yiRpKYXJ2gJ1HX+d7UaYvZT6P0rmKzh+LTqxhq
-Ib7Kk3FDkirQgYgGueAH3x/JfUvaAGvFrq+HvvlhHOs7M7iFU4nJA8jNfBolpTnG
-v5MMI+f8/GHGreVICJUoclE+4V/4LDHUlrc3l1kQk0keeD6ECw/pl48TNL6ncXKu
-baez1rfKbMPjhLUy2q5UZa93oW+olchEOXs1nUNKUhIOOr0f0YweYhUHNTineVM9
-yTecMIkCgYEA7CFQMyeLVeBA6C9AHBe8Zf/k64cMPyr0lUz6548ulil580PNPbvW
-kd2vIKfUMgCO5lMA47ArL4bXZ7cjTvJmPYE1Yv8z+F0Tk03fnTrudHOSBEiGXAu3
-MPTxCDU7Se5Dwj0Fq81aFRtCHl8Rrss+WiBD8eRoxb/vwXKFc6VUAWMCgYEA6CjZ
-XrZz11lySBhjkyVXcdLj89hDZ+bPxA7b3VB7TfCxsn5xVck7U3TFkg5Z9XwEQ7Ob
-XFAPuwT9GKm7QPp6L8T2RltoJ3ys40UH1RtcNLz2aIo/xSP7lopPdAfWHef5r4y9
-kRw+Gh4NP/l5wefXsRz/D0jY3+t+QnwnhuCKbocCgYEAiR6bPOlkvzyXVH1DxEyA
-Sdb8b00f7nqaRyzJsrfxvJ9fQsWHpKa0ZkYOUW9ECLlMQjHHHXEK0vGBmqe9qDWY
-63RhtRgvbLVYDb018k7rc9I846Hd7AudmJ9UbIjE4hyrWlsnNOntur32ej6IvTEn
-Bx0fd5NEyDi6GGLRXiOOkbMCgYAressLE/yqDlR68CZl/o5cAPU0TAKDyRSMUYQX
-9OTC+hstpMSxHlkADlSaQBnVAf8CdvbX2R65FfwYzGEHkGGl5KuDDcd57b2rathG
-rzMbpXA4r/u1fkG2Nf0fbABL5ZA7so4mSTXQSmSM4LpO+I7K2vVh9XC4rzAcX4g/
-mHoUrQKBgBf3rxp5h9P3HWoZYjzBDo2FqXUjKLLjE9ed5e/VqecqfHIkmueuNHlN
-xifHr7lpsYu6IXkTnlK14pvLoPuwP59dCIOUYwAFz8RlH4MSUGNhYeGA8cqRrhmJ
-tYk3OKExuN/+O12kUPVTy6BMH1hBdRJP+7y7lapWsRhZt18y+8Za
------END RSA PRIVATE KEY-----
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCgflyzKg1deXEX
+FJIzoyLIAfRPs8MpOsKt06DPVvyZp2ct+g2GCcZlwV4L/GunIV2sugeXZHcJ+B06
+gKSgouxOMFjTnBEdlygLegMeyrJI6TKREiiWMYYxfUVabpC0DtKeZxc/D9BY4qLj
+ngxbdRwS7l4eKv74jV9dowDfCNZxXLtzP3uj+AFlLuWk0LP6qFmJMUiitM7f3oLz
+xURxIddBASjz12dyQGdm/6v6UcVWnqSDXApozb9LPmapUiJM9axcEvjMC/Qr1402
+1OEgLVGGEeAAA4JHWZPCqQjbgaDHm5xa61KAMnwDxk/GbMX/TFSwgV4xpDChT9GK
+zVMHNf7PAgMBAAECgf9+Cl08oHSJPWifSeoBfvgCfJKel5pj60fT5kcO10Ghy6Vt
+IMKisi8dKET8wz/IGcFe8RtmpR4UVK4NXB6YguDhALWEtwwntwfURKiokiWJ+HVD
+8s4Fbyht/m1UTqUk23MG2xPgcorkBlWc/pqdaXOWjwpnUXVNXAJketvrKBHTbgq+
+XBvWlYNm+8ThGxm6Ryf3v6K04m5bVAzdQljGBUXRixqEZ71qkQ1TwEBaphbEWPLZ
+7nod0yKxQ38ydYrrqqjoINvNLr4OBrlsjuYFAXbwwJmY0L5EleleVDjuiKmbCLUX
+CKFaT+YlvHKY5m313ohVFldjRqThjuNt2VtuOFECgYEA0cyCH+sBTaMrYCgi8+l2
+nB+w9zVUqI70naT9zpBlei3DWgklMlNoGRbiIrvFyS2eH3GMq+WZub2/Ci0AU0RG
+QYcvp4dfJth6IoKif8Un+RSMW1rN4pPDA88YIr+BKlnkjwfj+71ldBuX75UHXut1
+8z8ThxrmpMCOFql6S5odkSsCgYEAw9ZANtjTBpo2Dff8uPE+Ml3rilRWk3RjX+iU
+UbKrsNWwU0QXQu4RbxHaCmJEPFP7bL6W/DkWSvzrxxnnmtQ2UXvXu1jf5D8asliz
++HMIbX3beiKec+C0xbt5xbOYkO4pfurS8V9fYsJTFyDDiMk9cDlHwKfH71yO2Qgp
+Zw4Szu0CgYEAtb8awxfMyzsdanGaxf5r+BgkMCQNMPCWzLKQBRBmOI/IegkOJijH
+N2TxhfFxCDTylH7DxG5k29ma0+/kJj4xNrcr+090iKxkMd1FdLaRSGAar1Fcpnon
+KPeRCxknhk7Vh2rof761Uv5MgwpxljMYvR7ZheMyB2ugK9Wp0jCyiH0CgYAus9B6
+g/jHUU1kxWgKftWTU1yRj41Z+t6cB64fUZmqQTucj9dwSa/0qfAym76kGG8UPtto
+6QBM/8YGpEHcZZFSm1MWRZqXJwlp0MeSj3RKEEKf/NOG1OanZQ8kO7E9lt5kewG1
+OEZaGfeQw2p+G2fAdJiM9DY0+gDC9zRQdEW9/QKBgQCIlZ6ToPdBJPuhKTtV0+c/
+e3k0+bKy3OIoW+laaRS3JMMyvveQECaLKMsVMYCppJfKCV/NOUnTIw/g1vXuom6I
+QZHGK10aWB8bunblKFrSFxfBStgDCPeZklb4ECQP4+QfLb7Xi7fAvfSi0hFP4LNo
+Ea+Ttp/shik8d0sy217IEQ==
+-----END PRIVATE KEY-----

tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper1.xml

@@ -11,6 +11,7 @@
             <session_timeout_ms>10000</session_timeout_ms>
             <snapshot_distance>75</snapshot_distance>
             <raft_logs_level>trace</raft_logs_level>
+            <startup_timeout>1000</startup_timeout>
         </coordination_settings>
 
         <raft_configuration>

tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper2.xml

@@ -11,6 +11,7 @@
             <session_timeout_ms>10000</session_timeout_ms>
             <snapshot_distance>75</snapshot_distance>
             <raft_logs_level>trace</raft_logs_level>
+            <startup_timeout>1000</startup_timeout>
         </coordination_settings>
 
         <raft_configuration>

tests/integration/test_keeper_internal_secure/configs/enable_secure_keeper3.xml

@@ -11,6 +11,7 @@
             <session_timeout_ms>10000</session_timeout_ms>
             <snapshot_distance>75</snapshot_distance>
             <raft_logs_level>trace</raft_logs_level>
+            <startup_timeout>1000</startup_timeout>
         </coordination_settings>
 
         <raft_configuration>

tests/integration/test_keeper_internal_secure/configs/rootCA.pem

@@ -1,21 +1,22 @@
 -----BEGIN CERTIFICATE-----
-MIIDazCCAlOgAwIBAgIUUiyhAav08YhTLfUIXLN/0Ln09n4wDQYJKoZIhvcNAQEL
-BQAwRTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
-GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDAeFw0yMTA0MTIxMTQ1MjBaFw0yMTA1
-MTIxMTQ1MjBaMEUxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEw
-HwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQwggEiMA0GCSqGSIb3DQEB
-AQUAA4IBDwAwggEKAoIBAQDK0Ww4voPlkePBPS2MsEi7e1ePS+CDxTdDuOwWWEA7
-JiOyqIGqdyL6AE2EqjL3sSdVFVxytpGQWDuM6JHXdb01AnMngBuql9Jkiln7i267
-v54HtMWdm8o3rik/b/mB+kkn/sP715tI49Ybh/RobtvtK16ZgHr1ombkq6rXiom2
-8GmSmpYFwZtZsXtm2JwbZVayupQpWwdu3KrTXKBtVyKVvvWdgkf47DWYtWDS3vqE
-cShM1H97G4DvI+4RX1WtQevQ0yCx1aFTg4xMHFkpUxlP8iW6mQaQPqy9rnI57e3L
-RHc2I/B56xa43R3GmQ2S7bE4hvm1SrZDtVgrZLf4nvwNAgMBAAGjUzBRMB0GA1Ud
-DgQWBBQ4+o0x1FzK7nRbcnm2pNLwaywCdzAfBgNVHSMEGDAWgBQ4+o0x1FzK7nRb
-cnm2pNLwaywCdzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQDE
-YmM8MH6RKcaqMqCBefWLj0LTcZ/Wm4G/eCFC51PkAIsf7thnzViemBHRXUSF8wzc
-1MBPD6II6OB1F0i7ntGjtlhnL2WcPYbo2Np59p7fo9SMbYwF49OZ40twsuKeeoAp
-pfow+y/EBZqa99MY2q6FU6FDA3Rpv0Sdk+/5PHdsSP6cgeMszFBUS0tCQEvEl83n
-FJUb0vjEX4x3J64XO/0DKXyCxFyF77OwHG2ZV5BeCpIhGXu+d/e221LJkGI2orKR
-kgsaUwrkS8HQt3Hd0gYpLI1Opx/JlRpB0VLYLzRGj7kDpbAcTj3SMEUp/FAZmlXR
-Iiebt73eE3rOWVFgyY9f
+MIIDtzCCAp+gAwIBAgIUeJXILNkZb1FYvV7YnFYDB1OUrB4wDQYJKoZIhvcNAQEL
+BQAwazELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5
+MRAwDgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRwwGgYDVQQDDBNUZXN0
+Y2x1c3RlciBSb290IENBMB4XDTI0MTIwMjE0MjkyNloXDTM0MTEzMDE0MjkyNlow
+azELMAkGA1UEBhMCREUxDjAMBgNVBAgMBVN0YXRlMQ0wCwYDVQQHDARDaXR5MRAw
+DgYDVQQKDAdDb21wYW55MQ0wCwYDVQQLDARVbml0MRwwGgYDVQQDDBNUZXN0Y2x1
+c3RlciBSb290IENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvhBQ
+1W4Swyw4g8VROLzDCYieR+6tyvUwkP/KyH9UapuCnQtZCaNhz6uCCouWONIV8LEx
+Mk6YnHJKkBfsWx2C2dKQo3PFyroDa+9J08eFglZCCUcYqYJSzHne07fniIug37w2
+hekFWPbl8dYaNrnRNVUqkAHkFcxJA7JHnEPx+N0V58+2OJrq8bucTVA35oCq6Cjj
+wBDJI9/puwtRpwTa3dcZ6bGFKArRKTKO5Nd6gufQKd2MrwXOOGFCltrPDbAUCbKU
+UpphEmZIB7rPhCl3qkUgiFM8obgVGgw7E6UD1BKkCS42SFlONAdxnVKNTghN7RK3
+dmAmUYvtT1O7tj4BPQIDAQABo1MwUTAdBgNVHQ4EFgQUxkqr+ix20JSq6T1rWBFa
+F/v3OXEwHwYDVR0jBBgwFoAUxkqr+ix20JSq6T1rWBFaF/v3OXEwDwYDVR0TAQH/
+BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAEyO49djQOI9qaHc0tvuAtio+qRqT
+zQN5H8FJS4T7RDZSc3sXaUElY8hD2ecGZPDtxmFZy+IMtSGZcMfKlEr3pugrYwRh
+571dFp1+o8wEmyOU0NHsGmSxCLZOk9nMxZEhLvc722B6oKHTIm3rvxkQqKpdfliE
+oFW2QZVBteZ04A4AKOs0mkZptycZKMLiht5I2s1gzlbK9084huGmnayW9a3pKWyV
+zpgnugfxEiwjKh7HpF9Mc2M4Z3f483bj/f6+G8Z7668dORQUFUwv/ohxN8w8zfgA
+pfrQNknzYfihuFam6/CFzOsT9Nndtuz14N/LKI9csd2ixWTknHPLyMGrVA==
 -----END CERTIFICATE-----

tests/integration/test_keeper_internal_secure/configs/ssl_conf.yml

@@ -8,3 +8,11 @@ openSSL:
     cacheSessions: true
     disableProtocols: 'sslv2,sslv3'
     preferServerCiphers: true
+  client:
+    certificateFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.crt'
+    caConfig: '/etc/clickhouse-server/config.d/rootCA.pem'
+    loadDefaultCAFile: true
+    verificationMode: 'none'
+    cacheSessions: true
+    disableProtocols: 'sslv2,sslv3'
+    preferServerCiphers: true

tests/integration/test_keeper_internal_secure/configs/ssl_conf_password.yml

@@ -2,10 +2,18 @@ openSSL:
   server:
     certificateFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.crt'
     privateKeyFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.key'
+    caConfig: '/etc/clickhouse-server/config.d/rootCA.pem'
     privateKeyPassphraseHandler:
         name: KeyFileHandler
         options:
             password: 'PASSWORD'
+    loadDefaultCAFile: true
+    verificationMode: 'none'
+    cacheSessions: true
+    disableProtocols: 'sslv2,sslv3'
+    preferServerCiphers: true
+  client:
+    certificateFile: '/etc/clickhouse-server/config.d/WithoutPassPhrase.crt'
     caConfig: '/etc/clickhouse-server/config.d/rootCA.pem'
     loadDefaultCAFile: true
     verificationMode: 'none'

tests/integration/test_keeper_internal_secure/test.py

@@ -158,6 +158,9 @@ def check_valid_configuration(filename, password):
     for node in nodes:
         setupSsl(node, filename, password)
     start_all_clickhouse()
+    nodes[0].wait_for_log_line(
+        "Raft ASIO listener initiated on :::9234, SSL enabled", look_behind_lines=1000
+    )
     run_test()
 
 
@@ -166,10 +169,11 @@ def check_invalid_configuration(filename, password):
     for node in nodes:
         setupSsl(node, filename, password)
 
-    nodes[0].start_clickhouse(expected_to_fail=True)
+    nodes[0].start_clickhouse()
     nodes[0].wait_for_log_line(
-        "OpenSSLException: EVPKey::loadKey.*error:0480006C:PEM routines::no start line",
+        "Raft ASIO listener initiated on :::9234, SSL enabled", look_behind_lines=1000
     )
+    nodes[0].wait_for_log_line("failed to connect to peer.*Connection refused")
 
 
 def test_secure_raft_works(started_cluster):