ClickHouse/tests/integration/test_reload_client_certificate/test.py

import os

import threading

import time

import pytest

from helpers.cluster import ClickHouseCluster


TEST_DIR = os.path.dirname(__file__)

cluster = ClickHouseCluster(
    __file__,
    zookeeper_certfile=os.path.join(TEST_DIR, "configs_secure", "first_client.crt"),
    zookeeper_keyfile=os.path.join(TEST_DIR, "configs_secure", "first_client.key"),
)

node1 = cluster.add_instance(
    "node1",
    main_configs=[
        "configs_secure/first_client.crt",
        "configs_secure/first_client.key",
        "configs_secure/second_client.crt",
        "configs_secure/second_client.key",
        "configs_secure/third_client.crt",
        "configs_secure/third_client.key",
        "configs_secure/conf.d/remote_servers.xml",
        "configs_secure/conf.d/ssl_conf.xml",
        "configs/zookeeper_config_with_ssl.xml",
    ],
    with_zookeeper_secure=True,
)
node2 = cluster.add_instance(
    "node2",
    main_configs=[
        "configs_secure/first_client.crt",
        "configs_secure/first_client.key",
        "configs_secure/second_client.crt",
        "configs_secure/second_client.key",
        "configs_secure/third_client.crt",
        "configs_secure/third_client.key",
        "configs_secure/conf.d/remote_servers.xml",
        "configs_secure/conf.d/ssl_conf.xml",
        "configs/zookeeper_config_with_ssl.xml",
    ],
    with_zookeeper_secure=True,
)

nodes = [node1, node2]

@pytest.fixture(scope="module", autouse=True)
def started_cluster():
    try:
        cluster.start()
        yield cluster

    finally:
        cluster.shutdown()


def secure_connection_test(started_cluster):
    # no asserts, connection works
    node1.query("SELECT count() FROM system.zookeeper WHERE path = '/'")
    node2.query("SELECT count() FROM system.zookeeper WHERE path = '/'")

    threads_number = 4
    iterations = 10
    threads = []

    # just checking for race conditions
    for _ in range(threads_number):
        threads.append(
            threading.Thread(
                target=(
                    lambda: [
                        node1.query(
                            "SELECT count() FROM system.zookeeper WHERE path = '/'"
                        )
                        for _ in range(iterations)
                    ]
                )
            )
        )

    for thread in threads:
        thread.start()

    for thread in threads:
        thread.join()


def change_config_to_key(name):
    """
    * Generate config with certificate/key name from args.
    * Reload config.
    """

    for node in nodes:
        node.exec_in_container(
            [
                "bash",
                "-c",
                """cat > /etc/clickhouse-server/config.d/ssl_conf.xml << EOF
<clickhouse>
    <openSSL>
        <client>
            <certificateFile>/etc/clickhouse-server/config.d/{cur_name}_client.crt</certificateFile>
            <privateKeyFile>/etc/clickhouse-server/config.d/{cur_name}_client.key</privateKeyFile>
            <loadDefaultCAFile>true</loadDefaultCAFile>
            <cacheSessions>true</cacheSessions>
            <disableProtocols>sslv2,sslv3</disableProtocols>
            <preferServerCiphers>true</preferServerCiphers>
            <verificationMode>none</verificationMode>
            <invalidCertificateHandler>
                <name>RejectCertificateHandler</name>
            </invalidCertificateHandler>
        </client>
    </openSSL>
</clickhouse>
EOF""".format(
                    cur_name=name
                ),
            ]
        )

        node.exec_in_container(
            ["bash", "-c", f"touch /etc/clickhouse-server/config.d/ssl_conf.xml"],
        )


def check_reload_successful(
    node, cert_name):
    return node.grep_in_log(f"Reloaded certificate (/etc/clickhouse-server/config.d/{cert_name}_client.crt)")

def check_error_handshake(node):
    return node.count_in_log("Code: 210.")

def clean_logs():
    for node in nodes:
        node.exec_in_container(
            [
                "bash",
                "-c",
                "echo -n > /var/log/clickhouse-server/clickhouse-server.log",
            ]
        )

def check_certificate_switch(
    first, second
):
    # Set first key
    change_config_to_key(first)

    # Restart zookeeper to reload the session
    cluster.stop_zookeeper_nodes(["zoo1", "zoo2", "zoo3"])
    cluster.start_zookeeper_nodes(["zoo1", "zoo2", "zoo3"])
    cluster.wait_zookeeper_nodes_to_start(["zoo1", "zoo2", "zoo3"])
    clean_logs()

    # Change to wrong certificate
    change_config_to_key(second)

    # Time to log
    time.sleep(10)

    # Check information about client certificates reloading in log
    reload_successful = any(check_reload_successful(node, second) for node in nodes)

    # Restart zookeeper to reload the session and clean logs for new check
    cluster.stop_zookeeper_nodes(["zoo1", "zoo2", "zoo3"])
    cluster.start_zookeeper_nodes(["zoo1", "zoo2", "zoo3"])
    clean_logs()
    cluster.wait_zookeeper_nodes_to_start(["zoo1", "zoo2", "zoo3"])

    if second == "second":
        time.sleep(10)
        error_handshake = any(check_error_handshake(node) != "0\n" for node in nodes)
    else:
        check_connection = secure_connection_test(started_cluster)
        error_handshake = any(check_error_handshake(node) == "0\n" for node in nodes)
    assert reload_successful and error_handshake


def test_wrong_cn_cert():
    """Checking the certificate reload with an incorrect CN, the expected behavior is Code: 210."""
    check_certificate_switch("first", "second")


def test_correct_cn_cert():
    """Replacement with a valid certificate, the expected behavior is to restore the connection with Zookeeper."""
    check_certificate_switch("second", "third")