Add test for communication between ClickHouse and Zookeeper over SSL

tests/integration/helpers/cluster.py

@@ -144,7 +144,8 @@ class ClickHouseCluster:
                      with_odbc_drivers=False, with_postgres=False, with_hdfs=False, with_mongo=False,
                      with_redis=False, with_minio=False,
                      hostname=None, env_variables=None, image="yandex/clickhouse-integration-test",
-                     stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None):
+                     stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None,
+                     zookeeper_docker_compose_path=None):
         """Add an instance to the cluster.
 
         name - the name of the instance directory and the value of the 'instance' macro in ClickHouse.
@@ -179,10 +180,13 @@ class ClickHouseCluster:
 
         cmds = []
         if with_zookeeper and not self.with_zookeeper:
+            if not zookeeper_docker_compose_path:
+                zookeeper_docker_compose_path = p.join(HELPERS_DIR, 'docker_compose_zookeeper.yml')
+
             self.with_zookeeper = True
-            self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_zookeeper.yml')])
+            self.base_cmd.extend(['--file', zookeeper_docker_compose_path])
             self.base_zookeeper_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name',
-                                       self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_zookeeper.yml')]
+                                       self.project_name, '--file', zookeeper_docker_compose_path]
             cmds.append(self.base_zookeeper_cmd)
 
         if with_mysql and not self.with_mysql:

tests/integration/helpers/zookeeper-ssl-entrypoint.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+set -e
+
+export ZOO_SERVER_CNXN_FACTORY=org.apache.zookeeper.server.NettyServerCnxnFactory
+export ZOO_SSL_KEYSTORE_LOCATION=/conf/certs/zookeeper.p12
+export ZOO_SSL_KEYSTORE_PASSWORD=password
+export ZOO_SSL_TRUSTSTORE_LOCATION=/conf/certs/truststore.p12
+export ZOO_SSL_TRUSTSTORE_PASSWORD=password
+
+
+# Allow the container to be started with `--user`
+if [[ "$1" = 'zkServer.sh' && "$(id -u)" = '0' ]]; then
+    chown -R zookeeper "$ZOO_DATA_DIR" "$ZOO_DATA_LOG_DIR" "$ZOO_LOG_DIR" "$ZOO_CONF_DIR"
+    exec gosu zookeeper "$0" "$@"
+fi
+
+# Generate the config only if it doesn't exist
+if [[ ! -f "$ZOO_CONF_DIR/zoo.cfg" ]]; then
+    CONFIG="$ZOO_CONF_DIR/zoo.cfg"
+    {
+        echo "dataDir=$ZOO_DATA_DIR"
+        echo "dataLogDir=$ZOO_DATA_LOG_DIR"
+
+        echo "tickTime=$ZOO_TICK_TIME"
+        echo "initLimit=$ZOO_INIT_LIMIT"
+        echo "syncLimit=$ZOO_SYNC_LIMIT"
+
+        echo "autopurge.snapRetainCount=$ZOO_AUTOPURGE_SNAPRETAINCOUNT"
+        echo "autopurge.purgeInterval=$ZOO_AUTOPURGE_PURGEINTERVAL"
+        echo "maxClientCnxns=$ZOO_MAX_CLIENT_CNXNS"
+        echo "standaloneEnabled=$ZOO_STANDALONE_ENABLED"
+        echo "admin.enableServer=$ZOO_ADMINSERVER_ENABLED"
+    } >> "$CONFIG"
+    if [[ -z $ZOO_SERVERS ]]; then
+      ZOO_SERVERS="server.1=localhost:2888:3888;2181"
+    fi
+
+    for server in $ZOO_SERVERS; do
+        echo "$server" >> "$CONFIG"
+    done
+
+    if [[ -n $ZOO_4LW_COMMANDS_WHITELIST ]]; then
+        echo "4lw.commands.whitelist=$ZOO_4LW_COMMANDS_WHITELIST" >> "$CONFIG"
+    fi
+
+
+    if [[ -n $ZOO_SSL_QUORUM ]]; then
+        {
+            echo "sslQuorum=$ZOO_SSL_QUORUM"
+            echo "serverCnxnFactory=$ZOO_SERVER_CNXN_FACTORY"
+            echo "ssl.quorum.keyStore.location=$ZOO_SSL_QUORUM_KEYSTORE_LOCATION"
+            echo "ssl.quorum.keyStore.password=$ZOO_SSL_QUORUM_KEYSTORE_PASSWORD"
+            echo "ssl.quorum.trustStore.location=$ZOO_SSL_QUORUM_TRUSTSTORE_LOCATION"
+            echo "ssl.quorum.trustStore.password=$ZOO_SSL_QUORUM_TRUSTSTORE_PASSWORD"
+        } >> "$CONFIG"
+    fi
+
+    if [[ -n $ZOO_PORT_UNIFICATION ]]; then
+        echo "portUnification=$ZOO_PORT_UNIFICATION" >> "$CONFIG"
+    fi
+
+    if [[ -n $ZOO_SECURE_CLIENT_PORT ]]; then
+        {
+            echo "secureClientPort=$ZOO_SECURE_CLIENT_PORT"
+            echo "serverCnxnFactory=$ZOO_SERVER_CNXN_FACTORY"
+            echo "ssl.keyStore.location=$ZOO_SSL_KEYSTORE_LOCATION"
+            echo "ssl.keyStore.password=$ZOO_SSL_KEYSTORE_PASSWORD"
+            echo "ssl.trustStore.location=$ZOO_SSL_TRUSTSTORE_LOCATION"
+            echo "ssl.trustStore.password=$ZOO_SSL_TRUSTSTORE_PASSWORD"
+        } >> "$CONFIG"
+    fi
+
+    if [[ -n $ZOO_CLIENT_PORT_UNIFICATION ]]; then
+        echo "client.portUnification=$ZOO_CLIENT_PORT_UNIFICATION" >> "$CONFIG"
+    fi
+fi
+
+# Write myid only if it doesn't exist
+if [[ ! -f "$ZOO_DATA_DIR/myid" ]]; then
+    echo "${ZOO_MY_ID:-1}" > "$ZOO_DATA_DIR/myid"
+fi
+
+mkdir -p $(dirname $ZOO_SSL_KEYSTORE_LOCATION)
+mkdir -p $(dirname $ZOO_SSL_TRUSTSTORE_LOCATION)
+
+if [[ ! -f "$ZOO_SSL_KEYSTORE_LOCATION" ]]; then
+    keytool -genkeypair -alias zookeeper -keyalg RSA -validity 365 -keysize 2048 -dname "cn=zookeeper" -keypass password -keystore $ZOO_SSL_KEYSTORE_LOCATION -storepass password -deststoretype pkcs12
+fi
+
+if [[ ! -f "$ZOO_SSL_TRUSTSTORE_LOCATION" ]]; then
+    keytool -importcert -alias zookeeper -file /clickhouse-config/client.crt -keystore $ZOO_SSL_TRUSTSTORE_LOCATION -storepass password -noprompt -deststoretype pkcs12
+fi
+
+exec "$@"

tests/integration/test_zookeeper_config/configs/zookeeper_config_with_ssl.xml

@@ -0,0 +1,20 @@
+<yandex>
+    <zookeeper>
+        <node index="1">
+            <host>zoo1</host>
+            <port>2281</port>
+            <secure>1</secure>
+        </node>
+        <node index="2">
+            <host>zoo2</host>
+            <port>2281</port>
+            <secure>1</secure>
+        </node>
+        <node index="3">
+            <host>zoo3</host>
+            <port>2281</port>
+            <secure>1</secure>
+        </node>
+        <session_timeout_ms>3000</session_timeout_ms>
+    </zookeeper>
+</yandex>

tests/integration/test_zookeeper_config/configs_secure/client.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV
+BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow
+FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI
+4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T
+4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU
+7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj
+sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg
+pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC
+kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw
+DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j
+4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr
+85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C
+L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD
++UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L
+P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp
+0g==
+-----END CERTIFICATE-----

tests/integration/test_zookeeper_config/configs_secure/client.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn
+DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L
+OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus
+TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7
+JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F
+KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6
+ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP
+TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f
+7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN
+MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5
+212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu
+plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr
+/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu
+Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa
+CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g
+1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5
+bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq
+cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez
+on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K
+GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54
+nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU
+2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh
+vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa
+dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr
+vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1
+KCBtmIrQLqzMpnBpLNuSY+Q=
+-----END PRIVATE KEY-----

tests/integration/test_zookeeper_config/configs_secure/conf.d/remote_servers.xml

@@ -0,0 +1,17 @@
+<yandex>
+    <remote_servers>
+        <test_cluster>
+            <shard>
+                <replica>
+                    <host>node1</host>
+                    <port>9000</port>
+                </replica>
+
+                <replica>
+                    <host>node2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </test_cluster>
+    </remote_servers>
+</yandex>

tests/integration/test_zookeeper_config/configs_secure/conf.d/ssl_conf.xml

@@ -0,0 +1,16 @@
+<yandex>
+    <openSSL>
+        <client>
+            <certificateFile>/etc/clickhouse-server/client.crt</certificateFile>
+            <privateKeyFile>/etc/clickhouse-server/client.key</privateKeyFile>
+            <loadDefaultCAFile>true</loadDefaultCAFile>
+            <cacheSessions>true</cacheSessions>
+            <disableProtocols>sslv2,sslv3</disableProtocols>
+            <preferServerCiphers>true</preferServerCiphers>
+            <verificationMode>none</verificationMode>
+            <invalidCertificateHandler>
+                <name>RejectCertificateHandler</name>
+            </invalidCertificateHandler>
+        </client>
+    </openSSL>
+</yandex>

tests/integration/test_zookeeper_config/test.py

@@ -1,7 +1,11 @@
 from __future__ import print_function
 from helpers.cluster import ClickHouseCluster
+import helpers
 import pytest
 import time
+from tempfile import NamedTemporaryFile
+from os import path as p, unlink
+
 
 def test_chroot_with_same_root():
 
@@ -100,10 +104,58 @@ def test_identity():
         with pytest.raises(Exception):
             cluster_2.start(destroy_dirs=False)
             node2.query('''
-            CREATE TABLE simple (date Date, id UInt32) 
+            CREATE TABLE simple (date Date, id UInt32)
             ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '1', date, id, 8192);
             ''')
 
     finally:
         cluster_1.shutdown()
         cluster_2.shutdown()
+
+
+def test_secure_connection():
+    # We need absolute path in zookeeper volumes. Generate it dynamically.
+    TEMPLATE = '''
+    zoo{zoo_id}:
+        image: zookeeper:3.5.6
+        restart: always
+        environment:
+            ZOO_TICK_TIME: 500
+            ZOO_MY_ID: {zoo_id}
+            ZOO_SERVERS: server.1=zoo1:2888:3888;2181 server.2=zoo2:2888:3888;2181 server.3=zoo3:2888:3888;2181
+            ZOO_SECURE_CLIENT_PORT: 2281
+        volumes:
+           - {helpers_dir}/zookeeper-ssl-entrypoint.sh:/zookeeper-ssl-entrypoint.sh
+           - {configs_dir}:/clickhouse-config
+        command: ["zkServer.sh", "start-foreground"]
+        entrypoint: /zookeeper-ssl-entrypoint.sh
+    '''
+    configs_dir = p.abspath(p.join(p.dirname(__file__), 'configs_secure'))
+    helpers_dir = p.abspath(p.dirname(helpers.__file__))
+
+    cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/zookeeper_config_with_ssl.xml')
+
+    docker_compose = NamedTemporaryFile(delete=False)
+
+    docker_compose.write(
+        "version: '2.2'\nservices:\n" +
+        TEMPLATE.format(zoo_id=1, configs_dir=configs_dir, helpers_dir=helpers_dir) +
+        TEMPLATE.format(zoo_id=2, configs_dir=configs_dir, helpers_dir=helpers_dir) +
+        TEMPLATE.format(zoo_id=3, configs_dir=configs_dir, helpers_dir=helpers_dir)
+    )
+    docker_compose.close()
+
+    node1 = cluster.add_instance('node1', config_dir='configs_secure', with_zookeeper=True,
+                                 zookeeper_docker_compose_path=docker_compose.name)
+    node2 = cluster.add_instance('node2', config_dir='configs_secure', with_zookeeper=True,
+                                 zookeeper_docker_compose_path=docker_compose.name)
+
+    try:
+        cluster.start()
+
+        assert node1.query("SELECT count() FROM system.zookeeper WHERE path = '/'") == '2\n'
+        assert node2.query("SELECT count() FROM system.zookeeper WHERE path = '/'") == '2\n'
+
+    finally:
+        cluster.shutdown()
+        unlink(docker_compose.name)