thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add integration test

Browse Source
This commit is contained in:
Pablo Marcos 2024-08-20 10:35:51 +00:00
parent c269e0f71c
commit f5a0202474
6 changed files with 177 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp
View File

@ -628,9 +628,9 @@ std::unique_ptr<NamedCollectionsMetadataStorage> NamedCollectionsMetadataStorage
const auto path = config.getString(named_collections_storage_config_path + ".path");
std::unique_ptr<INamedCollectionsStorage> zk_storage;
if (storage_type == "zookeeper" || storage_type == "keeper")
if (!storage_type.ends_with("_encrypted"))
zk_storage = std::make_unique<NamedCollectionsMetadataStorage::ZooKeeperStorage>(context_, path);
else if (storage_type == "zookeeper_encrypted" || storage_type == "keeper_encrypted")
else
zk_storage = std::make_unique<NamedCollectionsMetadataStorage::ZooKeeperStorageEncrypted>(context_, path);
LOG_TRACE(getLogger("NamedCollectionsMetadataStorage"),

0
tests/integration/test_named_collections_encrypted/__init__.py Normal file
View File

12
tests/integration/test_named_collections_encrypted/configs/config.d/named_collections_encrypted.xml Normal file
View File

@ -0,0 +1,12 @@
<clickhouse>
<named_collections_storage>
<type>local_encrypted</type>
<key_hex>bebec0cabebec0cabebec0cabebec0ca</key_hex>
</named_collections_storage>
<named_collections>
<collection1>
<key1>value1</key1>
</collection1>
</named_collections>
</clickhouse>

31
tests/integration/test_named_collections_encrypted/configs/config.d/named_collections_with_zookeeper_encrypted.xml Normal file
View File

@ -0,0 +1,31 @@
<clickhouse>
<named_collections_storage>
<type>zookeeper_encrypted</type>
<key_hex>bebec0cabebec0cabebec0cabebec0ca</key_hex>
<path>/named_collections_path/</path>
<update_timeout_ms>5000</update_timeout_ms>
</named_collections_storage>
<named_collections>
<collection1>
<key1>value1</key1>
</collection1>
</named_collections>
<remote_servers>
<replicated_nc_nodes_cluster>
<shard>
<internal_replication>true</internal_replication>
<replica>
<host>node_with_keeper</host>
<port>9000</port>
</replica>
<replica>
<host>node_with_keeper_2</host>
<port>9000</port>
</replica>
</shard>
<allow_distributed_ddl_queries>true</allow_distributed_ddl_queries>
</replicated_nc_nodes_cluster>
</remote_servers>
</clickhouse>

17
tests/integration/test_named_collections_encrypted/configs/users.d/users.xml Normal file
View File

@ -0,0 +1,17 @@
<clickhouse>
<profiles>
<default>
<ignore_on_cluster_for_replicated_named_collections_queries>0</ignore_on_cluster_for_replicated_named_collections_queries>
</default>
</profiles>
<users>
<default>
<password></password>
<profile>default</profile>
<quota>default</quota>
<named_collection_control>1</named_collection_control>
<show_named_collections>1</show_named_collections>
<show_named_collections_secrets>1</show_named_collections_secrets>
</default>
</users>
</clickhouse>

115
tests/integration/test_named_collections_encrypted/test.py Normal file
View File

@ -0,0 +1,115 @@
import logging
import pytest
import os
from helpers.cluster import ClickHouseCluster
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
NAMED_COLLECTIONS_CONFIG = os.path.join(
SCRIPT_DIR, "./configs/config.d/named_collections.xml"
)
ZK_PATH = "/named_collections_path"
@pytest.fixture(scope="module")
def cluster():
try:
cluster = ClickHouseCluster(__file__)
cluster.add_instance(
"node_encrypted",
main_configs=[
"configs/config.d/named_collections_encrypted.xml",
],
user_configs=[
"configs/users.d/users.xml",
],
stay_alive=True,
)
cluster.add_instance(
"node_with_keeper_encrypted",
main_configs=[
"configs/config.d/named_collections_with_zookeeper_encrypted.xml",
],
user_configs=[
"configs/users.d/users.xml",
],
stay_alive=True,
with_zookeeper=True,
)
cluster.add_instance(
"node_with_keeper_2_encrypted",
main_configs=[
"configs/config.d/named_collections_with_zookeeper_encrypted.xml",
],
user_configs=[
"configs/users.d/users.xml",
],
stay_alive=True,
with_zookeeper=True,
)
logging.info("Starting cluster...")
cluster.start()
logging.info("Cluster started")
yield cluster
finally:
cluster.shutdown()
def check_encrypted_content(node, zk=None):
assert (
"collection1\ncollection2"
== node.query("select name from system.named_collections").strip()
)
assert (
"['key1','key2']"
== node.query(
"select mapKeys(collection) from system.named_collections where name = 'collection2'"
).strip()
)
assert (
"1234\tvalue2"
== node.query(
"select collection['key1'], collection['key2'] from system.named_collections where name = 'collection2'"
).strip()
)
# Check that the underlying storage is encrypted
content = zk.get(ZK_PATH + "/collection2.sql")[0] if zk is not None else open(f"{node.path}/database/named_collections/collection2.sql", "rb").read()
assert content[0:3] == b"ENC" # file signature (aka magic number) of the encrypted file
assert b"key1" not in content
assert b"1234" not in content
assert b"key2" not in content
assert B"value2" not in content
def test_local_storage_encrypted(cluster):
node = cluster.instances["node_encrypted"]
node.query("CREATE NAMED COLLECTION collection2 AS key1=1234, key2='value2'")
check_encrypted_content(node)
node.restart_clickhouse()
check_encrypted_content(node)
node.query("DROP NAMED COLLECTION collection2")
def test_zookeper_storage_encrypted(cluster):
node1 = cluster.instances["node_with_keeper_encrypted"]
node2 = cluster.instances["node_with_keeper_2_encrypted"]
zk = cluster.get_kazoo_client("zoo1")
node1.query("CREATE NAMED COLLECTION collection2 AS key1=1234, key2='value2'")
check_encrypted_content(node1, zk)
check_encrypted_content(node2, zk)
node1.restart_clickhouse()
node2.restart_clickhouse()
check_encrypted_content(node1, zk)
check_encrypted_content(node2, zk)
node1.query("DROP NAMED COLLECTION collection2")