mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
Add tests for encrypted disk and codec and ReplicatedMergeTree. (#30172)
This commit is contained in:
Vitaly Baranov
GitHub
parent
151daf61c1
commit
c0d295980d
@ -0,0 +1,9 @@
|
||||
<clickhouse>
|
||||
<storage_configuration>
|
||||
<disks>
|
||||
<disk_encrypted>
|
||||
<key>aaaaaaaaaaaaaaaa</key>
|
||||
</disk_encrypted>
|
||||
</disks>
|
||||
</storage_configuration>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,9 @@
|
||||
<clickhouse>
|
||||
<storage_configuration>
|
||||
<disks>
|
||||
<disk_encrypted>
|
||||
<key>bbbbbbbbbbbbbbbb</key>
|
||||
</disk_encrypted>
|
||||
</disks>
|
||||
</storage_configuration>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,16 @@
|
||||
<clickhouse>
|
||||
<remote_servers>
|
||||
<cluster>
|
||||
<shard>
|
||||
<replica>
|
||||
<host>node1</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
<replica>
|
||||
<host>node2</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
</cluster>
|
||||
</remote_servers>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,25 @@
|
||||
<clickhouse>
|
||||
<storage_configuration>
|
||||
<disks>
|
||||
<disk_local>
|
||||
<type>local</type>
|
||||
<path>/disk/</path>
|
||||
</disk_local>
|
||||
<disk_encrypted>
|
||||
<type>encrypted</type>
|
||||
<disk>disk_local</disk>
|
||||
<path>encrypted/</path>
|
||||
<key>0000000000000000</key>
|
||||
</disk_encrypted>
|
||||
</disks>
|
||||
<policies>
|
||||
<encrypted_policy>
|
||||
<volumes>
|
||||
<main>
|
||||
<disk>disk_encrypted</disk>
|
||||
</main>
|
||||
</volumes>
|
||||
</encrypted_policy>
|
||||
</policies>
|
||||
</storage_configuration>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,87 @@
|
||||
import pytest
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.test_tools import assert_eq_with_retry, TSV
|
||||
import os
|
||||
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
|
||||
node1 = cluster.add_instance("node1",
|
||||
main_configs=["configs/remote_servers.xml", "configs/storage.xml"],
|
||||
tmpfs=["/disk:size=100M"],
|
||||
macros={'replica': 'node1'},
|
||||
with_zookeeper=True)
|
||||
|
||||
node2 = cluster.add_instance("node2",
|
||||
main_configs=["configs/remote_servers.xml", "configs/storage.xml"],
|
||||
tmpfs=["/disk:size=100M"],
|
||||
macros={'replica': 'node2'},
|
||||
with_zookeeper=True)
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def start_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
yield
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
def copy_keys(instance, keys_file_name):
|
||||
instance.copy_file_to_container(os.path.join(SCRIPT_DIR, f"configs/{keys_file_name}.xml"), "/etc/clickhouse-server/config.d/z_keys.xml")
|
||||
instance.query("SYSTEM RELOAD CONFIG")
|
||||
|
||||
def create_table():
|
||||
node1.query("DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster' NO DELAY")
|
||||
node1.query(
|
||||
"""
|
||||
CREATE TABLE tbl ON CLUSTER 'cluster' (
|
||||
id Int64,
|
||||
str String
|
||||
) ENGINE=ReplicatedMergeTree('/clickhouse/tables/tbl/', '{replica}')
|
||||
ORDER BY id
|
||||
SETTINGS storage_policy='encrypted_policy'
|
||||
"""
|
||||
)
|
||||
|
||||
def insert_data():
|
||||
node1.query("INSERT INTO tbl VALUES (1, 'str1')")
|
||||
node2.query("INSERT INTO tbl VALUES (1, 'str1')") # Test deduplication
|
||||
node2.query("INSERT INTO tbl VALUES (2, 'str2')")
|
||||
|
||||
def optimize_table():
|
||||
node1.query("OPTIMIZE TABLE tbl ON CLUSTER 'cluster' FINAL")
|
||||
|
||||
def check_table():
|
||||
expected=[[1, 'str1'], [2, 'str2']]
|
||||
assert node1.query("SELECT * FROM tbl ORDER BY id") == TSV(expected)
|
||||
assert node2.query("SELECT * FROM tbl ORDER BY id") == TSV(expected)
|
||||
assert node1.query("CHECK TABLE tbl") == "1\n"
|
||||
assert node2.query("CHECK TABLE tbl") == "1\n"
|
||||
|
||||
|
||||
# Actual tests:
|
||||
|
||||
def test_same_keys():
|
||||
copy_keys(node1, 'key_a')
|
||||
copy_keys(node2, 'key_a')
|
||||
create_table()
|
||||
|
||||
insert_data()
|
||||
check_table()
|
||||
|
||||
optimize_table()
|
||||
check_table()
|
||||
|
||||
|
||||
def test_different_keys():
|
||||
copy_keys(node1, 'key_a')
|
||||
copy_keys(node2, 'key_b')
|
||||
create_table()
|
||||
|
||||
insert_data()
|
||||
check_table()
|
||||
|
||||
optimize_table()
|
||||
check_table()
|
||||
@ -0,0 +1,7 @@
|
||||
<clickhouse>
|
||||
<encryption_codecs>
|
||||
<aes_128_gcm_siv>
|
||||
<key>0000000000000000</key>
|
||||
</aes_128_gcm_siv>
|
||||
</encryption_codecs>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,7 @@
|
||||
<clickhouse>
|
||||
<encryption_codecs>
|
||||
<aes_128_gcm_siv>
|
||||
<key>aaaaaaaaaaaaaaaa</key>
|
||||
</aes_128_gcm_siv>
|
||||
</encryption_codecs>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,10 @@
|
||||
<clickhouse>
|
||||
<encryption_codecs>
|
||||
<aes_128_gcm_siv>
|
||||
<key remove="1"/>
|
||||
<key id="0">aaaaaaaaaaaaaaaa</key>
|
||||
<key id="1">bbbbbbbbbbbbbbbb</key>
|
||||
<current_key_id>0</current_key_id>
|
||||
</aes_128_gcm_siv>
|
||||
</encryption_codecs>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,10 @@
|
||||
<clickhouse>
|
||||
<encryption_codecs>
|
||||
<aes_128_gcm_siv>
|
||||
<key remove="1"/>
|
||||
<key id="0">aaaaaaaaaaaaaaaa</key>
|
||||
<key id="1">bbbbbbbbbbbbbbbb</key>
|
||||
<current_key_id>1</current_key_id>
|
||||
</aes_128_gcm_siv>
|
||||
</encryption_codecs>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,8 @@
|
||||
<clickhouse>
|
||||
<encryption_codecs>
|
||||
<aes_128_gcm_siv>
|
||||
<key>aaaaaaaaaaaaaaaa</key>
|
||||
<nonce>xxxxxxxxxxxx</nonce>
|
||||
</aes_128_gcm_siv>
|
||||
</encryption_codecs>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,8 @@
|
||||
<clickhouse>
|
||||
<encryption_codecs>
|
||||
<aes_128_gcm_siv>
|
||||
<key>aaaaaaaaaaaaaaaa</key>
|
||||
<nonce>yyyyyyyyyyyy</nonce>
|
||||
</aes_128_gcm_siv>
|
||||
</encryption_codecs>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,7 @@
|
||||
<clickhouse>
|
||||
<encryption_codecs>
|
||||
<aes_128_gcm_siv>
|
||||
<key>bbbbbbbbbbbbbbbb</key>
|
||||
</aes_128_gcm_siv>
|
||||
</encryption_codecs>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,16 @@
|
||||
<clickhouse>
|
||||
<remote_servers>
|
||||
<cluster>
|
||||
<shard>
|
||||
<replica>
|
||||
<host>node1</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
<replica>
|
||||
<host>node2</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
</cluster>
|
||||
</remote_servers>
|
||||
</clickhouse>
|
||||
@ -0,0 +1,110 @@
|
||||
import pytest
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
from helpers.test_tools import assert_eq_with_retry, TSV
|
||||
import os
|
||||
|
||||
|
||||
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
|
||||
node1 = cluster.add_instance("node1",
|
||||
main_configs=["configs/remote_servers.xml", "configs/encryption_codec.xml"],
|
||||
macros={'replica': 'node1'},
|
||||
with_zookeeper=True)
|
||||
|
||||
node2 = cluster.add_instance("node2",
|
||||
main_configs=["configs/remote_servers.xml", "configs/encryption_codec.xml"],
|
||||
macros={'replica': 'node2'},
|
||||
with_zookeeper=True)
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def start_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
yield
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
def copy_keys(instance, keys_file_name):
|
||||
instance.copy_file_to_container(os.path.join(SCRIPT_DIR, f"configs/{keys_file_name}.xml"), "/etc/clickhouse-server/config.d/z_keys.xml")
|
||||
instance.query("SYSTEM RELOAD CONFIG")
|
||||
|
||||
def create_table():
|
||||
node1.query("DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster' NO DELAY")
|
||||
node1.query(
|
||||
"""
|
||||
CREATE TABLE tbl ON CLUSTER 'cluster' (
|
||||
id Int64,
|
||||
str String Codec(AES_128_GCM_SIV)
|
||||
) ENGINE=ReplicatedMergeTree('/clickhouse/tables/tbl/', '{replica}')
|
||||
ORDER BY id
|
||||
"""
|
||||
)
|
||||
|
||||
def insert_data():
|
||||
node1.query("INSERT INTO tbl VALUES (1, 'str1')")
|
||||
node2.query("INSERT INTO tbl VALUES (1, 'str1')") # Test deduplication
|
||||
node2.query("INSERT INTO tbl VALUES (2, 'str2')")
|
||||
|
||||
def optimize_table():
|
||||
node1.query("OPTIMIZE TABLE tbl ON CLUSTER 'cluster' FINAL")
|
||||
|
||||
def check_table():
|
||||
expected=[[1, 'str1'], [2, 'str2']]
|
||||
assert node1.query("SELECT * FROM tbl ORDER BY id") == TSV(expected)
|
||||
assert node2.query("SELECT * FROM tbl ORDER BY id") == TSV(expected)
|
||||
assert node1.query("CHECK TABLE tbl") == "1\n"
|
||||
assert node2.query("CHECK TABLE tbl") == "1\n"
|
||||
|
||||
|
||||
# Actual tests:
|
||||
|
||||
def test_same_keys():
|
||||
copy_keys(node1, 'key_a')
|
||||
copy_keys(node2, 'key_a')
|
||||
create_table()
|
||||
|
||||
insert_data()
|
||||
check_table()
|
||||
|
||||
optimize_table()
|
||||
check_table()
|
||||
|
||||
|
||||
def test_different_keys():
|
||||
copy_keys(node1, 'key_a')
|
||||
copy_keys(node2, 'key_b')
|
||||
create_table()
|
||||
|
||||
insert_data()
|
||||
assert "BAD_DECRYPT" in node1.query_and_get_error("SELECT * FROM tbl")
|
||||
assert "BAD_DECRYPT" in node2.query_and_get_error("SELECT * FROM tbl")
|
||||
|
||||
# Hang?
|
||||
#optimize_table()
|
||||
#check_table()
|
||||
|
||||
|
||||
def test_different_current_key_ids():
|
||||
copy_keys(node1, 'key_a_and_b_current_a')
|
||||
copy_keys(node2, 'key_a_and_b_current_b')
|
||||
create_table()
|
||||
|
||||
insert_data()
|
||||
check_table()
|
||||
|
||||
optimize_table()
|
||||
check_table()
|
||||
|
||||
|
||||
def test_different_nonces():
|
||||
copy_keys(node1, 'key_a_and_nonce_x')
|
||||
copy_keys(node2, 'key_a_and_nonce_y')
|
||||
create_table()
|
||||
|
||||
insert_data()
|
||||
check_table()
|
||||
|
||||
optimize_table()
|
||||
check_table()
|
||||
Loading…
Reference in New Issue
Block a user