2022-11-16 17:27:13 +00:00
import logging
import pytest
import os
2022-11-17 14:33:06 +00:00
import time
2022-11-16 17:27:13 +00:00
from helpers . cluster import ClickHouseCluster
2024-07-09 17:36:09 +00:00
from contextlib import nullcontext as does_not_raise
from helpers . client import QueryRuntimeException
2022-11-16 17:27:13 +00:00
SCRIPT_DIR = os . path . dirname ( os . path . realpath ( __file__ ) )
NAMED_COLLECTIONS_CONFIG = os . path . join (
SCRIPT_DIR , " ./configs/config.d/named_collections.xml "
)
2024-05-29 10:53:28 +00:00
ZK_PATH = " /named_collections_path "
2022-11-16 17:27:13 +00:00
2024-05-29 11:05:47 +00:00
2022-11-16 17:27:13 +00:00
@pytest.fixture ( scope = " module " )
def cluster ( ) :
try :
cluster = ClickHouseCluster ( __file__ )
cluster . add_instance (
" node " ,
main_configs = [
" configs/config.d/named_collections.xml " ,
] ,
2022-11-17 14:33:06 +00:00
user_configs = [
" configs/users.d/users.xml " ,
] ,
2022-11-16 17:27:13 +00:00
stay_alive = True ,
)
2024-05-29 10:53:28 +00:00
cluster . add_instance (
" node_with_keeper " ,
main_configs = [
" configs/config.d/named_collections_with_zookeeper.xml " ,
] ,
user_configs = [
" configs/users.d/users.xml " ,
] ,
stay_alive = True ,
with_zookeeper = True ,
)
2024-05-31 12:45:49 +00:00
cluster . add_instance (
" node_with_keeper_2 " ,
main_configs = [
" configs/config.d/named_collections_with_zookeeper.xml " ,
] ,
user_configs = [
" configs/users.d/users.xml " ,
] ,
stay_alive = True ,
with_zookeeper = True ,
)
2023-02-03 12:11:21 +00:00
cluster . add_instance (
2023-02-27 13:24:22 +00:00
" node_only_named_collection_control " ,
2023-02-03 12:11:21 +00:00
main_configs = [
" configs/config.d/named_collections.xml " ,
] ,
user_configs = [
2023-02-27 13:24:22 +00:00
" configs/users.d/users_only_named_collection_control.xml " ,
2023-02-03 12:11:21 +00:00
] ,
stay_alive = True ,
)
cluster . add_instance (
2023-02-27 13:24:22 +00:00
" node_no_default_access " ,
2023-02-03 12:11:21 +00:00
main_configs = [
" configs/config.d/named_collections.xml " ,
] ,
user_configs = [
2023-02-27 13:24:22 +00:00
" configs/users.d/users_no_default_access.xml " ,
2023-02-03 12:11:21 +00:00
] ,
stay_alive = True ,
)
2022-11-16 17:27:13 +00:00
logging . info ( " Starting cluster... " )
cluster . start ( )
logging . info ( " Cluster started " )
yield cluster
finally :
cluster . shutdown ( )
2024-05-29 11:05:47 +00:00
2023-02-03 12:11:21 +00:00
def replace_in_server_config ( node , old , new ) :
2022-11-17 14:33:06 +00:00
node . replace_in_config (
" /etc/clickhouse-server/config.d/named_collections.xml " ,
old ,
new ,
)
2022-11-16 17:27:13 +00:00
2023-02-03 12:11:21 +00:00
def replace_in_users_config ( node , old , new ) :
node . replace_in_config (
" /etc/clickhouse-server/users.d/users.xml " ,
old ,
new ,
)
2023-02-11 18:57:41 +00:00
def test_default_access ( cluster ) :
2023-02-03 12:11:21 +00:00
node = cluster . instances [ " node_no_default_access " ]
2023-02-11 18:57:41 +00:00
assert 0 == int ( node . query ( " select count() from system.named_collections " ) )
2023-02-27 13:24:22 +00:00
node = cluster . instances [ " node_only_named_collection_control " ]
assert 1 == int ( node . query ( " select count() from system.named_collections " ) )
assert (
node . query ( " select collection[ ' key1 ' ] from system.named_collections " ) . strip ( )
== " [HIDDEN] "
)
2023-02-03 12:11:21 +00:00
node = cluster . instances [ " node " ]
assert int ( node . query ( " select count() from system.named_collections " ) ) > 0
2023-02-11 18:57:41 +00:00
2023-02-03 12:11:21 +00:00
replace_in_users_config (
2023-02-27 13:24:22 +00:00
node , " named_collection_control>1 " , " named_collection_control>0 "
2023-02-03 12:11:21 +00:00
)
2023-02-27 13:24:22 +00:00
assert " named_collection_control>0 " in node . exec_in_container (
2023-02-03 12:11:21 +00:00
[ " bash " , " -c " , f " cat /etc/clickhouse-server/users.d/users.xml " ]
)
node . restart_clickhouse ( )
2023-02-11 18:57:41 +00:00
assert 0 == int ( node . query ( " select count() from system.named_collections " ) )
2023-02-03 12:11:21 +00:00
replace_in_users_config (
2023-02-27 13:24:22 +00:00
node , " named_collection_control>0 " , " named_collection_control>1 "
2023-02-03 12:11:21 +00:00
)
2023-02-27 13:24:22 +00:00
assert " named_collection_control>1 " in node . exec_in_container (
2023-02-03 12:11:21 +00:00
[ " bash " , " -c " , f " cat /etc/clickhouse-server/users.d/users.xml " ]
)
node . restart_clickhouse ( )
2023-02-21 17:27:37 +00:00
assert (
2023-03-03 13:04:32 +00:00
node . query (
" select collection[ ' key1 ' ] from system.named_collections where name = ' collection1 ' "
) . strip ( )
2023-02-21 17:27:37 +00:00
== " value1 "
)
2023-02-21 17:07:57 +00:00
replace_in_users_config (
node , " show_named_collections_secrets>1 " , " show_named_collections_secrets>0 "
)
assert " show_named_collections_secrets>0 " in node . exec_in_container (
[ " bash " , " -c " , f " cat /etc/clickhouse-server/users.d/users.xml " ]
)
node . restart_clickhouse ( )
2023-02-21 17:27:37 +00:00
assert (
2023-03-03 13:04:32 +00:00
node . query (
" select collection[ ' key1 ' ] from system.named_collections where name = ' collection1 ' "
) . strip ( )
2023-02-21 17:27:37 +00:00
== " [HIDDEN] "
)
2023-02-21 17:07:57 +00:00
replace_in_users_config (
node , " show_named_collections_secrets>0 " , " show_named_collections_secrets>1 "
)
assert " show_named_collections_secrets>1 " in node . exec_in_container (
[ " bash " , " -c " , f " cat /etc/clickhouse-server/users.d/users.xml " ]
)
node . restart_clickhouse ( )
2023-02-21 17:27:37 +00:00
assert (
2023-03-03 13:04:32 +00:00
node . query (
" select collection[ ' key1 ' ] from system.named_collections where name = ' collection1 ' "
) . strip ( )
2023-02-21 17:27:37 +00:00
== " value1 "
)
2023-02-03 12:11:21 +00:00
2023-02-11 11:36:25 +00:00
def test_granular_access_show_query ( cluster ) :
2023-02-09 14:52:11 +00:00
node = cluster . instances [ " node " ]
2023-03-03 13:04:32 +00:00
assert (
" GRANT ALL ON *.* TO default WITH GRANT OPTION "
== node . query ( " SHOW GRANTS FOR default " ) . strip ( )
) # includes named collections control
2023-02-09 14:52:11 +00:00
assert 1 == int ( node . query ( " SELECT count() FROM system.named_collections " ) )
assert (
" collection1 " == node . query ( " SELECT name FROM system.named_collections " ) . strip ( )
)
2023-02-11 18:57:41 +00:00
node . query ( " DROP USER IF EXISTS kek " )
2023-02-09 14:52:11 +00:00
node . query ( " CREATE USER kek " )
node . query ( " GRANT select ON *.* TO kek " )
assert 0 == int (
node . query ( " SELECT count() FROM system.named_collections " , user = " kek " )
)
node . query ( " GRANT show named collections ON collection1 TO kek " )
assert 1 == int (
node . query ( " SELECT count() FROM system.named_collections " , user = " kek " )
)
assert (
" collection1 "
== node . query ( " SELECT name FROM system.named_collections " , user = " kek " ) . strip ( )
)
node . query ( " CREATE NAMED COLLECTION collection2 AS key1=1, key2= ' value2 ' " )
assert 2 == int ( node . query ( " SELECT count() FROM system.named_collections " ) )
assert (
" collection1 \n collection2 "
== node . query ( " select name from system.named_collections " ) . strip ( )
)
assert 1 == int (
node . query ( " SELECT count() FROM system.named_collections " , user = " kek " )
)
assert (
" collection1 "
== node . query ( " select name from system.named_collections " , user = " kek " ) . strip ( )
)
node . query ( " GRANT show named collections ON collection2 TO kek " )
assert 2 == int (
node . query ( " SELECT count() FROM system.named_collections " , user = " kek " )
)
assert (
" collection1 \n collection2 "
== node . query ( " select name from system.named_collections " , user = " kek " ) . strip ( )
)
node . restart_clickhouse ( )
assert (
" collection1 \n collection2 "
== node . query ( " select name from system.named_collections " , user = " kek " ) . strip ( )
)
2023-02-24 13:44:47 +00:00
# check:
# GRANT show named collections ON *
# REVOKE show named collections ON collection
2023-02-11 18:57:41 +00:00
node . query ( " DROP USER IF EXISTS koko " )
2023-02-09 14:52:11 +00:00
node . query ( " CREATE USER koko " )
node . query ( " GRANT select ON *.* TO koko " )
assert 0 == int (
node . query ( " SELECT count() FROM system.named_collections " , user = " koko " )
)
2023-03-20 15:53:54 +00:00
assert " GRANT SELECT ON *.* TO koko " == node . query ( " SHOW GRANTS FOR koko; " ) . strip ( )
2023-02-09 14:52:11 +00:00
node . query ( " GRANT show named collections ON * TO koko " )
2023-03-20 15:53:54 +00:00
assert (
" GRANT SELECT ON *.* TO koko \n GRANT SHOW NAMED COLLECTIONS ON * TO koko "
== node . query ( " SHOW GRANTS FOR koko; " ) . strip ( )
)
2023-02-09 14:52:11 +00:00
assert (
" collection1 \n collection2 "
== node . query ( " select name from system.named_collections " , user = " koko " ) . strip ( )
)
node . restart_clickhouse ( )
2023-03-20 15:53:54 +00:00
assert (
" GRANT SELECT ON *.* TO koko \n GRANT SHOW NAMED COLLECTIONS ON * TO koko "
== node . query ( " SHOW GRANTS FOR koko; " ) . strip ( )
)
2023-02-09 14:52:11 +00:00
assert (
" collection1 \n collection2 "
== node . query ( " select name from system.named_collections " , user = " koko " ) . strip ( )
)
2023-02-24 13:44:47 +00:00
node . query ( " REVOKE show named collections ON collection1 FROM koko; " )
2023-03-20 15:53:54 +00:00
assert (
" GRANT SELECT ON *.* TO koko \n GRANT SHOW NAMED COLLECTIONS ON * TO koko \n REVOKE SHOW NAMED COLLECTIONS ON collection1 FROM koko "
== node . query ( " SHOW GRANTS FOR koko; " ) . strip ( )
)
2023-02-24 13:44:47 +00:00
assert (
" collection2 "
== node . query ( " select name from system.named_collections " , user = " koko " ) . strip ( )
)
node . restart_clickhouse ( )
2023-03-20 15:53:54 +00:00
assert (
" GRANT SELECT ON *.* TO koko \n GRANT SHOW NAMED COLLECTIONS ON * TO koko \n REVOKE SHOW NAMED COLLECTIONS ON collection1 FROM koko "
== node . query ( " SHOW GRANTS FOR koko; " ) . strip ( )
)
2023-02-24 13:44:47 +00:00
assert (
" collection2 "
== node . query ( " select name from system.named_collections " , user = " koko " ) . strip ( )
)
node . query ( " REVOKE show named collections ON collection2 FROM koko; " )
assert (
" " == node . query ( " select * from system.named_collections " , user = " koko " ) . strip ( )
)
2023-03-20 15:53:54 +00:00
assert (
" GRANT SELECT ON *.* TO koko \n GRANT SHOW NAMED COLLECTIONS ON * TO koko \n REVOKE SHOW NAMED COLLECTIONS ON collection1 FROM koko \n REVOKE SHOW NAMED COLLECTIONS ON collection2 FROM koko "
== node . query ( " SHOW GRANTS FOR koko; " ) . strip ( )
)
2023-02-24 13:44:47 +00:00
# check:
# GRANT show named collections ON collection
# REVOKE show named collections ON *
node . query ( " GRANT show named collections ON collection2 TO koko " )
2023-03-20 15:53:54 +00:00
assert (
" GRANT SELECT ON *.* TO koko \n GRANT SHOW NAMED COLLECTIONS ON * TO koko \n REVOKE SHOW NAMED COLLECTIONS ON collection1 FROM koko "
== node . query ( " SHOW GRANTS FOR koko; " ) . strip ( )
)
2023-02-24 13:44:47 +00:00
assert (
" collection2 "
== node . query ( " select name from system.named_collections " , user = " koko " ) . strip ( )
)
node . query ( " REVOKE show named collections ON * FROM koko; " )
2023-03-20 15:53:54 +00:00
assert " GRANT SELECT ON *.* TO koko " == node . query ( " SHOW GRANTS FOR koko; " ) . strip ( )
2023-02-24 13:44:47 +00:00
assert (
" " == node . query ( " select * from system.named_collections " , user = " koko " ) . strip ( )
)
2023-02-09 14:52:11 +00:00
node . query ( " DROP NAMED COLLECTION collection2 " )
2023-02-24 15:19:36 +00:00
def test_show_grants ( cluster ) :
node = cluster . instances [ " node " ]
node . query ( " DROP USER IF EXISTS koko " )
node . query ( " CREATE USER koko " )
node . query ( " GRANT CREATE NAMED COLLECTION ON name1 TO koko " )
node . query ( " GRANT select ON name1.* TO koko " )
assert (
" GRANT SELECT ON name1.* TO koko \n GRANT CREATE NAMED COLLECTION ON name1 TO koko "
2023-03-20 15:53:54 +00:00
== node . query ( " SHOW GRANTS FOR koko; " ) . strip ( )
2023-02-24 15:19:36 +00:00
)
node . query ( " DROP USER IF EXISTS koko " )
node . query ( " CREATE USER koko " )
node . query ( " GRANT CREATE NAMED COLLECTION ON name1 TO koko " )
node . query ( " GRANT select ON name1 TO koko " )
assert (
" GRANT SELECT ON default.name1 TO koko \n GRANT CREATE NAMED COLLECTION ON name1 TO koko "
2023-03-20 15:53:54 +00:00
== node . query ( " SHOW GRANTS FOR koko; " ) . strip ( )
2023-02-24 15:19:36 +00:00
)
node . query ( " DROP USER IF EXISTS koko " )
node . query ( " CREATE USER koko " )
node . query ( " GRANT select ON name1 TO koko " )
node . query ( " GRANT CREATE NAMED COLLECTION ON name1 TO koko " )
assert (
" GRANT SELECT ON default.name1 TO koko \n GRANT CREATE NAMED COLLECTION ON name1 TO koko "
2023-03-20 15:53:54 +00:00
== node . query ( " SHOW GRANTS FOR koko; " ) . strip ( )
2023-02-24 15:19:36 +00:00
)
node . query ( " DROP USER IF EXISTS koko " )
node . query ( " CREATE USER koko " )
node . query ( " GRANT select ON *.* TO koko " )
node . query ( " GRANT CREATE NAMED COLLECTION ON * TO koko " )
assert (
" GRANT SELECT ON *.* TO koko \n GRANT CREATE NAMED COLLECTION ON * TO koko "
2023-03-20 15:53:54 +00:00
== node . query ( " SHOW GRANTS FOR koko; " ) . strip ( )
2023-02-24 15:19:36 +00:00
)
node . query ( " DROP USER IF EXISTS koko " )
node . query ( " CREATE USER koko " )
node . query ( " GRANT CREATE NAMED COLLECTION ON * TO koko " )
node . query ( " GRANT select ON *.* TO koko " )
assert (
" GRANT SELECT ON *.* TO koko \n GRANT CREATE NAMED COLLECTION ON * TO koko "
2023-03-20 15:53:54 +00:00
== node . query ( " SHOW GRANTS FOR koko; " ) . strip ( )
2023-02-24 15:49:25 +00:00
)
node . query ( " DROP USER IF EXISTS koko " )
node . query ( " CREATE USER koko " )
node . query ( " GRANT CREATE NAMED COLLECTION ON * TO koko " )
node . query ( " GRANT select ON * TO koko " )
assert (
" GRANT CREATE NAMED COLLECTION ON * TO koko \n GRANT SELECT ON default.* TO koko "
2023-03-20 15:53:54 +00:00
== node . query ( " SHOW GRANTS FOR koko; " ) . strip ( )
2023-02-24 15:49:25 +00:00
)
node . query ( " DROP USER IF EXISTS koko " )
node . query ( " CREATE USER koko " )
node . query ( " GRANT select ON * TO koko " )
node . query ( " GRANT CREATE NAMED COLLECTION ON * TO koko " )
assert (
2023-02-24 16:08:05 +00:00
" GRANT CREATE NAMED COLLECTION ON * TO koko \n GRANT SELECT ON default.* TO koko "
2023-03-20 15:53:54 +00:00
== node . query ( " SHOW GRANTS FOR koko; " ) . strip ( )
2023-02-24 15:19:36 +00:00
)
2023-02-11 11:36:25 +00:00
def test_granular_access_create_alter_drop_query ( cluster ) :
node = cluster . instances [ " node " ]
2023-02-11 18:57:41 +00:00
node . query ( " DROP USER IF EXISTS kek " )
2023-02-11 11:36:25 +00:00
node . query ( " CREATE USER kek " )
node . query ( " GRANT select ON *.* TO kek " )
assert 0 == int (
node . query ( " SELECT count() FROM system.named_collections " , user = " kek " )
)
assert (
2023-08-06 14:29:15 +00:00
" DB::Exception: kek: Not enough privileges. To execute this query, it ' s necessary to have the grant CREATE NAMED COLLECTION "
2023-02-11 11:36:25 +00:00
in node . query_and_get_error (
" CREATE NAMED COLLECTION collection2 AS key1=1, key2= ' value2 ' " , user = " kek "
)
)
node . query ( " GRANT create named collection ON collection2 TO kek " )
2023-02-11 18:57:41 +00:00
node . query (
2023-02-11 11:36:25 +00:00
" CREATE NAMED COLLECTION collection2 AS key1=1, key2= ' value2 ' " , user = " kek "
)
assert 0 == int (
node . query ( " select count() from system.named_collections " , user = " kek " )
)
node . query ( " GRANT show named collections ON collection2 TO kek " )
2023-02-11 18:57:41 +00:00
assert (
" collection2 "
== node . query ( " select name from system.named_collections " , user = " kek " ) . strip ( )
)
assert (
" 1 "
== node . query (
" select collection[ ' key1 ' ] from system.named_collections where name = ' collection2 ' "
) . strip ( )
)
assert (
2023-08-06 14:29:15 +00:00
" DB::Exception: kek: Not enough privileges. To execute this query, it ' s necessary to have the grant ALTER NAMED COLLECTION "
2023-02-11 18:57:41 +00:00
in node . query_and_get_error (
" ALTER NAMED COLLECTION collection2 SET key1=2 " , user = " kek "
)
)
node . query ( " GRANT alter named collection ON collection2 TO kek " )
node . query ( " ALTER NAMED COLLECTION collection2 SET key1=2 " , user = " kek " )
assert (
" 2 "
== node . query (
" select collection[ ' key1 ' ] from system.named_collections where name = ' collection2 ' "
) . strip ( )
)
2023-02-24 15:19:36 +00:00
node . query ( " REVOKE alter named collection ON collection2 FROM kek " )
2023-02-24 13:44:47 +00:00
assert (
2023-08-06 14:29:15 +00:00
" DB::Exception: kek: Not enough privileges. To execute this query, it ' s necessary to have the grant ALTER NAMED COLLECTION "
2023-02-24 13:44:47 +00:00
in node . query_and_get_error (
" ALTER NAMED COLLECTION collection2 SET key1=3 " , user = " kek "
)
)
2023-02-11 18:57:41 +00:00
assert (
2023-08-06 14:29:15 +00:00
" DB::Exception: kek: Not enough privileges. To execute this query, it ' s necessary to have the grant DROP NAMED COLLECTION "
2023-02-11 18:57:41 +00:00
in node . query_and_get_error ( " DROP NAMED COLLECTION collection2 " , user = " kek " )
)
node . query ( " GRANT drop named collection ON collection2 TO kek " )
node . query ( " DROP NAMED COLLECTION collection2 " , user = " kek " )
assert 0 == int (
node . query ( " select count() from system.named_collections " , user = " kek " )
)
2023-02-11 11:36:25 +00:00
2022-11-16 17:27:13 +00:00
def test_config_reload ( cluster ) :
node = cluster . instances [ " node " ]
assert (
" collection1 " == node . query ( " select name from system.named_collections " ) . strip ( )
)
assert (
" [ ' key1 ' ] "
== node . query (
" select mapKeys(collection) from system.named_collections where name = ' collection1 ' "
) . strip ( )
)
assert (
" value1 "
== node . query (
" select collection[ ' key1 ' ] from system.named_collections where name = ' collection1 ' "
) . strip ( )
)
2023-02-03 12:11:21 +00:00
replace_in_server_config ( node , " value1 " , " value2 " )
2022-11-16 17:27:13 +00:00
node . query ( " SYSTEM RELOAD CONFIG " )
assert (
" [ ' key1 ' ] "
== node . query (
" select mapKeys(collection) from system.named_collections where name = ' collection1 ' "
) . strip ( )
)
assert (
" value2 "
== node . query (
" select collection[ ' key1 ' ] from system.named_collections where name = ' collection1 ' "
) . strip ( )
)
2023-03-05 12:13:32 +00:00
replace_in_server_config ( node , " value2 " , " value1 " )
node . query ( " SYSTEM RELOAD CONFIG " )
assert (
" value1 "
== node . query (
" select collection[ ' key1 ' ] from system.named_collections where name = ' collection1 ' "
) . strip ( )
)
2022-11-16 17:27:13 +00:00
2024-05-29 10:53:28 +00:00
@pytest.mark.parametrize ( " with_keeper " , [ False , True ] )
def test_sql_commands ( cluster , with_keeper ) :
zk = None
node = None
if with_keeper :
node = cluster . instances [ " node_with_keeper " ]
zk = cluster . get_kazoo_client ( " zoo1 " )
else :
node = cluster . instances [ " node " ]
2022-11-16 17:27:13 +00:00
assert " 1 " == node . query ( " select count() from system.named_collections " ) . strip ( )
node . query ( " CREATE NAMED COLLECTION collection2 AS key1=1, key2= ' value2 ' " )
def check_created ( ) :
assert (
" collection1 \n collection2 "
== node . query ( " select name from system.named_collections " ) . strip ( )
)
assert (
" [ ' key1 ' , ' key2 ' ] "
== node . query (
" select mapKeys(collection) from system.named_collections where name = ' collection2 ' "
) . strip ( )
)
assert (
" 1 "
== node . query (
" select collection[ ' key1 ' ] from system.named_collections where name = ' collection2 ' "
) . strip ( )
)
assert (
" value2 "
== node . query (
" select collection[ ' key2 ' ] from system.named_collections where name = ' collection2 ' "
) . strip ( )
)
2024-05-29 10:53:28 +00:00
if zk is not None :
children = zk . get_children ( ZK_PATH )
assert 1 == len ( children )
assert " collection2.sql " in children
assert (
b " CREATE NAMED COLLECTION collection2 AS key1 = 1, key2 = ' value2 ' "
in zk . get ( ZK_PATH + " /collection2.sql " ) [ 0 ]
)
2022-11-16 17:27:13 +00:00
check_created ( )
node . restart_clickhouse ( )
check_created ( )
2022-11-17 14:33:06 +00:00
node . query ( " ALTER NAMED COLLECTION collection2 SET key1=4, key3= ' value3 ' " )
def check_altered ( ) :
assert (
" [ ' key1 ' , ' key2 ' , ' key3 ' ] "
== node . query (
" select mapKeys(collection) from system.named_collections where name = ' collection2 ' "
) . strip ( )
)
assert (
" 4 "
== node . query (
" select collection[ ' key1 ' ] from system.named_collections where name = ' collection2 ' "
) . strip ( )
)
assert (
" value3 "
== node . query (
" select collection[ ' key3 ' ] from system.named_collections where name = ' collection2 ' "
) . strip ( )
)
2024-05-29 10:53:28 +00:00
if zk is not None :
children = zk . get_children ( ZK_PATH )
assert 1 == len ( children )
assert " collection2.sql " in children
assert (
b " CREATE NAMED COLLECTION collection2 AS key1 = 4, key2 = ' value2 ' , key3 = ' value3 ' "
in zk . get ( ZK_PATH + " /collection2.sql " ) [ 0 ]
)
2022-11-17 15:01:42 +00:00
check_altered ( )
2022-11-17 14:33:06 +00:00
node . restart_clickhouse ( )
2022-11-17 15:01:42 +00:00
check_altered ( )
2022-11-17 14:33:06 +00:00
node . query ( " ALTER NAMED COLLECTION collection2 DELETE key2 " )
def check_deleted ( ) :
assert (
" [ ' key1 ' , ' key3 ' ] "
== node . query (
" select mapKeys(collection) from system.named_collections where name = ' collection2 ' "
) . strip ( )
)
2024-05-29 10:53:28 +00:00
if zk is not None :
children = zk . get_children ( ZK_PATH )
assert 1 == len ( children )
assert " collection2.sql " in children
assert (
b " CREATE NAMED COLLECTION collection2 AS key1 = 4, key3 = ' value3 ' "
in zk . get ( ZK_PATH + " /collection2.sql " ) [ 0 ]
)
2022-11-17 15:01:42 +00:00
check_deleted ( )
2022-11-17 14:33:06 +00:00
node . restart_clickhouse ( )
2022-11-17 15:01:42 +00:00
check_deleted ( )
2022-11-17 14:33:06 +00:00
2022-11-17 15:01:42 +00:00
node . query (
" ALTER NAMED COLLECTION collection2 SET key3=3, key4= ' value4 ' DELETE key1 "
)
2024-05-31 12:45:49 +00:00
time . sleep ( 2 )
2022-11-17 14:33:06 +00:00
def check_altered_and_deleted ( ) :
assert (
" [ ' key3 ' , ' key4 ' ] "
== node . query (
" select mapKeys(collection) from system.named_collections where name = ' collection2 ' "
) . strip ( )
)
assert (
" 3 "
== node . query (
" select collection[ ' key3 ' ] from system.named_collections where name = ' collection2 ' "
) . strip ( )
)
2022-11-17 15:01:42 +00:00
2022-11-17 14:33:06 +00:00
assert (
" value4 "
== node . query (
" select collection[ ' key4 ' ] from system.named_collections where name = ' collection2 ' "
) . strip ( )
)
2024-05-29 10:53:28 +00:00
if zk is not None :
children = zk . get_children ( ZK_PATH )
assert 1 == len ( children )
assert " collection2.sql " in children
assert (
b " CREATE NAMED COLLECTION collection2 AS key3 = 3, key4 = ' value4 ' "
in zk . get ( ZK_PATH + " /collection2.sql " ) [ 0 ]
)
2022-11-17 15:01:42 +00:00
check_altered_and_deleted ( )
2022-11-17 14:33:06 +00:00
node . restart_clickhouse ( )
2022-11-17 15:01:42 +00:00
check_altered_and_deleted ( )
2022-11-17 14:33:06 +00:00
2022-11-16 17:27:13 +00:00
node . query ( " DROP NAMED COLLECTION collection2 " )
def check_dropped ( ) :
assert " 1 " == node . query ( " select count() from system.named_collections " ) . strip ( )
assert (
" collection1 "
== node . query ( " select name from system.named_collections " ) . strip ( )
)
2024-05-29 10:53:28 +00:00
if zk is not None :
children = zk . get_children ( ZK_PATH )
assert 0 == len ( children )
2022-11-16 17:27:13 +00:00
check_dropped ( )
node . restart_clickhouse ( )
check_dropped ( )
2024-05-31 12:45:49 +00:00
def test_keeper_storage ( cluster ) :
node1 = cluster . instances [ " node_with_keeper " ]
node2 = cluster . instances [ " node_with_keeper_2 " ]
zk = cluster . get_kazoo_client ( " zoo1 " )
assert " 1 " == node1 . query ( " select count() from system.named_collections " ) . strip ( )
assert " 1 " == node2 . query ( " select count() from system.named_collections " ) . strip ( )
node1 . query ( " CREATE NAMED COLLECTION collection2 AS key1=1, key2= ' value2 ' " )
def check_created ( node ) :
assert (
" collection1 \n collection2 "
== node . query ( " select name from system.named_collections " ) . strip ( )
)
assert (
" [ ' key1 ' , ' key2 ' ] "
== node . query (
" select mapKeys(collection) from system.named_collections where name = ' collection2 ' "
) . strip ( )
)
assert (
" 1 "
== node . query (
" select collection[ ' key1 ' ] from system.named_collections where name = ' collection2 ' "
) . strip ( )
)
assert (
" value2 "
== node . query (
" select collection[ ' key2 ' ] from system.named_collections where name = ' collection2 ' "
) . strip ( )
)
children = zk . get_children ( ZK_PATH )
assert 1 == len ( children )
assert " collection2.sql " in children
assert (
b " CREATE NAMED COLLECTION collection2 AS key1 = 1, key2 = ' value2 ' "
in zk . get ( ZK_PATH + " /collection2.sql " ) [ 0 ]
)
check_created ( node1 )
check_created ( node2 )
node1 . restart_clickhouse ( )
node2 . restart_clickhouse ( )
check_created ( node1 )
check_created ( node2 )
node2 . query ( " ALTER NAMED COLLECTION collection2 SET key1=4, key3= ' value3 ' " )
time . sleep ( 5 )
def check_altered ( node ) :
assert (
" [ ' key1 ' , ' key2 ' , ' key3 ' ] "
== node . query (
" select mapKeys(collection) from system.named_collections where name = ' collection2 ' "
) . strip ( )
)
assert (
" 4 "
== node . query (
" select collection[ ' key1 ' ] from system.named_collections where name = ' collection2 ' "
) . strip ( )
)
assert (
" value3 "
== node . query (
" select collection[ ' key3 ' ] from system.named_collections where name = ' collection2 ' "
) . strip ( )
)
if zk is not None :
children = zk . get_children ( ZK_PATH )
assert 1 == len ( children )
assert " collection2.sql " in children
assert (
b " CREATE NAMED COLLECTION collection2 AS key1 = 4, key2 = ' value2 ' , key3 = ' value3 ' "
in zk . get ( ZK_PATH + " /collection2.sql " ) [ 0 ]
)
check_altered ( node2 )
check_altered ( node1 )
node1 . restart_clickhouse ( )
node2 . restart_clickhouse ( )
check_altered ( node1 )
check_altered ( node2 )
node1 . query ( " DROP NAMED COLLECTION collection2 " )
time . sleep ( 5 )
def check_dropped ( node ) :
assert " 1 " == node . query ( " select count() from system.named_collections " ) . strip ( )
assert (
" collection1 "
== node . query ( " select name from system.named_collections " ) . strip ( )
)
if zk is not None :
children = zk . get_children ( ZK_PATH )
assert 0 == len ( children )
check_dropped ( node1 )
check_dropped ( node2 )
node1 . restart_clickhouse ( )
node2 . restart_clickhouse ( )
check_dropped ( node1 )
check_dropped ( node2 )
2024-07-09 17:36:09 +00:00
@pytest.mark.parametrize (
" ignore, expected_raise " ,
[ ( True , does_not_raise ( ) ) , ( False , pytest . raises ( QueryRuntimeException ) ) ] ,
)
def test_keeper_storage_remove_on_cluster ( cluster , ignore , expected_raise ) :
node = cluster . instances [ " node_with_keeper " ]
replace_in_users_config (
node ,
" ignore_on_cluster_for_replicated_named_collections_queries>. " ,
f " ignore_on_cluster_for_replicated_named_collections_queries> { int ( ignore ) } " ,
)
node . query ( " SYSTEM RELOAD CONFIG " )
with expected_raise :
2024-07-10 12:47:19 +00:00
node . query (
" DROP NAMED COLLECTION IF EXISTS test_nc ON CLUSTER `replicated_nc_nodes_cluster` "
)
2024-07-09 17:36:09 +00:00
node . query (
f " CREATE NAMED COLLECTION test_nc ON CLUSTER `replicated_nc_nodes_cluster` AS key1=1, key2=2 OVERRIDABLE "
)
node . query (
f " ALTER NAMED COLLECTION test_nc ON CLUSTER `replicated_nc_nodes_cluster` SET key2=3 "
)
node . query (
f " DROP NAMED COLLECTION test_nc ON CLUSTER `replicated_nc_nodes_cluster` "
)
2024-11-05 12:10:14 +00:00
@pytest.mark.parametrize (
" instance_name " ,
[ ( " node " ) , ( " node_with_keeper " ) ] ,
)
def test_name_escaping ( cluster , instance_name ) :
node = cluster . instances [ instance_name ]
node . query ( " DROP NAMED COLLECTION IF EXISTS `test_!strange/symbols!`; " )
node . query ( " CREATE NAMED COLLECTION `test_!strange/symbols!` AS key1=1, key2=2 " )
node . restart_clickhouse ( )
node . query ( " DROP NAMED COLLECTION `test_!strange/symbols!` " )