thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 17:41:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Automatic style fix

Browse Source
This commit is contained in:
robot-clickhouse 2023-06-05 23:58:43 +00:00
parent b22e664e25
commit cf9936ad32
1 changed files with 68 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
tests/integration/test_storage_azure_blob_storage/test.py
View File

@ -17,6 +17,7 @@ from helpers.network import PartitionManager
from helpers.mock_servers import start_mock_servers from helpers.mock_servers import start_mock_servers
from helpers.test_tools import exec_query_with_retry from helpers.test_tools import exec_query_with_retry
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def cluster(): def cluster():
try: try:
@ -32,6 +33,7 @@ def cluster():
finally: finally:
cluster.shutdown() cluster.shutdown()
def azure_query(node, query, try_num=3, settings={}): def azure_query(node, query, try_num=3, settings={}):
for i in range(try_num): for i in range(try_num):
try: try:
@ -50,6 +52,7 @@ def azure_query(node, query, try_num=3, settings={}):
raise Exception(ex) raise Exception(ex)
continue continue
def get_azure_file_content(filename): def get_azure_file_content(filename):
container_name = "cont" container_name = "cont"
connection_string = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" connection_string = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;"
@ -57,43 +60,72 @@ def get_azure_file_content(filename):
container_client = blob_service_client.get_container_client(container_name) container_client = blob_service_client.get_container_client(container_name)
blob_client = container_client.get_blob_client(filename) blob_client = container_client.get_blob_client(filename)
download_stream = blob_client.download_blob() download_stream = blob_client.download_blob()
return download_stream.readall().decode('utf-8') return download_stream.readall().decode("utf-8")
def test_create_table_connection_string(cluster): def test_create_table_connection_string(cluster):
node = cluster.instances["node"] node = cluster.instances["node"]
azure_query(node, "CREATE TABLE test_create_table_conn_string (key UInt64, data String) Engine = Azure('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite1:10000/devstoreaccount1/;', 'cont', 'test_create_connection_string', 'CSV')") azure_query(
node,
"CREATE TABLE test_create_table_conn_string (key UInt64, data String) Engine = Azure('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite1:10000/devstoreaccount1/;', 'cont', 'test_create_connection_string', 'CSV')",
)
def test_create_table_account_string(cluster): def test_create_table_account_string(cluster):
node = cluster.instances["node"] node = cluster.instances["node"]
azure_query(node, "CREATE TABLE test_create_table_account_url (key UInt64, data String) Engine = Azure('http://azurite1:10000/devstoreaccount1', 'cont', 'test_create_connection_string', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV')") azure_query(
node,
"CREATE TABLE test_create_table_account_url (key UInt64, data String) Engine = Azure('http://azurite1:10000/devstoreaccount1', 'cont', 'test_create_connection_string', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV')",
)
def test_simple_write_account_string(cluster): def test_simple_write_account_string(cluster):
node = cluster.instances["node"] node = cluster.instances["node"]
azure_query(node, "CREATE TABLE test_simple_write (key UInt64, data String) Engine = Azure('http://azurite1:10000/devstoreaccount1', 'cont', 'test_simple_write.csv', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV')") azure_query(
node,
"CREATE TABLE test_simple_write (key UInt64, data String) Engine = Azure('http://azurite1:10000/devstoreaccount1', 'cont', 'test_simple_write.csv', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV')",
)
azure_query(node, "INSERT INTO test_simple_write VALUES (1, 'a')") azure_query(node, "INSERT INTO test_simple_write VALUES (1, 'a')")
print(get_azure_file_content('test_simple_write.csv')) print(get_azure_file_content("test_simple_write.csv"))
assert get_azure_file_content('test_simple_write.csv') == '1,"a"\n' assert get_azure_file_content("test_simple_write.csv") == '1,"a"\n'
def test_simple_write_connection_string(cluster): def test_simple_write_connection_string(cluster):
node = cluster.instances["node"] node = cluster.instances["node"]
azure_query(node, "CREATE TABLE test_simple_write_connection_string (key UInt64, data String) Engine = Azure('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite1:10000/devstoreaccount1;', 'cont', 'test_simple_write_c.csv', 'CSV')") azure_query(
node,
"CREATE TABLE test_simple_write_connection_string (key UInt64, data String) Engine = Azure('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite1:10000/devstoreaccount1;', 'cont', 'test_simple_write_c.csv', 'CSV')",
)
azure_query(node, "INSERT INTO test_simple_write_connection_string VALUES (1, 'a')") azure_query(node, "INSERT INTO test_simple_write_connection_string VALUES (1, 'a')")
print(get_azure_file_content('test_simple_write_c.csv')) print(get_azure_file_content("test_simple_write_c.csv"))
assert get_azure_file_content('test_simple_write_c.csv') == '1,"a"\n' assert get_azure_file_content("test_simple_write_c.csv") == '1,"a"\n'
def test_simple_write_named_collection_1(cluster): def test_simple_write_named_collection_1(cluster):
node = cluster.instances["node"] node = cluster.instances["node"]
azure_query(node, "CREATE TABLE test_simple_write_named_collection_1 (key UInt64, data String) Engine = Azure(azure_conf1)") azure_query(
azure_query(node, "INSERT INTO test_simple_write_named_collection_1 VALUES (1, 'a')") node,
print(get_azure_file_content('test_simple_write_named.csv')) "CREATE TABLE test_simple_write_named_collection_1 (key UInt64, data String) Engine = Azure(azure_conf1)",
assert get_azure_file_content('test_simple_write_named.csv') == '1,"a"\n' )
azure_query(
node, "INSERT INTO test_simple_write_named_collection_1 VALUES (1, 'a')"
)
print(get_azure_file_content("test_simple_write_named.csv"))
assert get_azure_file_content("test_simple_write_named.csv") == '1,"a"\n'
def test_simple_write_named_collection_2(cluster): def test_simple_write_named_collection_2(cluster):
node = cluster.instances["node"] node = cluster.instances["node"]
azure_query(node, "CREATE TABLE test_simple_write_named_collection_2 (key UInt64, data String) Engine = Azure(azure_conf2, container='cont', blob_path='test_simple_write_named_2.csv', format='CSV')") azure_query(
azure_query(node, "INSERT INTO test_simple_write_named_collection_2 VALUES (1, 'a')") node,
print(get_azure_file_content('test_simple_write_named_2.csv')) "CREATE TABLE test_simple_write_named_collection_2 (key UInt64, data String) Engine = Azure(azure_conf2, container='cont', blob_path='test_simple_write_named_2.csv', format='CSV')",
assert get_azure_file_content('test_simple_write_named_2.csv') == '1,"a"\n' )
azure_query(
node, "INSERT INTO test_simple_write_named_collection_2 VALUES (1, 'a')"
)
print(get_azure_file_content("test_simple_write_named_2.csv"))
assert get_azure_file_content("test_simple_write_named_2.csv") == '1,"a"\n'
def test_partition_by(cluster): def test_partition_by(cluster):
node = cluster.instances["node"] node = cluster.instances["node"]
@ -102,7 +134,10 @@ def test_partition_by(cluster):
values = "(1, 2, 3), (3, 2, 1), (78, 43, 45)" values = "(1, 2, 3), (3, 2, 1), (78, 43, 45)"
filename = "test_{_partition_id}.csv" filename = "test_{_partition_id}.csv"
azure_query(node, f"CREATE TABLE test_partitioned_write ({table_format}) Engine = Azure('http://azurite1:10000/devstoreaccount1', 'cont', '{filename}', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV') PARTITION BY {partition_by}") azure_query(
node,
f"CREATE TABLE test_partitioned_write ({table_format}) Engine = Azure('http://azurite1:10000/devstoreaccount1', 'cont', '{filename}', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV') PARTITION BY {partition_by}",
)
azure_query(node, f"INSERT INTO test_partitioned_write VALUES {values}") azure_query(node, f"INSERT INTO test_partitioned_write VALUES {values}")
assert "1,2,3\n" == get_azure_file_content("test_3.csv") assert "1,2,3\n" == get_azure_file_content("test_3.csv")
@ -116,7 +151,10 @@ def test_partition_by_string_column(cluster):
partition_by = "col_str" partition_by = "col_str"
values = "(1, 'foo/bar'), (3, 'йцук'), (78, '你好')" values = "(1, 'foo/bar'), (3, 'йцук'), (78, '你好')"
filename = "test_{_partition_id}.csv" filename = "test_{_partition_id}.csv"
azure_query(node, f"CREATE TABLE test_partitioned_string_write ({table_format}) Engine = Azure('http://azurite1:10000/devstoreaccount1', 'cont', '{filename}', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV') PARTITION BY {partition_by}") azure_query(
node,
f"CREATE TABLE test_partitioned_string_write ({table_format}) Engine = Azure('http://azurite1:10000/devstoreaccount1', 'cont', '{filename}', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV') PARTITION BY {partition_by}",
)
azure_query(node, f"INSERT INTO test_partitioned_string_write VALUES {values}") azure_query(node, f"INSERT INTO test_partitioned_string_write VALUES {values}")
assert '1,"foo/bar"\n' == get_azure_file_content("test_foo/bar.csv") assert '1,"foo/bar"\n' == get_azure_file_content("test_foo/bar.csv")
@ -131,15 +169,22 @@ def test_partition_by_const_column(cluster):
partition_by = "'88'" partition_by = "'88'"
values_csv = "1,2,3\n3,2,1\n78,43,45\n" values_csv = "1,2,3\n3,2,1\n78,43,45\n"
filename = "test_{_partition_id}.csv" filename = "test_{_partition_id}.csv"
azure_query(node, f"CREATE TABLE test_partitioned_const_write ({table_format}) Engine = Azure('http://azurite1:10000/devstoreaccount1', 'cont', '{filename}', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV') PARTITION BY {partition_by}") azure_query(
node,
f"CREATE TABLE test_partitioned_const_write ({table_format}) Engine = Azure('http://azurite1:10000/devstoreaccount1', 'cont', '{filename}', 'devstoreaccount1', 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'CSV') PARTITION BY {partition_by}",
)
azure_query(node, f"INSERT INTO test_partitioned_const_write VALUES {values}") azure_query(node, f"INSERT INTO test_partitioned_const_write VALUES {values}")
assert values_csv == get_azure_file_content("test_88.csv") assert values_csv == get_azure_file_content("test_88.csv")
def test_truncate(cluster): def test_truncate(cluster):
node = cluster.instances["node"] node = cluster.instances["node"]
azure_query(node, "CREATE TABLE test_truncate (key UInt64, data String) Engine = Azure(azure_conf2, container='cont', blob_path='test_truncate.csv', format='CSV')") azure_query(
node,
"CREATE TABLE test_truncate (key UInt64, data String) Engine = Azure(azure_conf2, container='cont', blob_path='test_truncate.csv', format='CSV')",
)
azure_query(node, "INSERT INTO test_truncate VALUES (1, 'a')") azure_query(node, "INSERT INTO test_truncate VALUES (1, 'a')")
assert get_azure_file_content('test_truncate.csv') == '1,"a"\n' assert get_azure_file_content("test_truncate.csv") == '1,"a"\n'
azure_query(node, "TRUNCATE TABLE test_truncate") azure_query(node, "TRUNCATE TABLE test_truncate")
with pytest.raises(Exception): with pytest.raises(Exception):
print(get_azure_file_content('test_truncate.csv')) print(get_azure_file_content("test_truncate.csv"))