thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-25 17:12:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix flaky azure test

Browse Source
This commit is contained in:
alesapin 2023-01-10 19:05:02 +01:00
parent 37da8e206b
commit 6d86b8dd47
1 changed files with 46 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
tests/integration/test_merge_tree_azure_blob_storage/test.py
View File

@ -42,10 +42,10 @@ def cluster():
# For inserts there is no guarantee that retries will not result in duplicates.
# But it is better to retry anyway because 'Connection was closed by the server' error
# happens in fact only for inserts because reads already have build-in retries in code.
def azure_query(node, query, try_num=3):
def azure_query(node, query, try_num=3, settings={}):
for i in range(try_num):
try:
return node.query(query)
return node.query(query, settings=settings)
except Exception as ex:
retriable_errors = [
"DB::Exception: Azure::Core::Http::TransportException: Connection was closed by the server while trying to read a response"
@ -80,7 +80,7 @@ def create_table(node, table_name, **additional_settings):
ORDER BY (dt, id)
SETTINGS {",".join((k+"="+repr(v) for k, v in settings.items()))}"""
node.query(f"DROP TABLE IF EXISTS {table_name}")
azure_query(node, f"DROP TABLE IF EXISTS {table_name}")
azure_query(node, create_table_statement)
assert (
azure_query(node, f"SELECT COUNT(*) FROM {table_name} FORMAT Values") == "(0)"
@ -230,9 +230,9 @@ def test_alter_table_columns(cluster):
f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096, -1)}",
)
node.query(f"ALTER TABLE {TABLE_NAME} ADD COLUMN col1 UInt64 DEFAULT 1")
azure_query(node, f"ALTER TABLE {TABLE_NAME} ADD COLUMN col1 UInt64 DEFAULT 1")
# To ensure parts have been merged
node.query(f"OPTIMIZE TABLE {TABLE_NAME}")
azure_query(node, f"OPTIMIZE TABLE {TABLE_NAME}")
assert (
azure_query(node, f"SELECT sum(col1) FROM {TABLE_NAME} FORMAT Values")
@ -245,7 +245,8 @@ def test_alter_table_columns(cluster):
== "(4096)"
)
node.query(
azure_query(
node,
f"ALTER TABLE {TABLE_NAME} MODIFY COLUMN col1 String",
settings={"mutations_sync": 2},
)
@ -271,26 +272,27 @@ def test_attach_detach_partition(cluster):
== "(8192)"
)
node.query(f"ALTER TABLE {TABLE_NAME} DETACH PARTITION '2020-01-03'")
azure_query(node, f"ALTER TABLE {TABLE_NAME} DETACH PARTITION '2020-01-03'")
assert (
azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values")
== "(4096)"
)
node.query(f"ALTER TABLE {TABLE_NAME} ATTACH PARTITION '2020-01-03'")
azure_query(node, f"ALTER TABLE {TABLE_NAME} ATTACH PARTITION '2020-01-03'")
assert (
azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values")
== "(8192)"
)
node.query(f"ALTER TABLE {TABLE_NAME} DROP PARTITION '2020-01-03'")
azure_query(node, f"ALTER TABLE {TABLE_NAME} DROP PARTITION '2020-01-03'")
assert (
azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values")
== "(4096)"
)
node.query(f"ALTER TABLE {TABLE_NAME} DETACH PARTITION '2020-01-04'")
node.query(
azure_query(node, f"ALTER TABLE {TABLE_NAME} DETACH PARTITION '2020-01-04'")
azure_query(
node,
f"ALTER TABLE {TABLE_NAME} DROP DETACHED PARTITION '2020-01-04'",
settings={"allow_drop_detached": 1},
)
@ -314,16 +316,18 @@ def test_move_partition_to_another_disk(cluster):
== "(8192)"
)
node.query(
f"ALTER TABLE {TABLE_NAME} MOVE PARTITION '2020-01-04' TO DISK '{LOCAL_DISK}'"
azure_query(
node,
f"ALTER TABLE {TABLE_NAME} MOVE PARTITION '2020-01-04' TO DISK '{LOCAL_DISK}'",
)
assert (
azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values")
== "(8192)"
)
node.query(
f"ALTER TABLE {TABLE_NAME} MOVE PARTITION '2020-01-04' TO DISK '{AZURE_BLOB_STORAGE_DISK}'"
azure_query(
node,
f"ALTER TABLE {TABLE_NAME} MOVE PARTITION '2020-01-04' TO DISK '{AZURE_BLOB_STORAGE_DISK}'",
)
assert (
azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values")
@ -344,14 +348,14 @@ def test_table_manipulations(cluster):
f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-04', 4096)}"
)
node.query(f"RENAME TABLE {TABLE_NAME} TO {renamed_table}")
azure_query(node, f"RENAME TABLE {TABLE_NAME} TO {renamed_table}")
assert (
azure_query(node, f"SELECT count(*) FROM {renamed_table} FORMAT Values")
== "(8192)"
)
node.query(f"RENAME TABLE {renamed_table} TO {TABLE_NAME}")
assert node.query(f"CHECK TABLE {TABLE_NAME} FORMAT Values") == "(1)"
azure_query(node, f"RENAME TABLE {renamed_table} TO {TABLE_NAME}")
assert azure_query(node, f"CHECK TABLE {TABLE_NAME} FORMAT Values") == "(1)"
node.query(f"DETACH TABLE {TABLE_NAME}")
node.query(f"ATTACH TABLE {TABLE_NAME}")
@ -360,7 +364,7 @@ def test_table_manipulations(cluster):
== "(8192)"
)
node.query(f"TRUNCATE TABLE {TABLE_NAME}")
azure_query(node, f"TRUNCATE TABLE {TABLE_NAME}")
assert (
azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") == "(0)"
)
@ -395,11 +399,13 @@ def test_move_replace_partition_to_another_table(cluster):
create_table(node, table_clone_name)
node.query(
f"ALTER TABLE {TABLE_NAME} MOVE PARTITION '2020-01-03' TO TABLE {table_clone_name}"
azure_query(
node,
f"ALTER TABLE {TABLE_NAME} MOVE PARTITION '2020-01-03' TO TABLE {table_clone_name}",
)
node.query(
f"ALTER TABLE {TABLE_NAME} MOVE PARTITION '2020-01-05' TO TABLE {table_clone_name}"
azure_query(
node,
f"ALTER TABLE {TABLE_NAME} MOVE PARTITION '2020-01-05' TO TABLE {table_clone_name}",
)
assert azure_query(node, f"SELECT sum(id) FROM {TABLE_NAME} FORMAT Values") == "(0)"
assert (
@ -428,11 +434,13 @@ def test_move_replace_partition_to_another_table(cluster):
== "(1024)"
)
node.query(
f"ALTER TABLE {TABLE_NAME} REPLACE PARTITION '2020-01-03' FROM {table_clone_name}"
azure_query(
node,
f"ALTER TABLE {TABLE_NAME} REPLACE PARTITION '2020-01-03' FROM {table_clone_name}",
)
node.query(
f"ALTER TABLE {TABLE_NAME} REPLACE PARTITION '2020-01-05' FROM {table_clone_name}"
azure_query(
node,
f"ALTER TABLE {TABLE_NAME} REPLACE PARTITION '2020-01-05' FROM {table_clone_name}",
)
assert azure_query(node, f"SELECT sum(id) FROM {TABLE_NAME} FORMAT Values") == "(0)"
assert (
@ -448,16 +456,16 @@ def test_move_replace_partition_to_another_table(cluster):
== "(512)"
)
node.query(f"DROP TABLE {table_clone_name} NO DELAY")
azure_query(node, f"DROP TABLE {table_clone_name} NO DELAY")
assert azure_query(node, f"SELECT sum(id) FROM {TABLE_NAME} FORMAT Values") == "(0)"
assert (
azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values")
== "(1024)"
)
node.query(f"ALTER TABLE {TABLE_NAME} FREEZE")
azure_query(node, f"ALTER TABLE {TABLE_NAME} FREEZE")
node.query(f"DROP TABLE {TABLE_NAME} NO DELAY")
azure_query(node, f"DROP TABLE {TABLE_NAME} NO DELAY")
def test_freeze_unfreeze(cluster):
@ -470,20 +478,21 @@ def test_freeze_unfreeze(cluster):
azure_query(
node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096)}"
)
node.query(f"ALTER TABLE {TABLE_NAME} FREEZE WITH NAME '{backup1}'")
azure_query(node, f"ALTER TABLE {TABLE_NAME} FREEZE WITH NAME '{backup1}'")
azure_query(
node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-04', 4096)}"
)
node.query(f"ALTER TABLE {TABLE_NAME} FREEZE WITH NAME '{backup2}'")
azure_query(node, f"ALTER TABLE {TABLE_NAME} FREEZE WITH NAME '{backup2}'")
azure_query(node, f"TRUNCATE TABLE {TABLE_NAME}")
# Unfreeze single partition from backup1.
node.query(
f"ALTER TABLE {TABLE_NAME} UNFREEZE PARTITION '2020-01-03' WITH NAME '{backup1}'"
azure_query(
node,
f"ALTER TABLE {TABLE_NAME} UNFREEZE PARTITION '2020-01-03' WITH NAME '{backup1}'",
)
# Unfreeze all partitions from backup2.
node.query(f"ALTER TABLE {TABLE_NAME} UNFREEZE WITH NAME '{backup2}'")
azure_query(node, f"ALTER TABLE {TABLE_NAME} UNFREEZE WITH NAME '{backup2}'")
def test_apply_new_settings(cluster):
@ -524,8 +533,8 @@ def test_big_insert(cluster):
node,
f"INSERT INTO {TABLE_NAME} {check_query}",
)
assert azure_query(node, f"SELECT * FROM {TABLE_NAME} ORDER BY id") == node.query(
check_query
assert azure_query(node, f"SELECT * FROM {TABLE_NAME} ORDER BY id") == azure_query(
node, check_query
)
blob_container_client = cluster.blob_service_client.get_container_client(