thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

add metrics and budget checks

Browse Source
This commit is contained in:
serxa 2024-06-14 16:09:00 +00:00
parent ec71d35aa6
commit 709ef2ba85
1 changed files with 51 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
tests/integration/test_scheduler/test.py
View File

@ -28,6 +28,19 @@ def start_cluster():
cluster.shutdown()
def check_profile_event_for_query(workload, profile_event, amount=1):
node.query("system flush logs")
query_pattern = f"workload='{workload}'".replace("'", "\\'")
assert (
int(
node.query(
f"select ProfileEvents['{profile_event}'] from system.query_log where query ilike '%{query_pattern}%' and type = 'QueryFinish' order by query_start_time_microseconds desc limit 1"
)
)
== amount
)
def test_s3_resource_request_granularity():
node.query(
f"""
@ -50,6 +63,11 @@ def test_s3_resource_request_granularity():
f"select dequeued_cost from system.scheduler where resource='network_write' and path='/prio/admin'"
).strip()
)
write_budget_before = int(
node.query(
f"select budget from system.scheduler where resource='network_write' and path='/prio/admin'"
).strip()
)
node.query(f"insert into data select number, randomString(10000000) from numbers(5) SETTINGS workload='admin'")
writes_after = int(
node.query(
@ -61,11 +79,22 @@ def test_s3_resource_request_granularity():
f"select dequeued_cost from system.scheduler where resource='network_write' and path='/prio/admin'"
).strip()
)
write_budget_after = int(
node.query(
f"select budget from system.scheduler where resource='network_write' and path='/prio/admin'"
).strip()
)
assert write_bytes_after - write_bytes_before > 1.0 * total_bytes
assert write_bytes_after - write_bytes_before < 1.2 * total_bytes
assert (write_bytes_after - write_bytes_before) / (writes_after - writes_before) < max_bytes_per_request
assert (write_bytes_after - write_bytes_before) / (writes_after - writes_before) > min_bytes_per_request
write_requests = writes_after - writes_before
write_bytes = (write_bytes_after - write_bytes_before) - (write_budget_after - write_budget_before)
assert write_bytes > 1.0 * total_bytes
assert write_bytes < 1.05 * total_bytes
assert write_bytes / write_requests < max_bytes_per_request
assert write_bytes / write_requests > min_bytes_per_request
check_profile_event_for_query("admin", "SchedulerIOWriteRequests", write_requests)
check_profile_event_for_query("admin", "SchedulerIOWriteBytes", write_bytes)
node.query(f"optimize table data final")
reads_before = int(
node.query(
@ -77,6 +106,11 @@ def test_s3_resource_request_granularity():
f"select dequeued_cost from system.scheduler where resource='network_read' and path='/prio/admin'"
).strip()
)
read_budget_before = int(
node.query(
f"select budget from system.scheduler where resource='network_read' and path='/prio/admin'"
).strip()
)
node.query(f"select count() from data where not ignore(*) SETTINGS workload='admin'")
reads_after = int(
node.query(
@ -88,11 +122,20 @@ def test_s3_resource_request_granularity():
f"select dequeued_cost from system.scheduler where resource='network_read' and path='/prio/admin'"
).strip()
)
read_budget_after = int(
node.query(
f"select budget from system.scheduler where resource='network_read' and path='/prio/admin'"
).strip()
)
assert read_bytes_after - read_bytes_before > 1.0 * total_bytes
assert read_bytes_after - read_bytes_before < 1.2 * total_bytes
assert (read_bytes_after - read_bytes_before) / (reads_after - reads_before) < max_bytes_per_request
assert (read_bytes_after - read_bytes_before) / (reads_after - reads_before) > min_bytes_per_request
read_bytes = (read_bytes_after - read_bytes_before) - (read_budget_after - read_budget_before)
read_requests = reads_after - reads_before
assert read_bytes > 1.0 * total_bytes
assert read_bytes < 1.05 * total_bytes
assert read_bytes / read_requests < max_bytes_per_request
assert read_bytes / read_requests > min_bytes_per_request
check_profile_event_for_query("admin", "SchedulerIOReadRequests", read_requests)
check_profile_event_for_query("admin", "SchedulerIOReadBytes", read_bytes)
def test_s3_disk():