From 0e36db543762cb146aa6c233d4536fd62c6101b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?= Date: Wed, 31 Jul 2024 22:09:04 +0000 Subject: [PATCH] Do not check the value of `num_messages_read` because it is not reliable librdkafka usually polls both messages on the first SELECT, but sometimes only one. If it polls only one message first, then it will read two messages before rebalancing at the second SELECT from the table. This means it usually reads a single message twice (thus num_messages_read = 4 is usually fine as 1 discarded message + 3 actually consumed message). But when only one message is read in the first SELECT, then 2 messages are discarded, thus num_messages_read will be 5 as 2 discarded message + 3 actually consumed messages. --- tests/integration/test_storage_kafka/test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index 8393e88db88..37457e00701 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -4771,7 +4771,7 @@ def test_system_kafka_consumers_rebalance(kafka_cluster, max_retries=15): assignments.current_offset, if(length(exceptions.time)>0, exceptions.time[1]::String, 'never') as last_exception_time_, if(length(exceptions.text)>0, exceptions.text[1], 'no exception') as last_exception_, - stable_timestamp(last_poll_time) as last_poll_time_, num_messages_read, stable_timestamp(last_commit_time) as last_commit_time_, + stable_timestamp(last_poll_time) as last_poll_time_, stable_timestamp(last_commit_time) as last_commit_time_, num_commits, stable_timestamp(last_rebalance_time) as last_rebalance_time_, num_rebalance_revocations, num_rebalance_assignments, is_currently_used FROM system.kafka_consumers WHERE database='test' and table IN ('kafka', 'kafka2') format Vertical; @@ -4791,7 +4791,6 @@ assignments.current_offset: [2] last_exception_time_: never last_exception_: no exception last_poll_time_: now -num_messages_read: 4 last_commit_time_: now num_commits: 2 last_rebalance_time_: now @@ -4810,7 +4809,6 @@ assignments.current_offset: [2] last_exception_time_: never last_exception_: no exception last_poll_time_: now -num_messages_read: 1 last_commit_time_: now num_commits: 1 last_rebalance_time_: never