diff --git a/tests/queries/0_stateless/01548_query_log_query_execution_ms.reference b/tests/queries/0_stateless/01548_query_log_query_execution_ms.reference
index 6ed281c757a..e69de29bb2d 100644
--- a/tests/queries/0_stateless/01548_query_log_query_execution_ms.reference
+++ b/tests/queries/0_stateless/01548_query_log_query_execution_ms.reference
@@ -1,2 +0,0 @@
-1
-1
diff --git a/tests/queries/0_stateless/01548_query_log_query_execution_ms.sh b/tests/queries/0_stateless/01548_query_log_query_execution_ms.sh
new file mode 100755
index 00000000000..c973612c80d
--- /dev/null
+++ b/tests/queries/0_stateless/01548_query_log_query_execution_ms.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+function random_str()
+{
+    local n=$1 && shift
+    tr -cd '[:lower:]' < /dev/urandom | head -c"$n"
+}
+function test_query_duration_ms()
+{
+    local query_id
+    query_id="01548_query_log_query_execution_ms-$SECONDS-$(random_str 6)"
+    local query_opts=(
+        "--log_query_threads=1"
+        "--log_queries_min_type=QUERY_FINISH"
+        "--log_queries=1"
+        "--query_id=$query_id"
+        "--format=Null"
+    )
+    $CLICKHOUSE_CLIENT "${query_opts[@]}" -q "select sleep(0.4)" || exit 1
+    $CLICKHOUSE_CLIENT -q "system flush logs" || exit 1
+
+    $CLICKHOUSE_CLIENT -q "
+        select count()
+        from system.query_log
+        where
+            query_id = '$query_id'
+            and current_database = currentDatabase()
+            and query_duration_ms between 400 and 800
+            and event_date >= yesterday()
+            and event_time >= now() - interval 1 minute;
+    " || exit 1
+
+    $CLICKHOUSE_CLIENT -q "
+        -- at least two threads for processing
+        -- (but one just waits for another, sigh)
+        select count() == 2
+        from system.query_thread_log
+        where
+            query_id = '$query_id'
+            and current_database = currentDatabase()
+            and query_duration_ms between 400 and 800
+            and event_date >= yesterday()
+            and event_time >= now() - interval 1 minute;
+    " || exit 1
+}
+
+function main()
+{
+    # retries, since there is no guarantee that every time query will take ~0.4 second.
+    local retries=20 i=0
+    while [ "$(test_query_duration_ms | xargs)" != '1 1' ] && [[ $i < $retries ]]; do
+        ((++i))
+    done
+}
+main "$@"
diff --git a/tests/queries/0_stateless/01548_query_log_query_execution_ms.sql b/tests/queries/0_stateless/01548_query_log_query_execution_ms.sql
deleted file mode 100644
index 41d43444944..00000000000
--- a/tests/queries/0_stateless/01548_query_log_query_execution_ms.sql
+++ /dev/null
@@ -1,27 +0,0 @@
-set log_query_threads=1;
-set log_queries_min_type='QUERY_FINISH';
-set log_queries=1;
-select '01548_query_log_query_execution_ms', sleep(0.4) format Null;
-set log_queries=0;
-set log_query_threads=0;
-
-system flush logs;
-
-select count()
-from system.query_log
-where
-    query like '%01548_query_log_query_execution_ms%'
-    and current_database = currentDatabase()
-    and query_duration_ms between 400 and 3000
-    and event_date >= yesterday();
-
--- at least two threads for processing
--- (but one just waits for another, sigh)
-select count() == 2
-from system.query_thread_log
-where
-    query like '%01548_query_log_query_execution_ms%'
-    and current_database = currentDatabase()
-    and query_duration_ms between 400 and 3000
-    and event_date = today()
-    and event_time >= now() - interval 1 minute;