From 4d732cdf1e38913f536727933e733b13b901f13d Mon Sep 17 00:00:00 2001 From: Michael Kolupaev Date: Wed, 6 Dec 2023 03:13:59 +0000 Subject: [PATCH] Add to system.process, improve test slightly --- src/Storages/MaterializedView/RefreshTask.cpp | 15 ++++++ .../02932_refreshable_materialized_views.sh | 52 +++++++++---------- 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/src/Storages/MaterializedView/RefreshTask.cpp b/src/Storages/MaterializedView/RefreshTask.cpp index ab96aaaa455..5e4ae5f4059 100644 --- a/src/Storages/MaterializedView/RefreshTask.cpp +++ b/src/Storages/MaterializedView/RefreshTask.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -419,14 +420,28 @@ bool RefreshTask::executeRefreshUnlocked() CurrentThread::QueryScope query_scope(refresh_context); // create a thread group for the query /// TODO: Execute in multiple threads. + /// TODO: interrupt_execution is not enough to interrupt the PipelineExecutor reliably quickly, + /// it doesn't interrupt the epoll for async tasks (if any), see async_task_queue. + /// Either do refresh_executor.cancel() (which won't work for pausing) or add a periodic + /// check of the flag in the epoll wait (wake up every e.g. 500 ms). if (!refresh_executor) { refresh_pipeline = InterpreterInsertQuery(refresh_query, refresh_context).execute(); + refresh_pipeline->pipeline.setProgressCallback([this](const Progress & prog) { /// TODO: Investigate why most fields are not populated. Change columns in system.view_refreshes as needed, update documentation (docs/en/operations/system-tables/view_refreshes.md). progress.incrementPiecewiseAtomically(prog); }); + + /// Add the query to system.processes and allow it to be killed with KILL QUERY. + String query_for_logging = refresh_query->formatForLogging( + refresh_context->getSettingsRef().log_queries_cut_to_length); + refresh_pipeline->process_list_entry = refresh_context->getProcessList().insert( + query_for_logging, refresh_query.get(), refresh_context, Stopwatch{CLOCK_MONOTONIC}.getStart()); + refresh_pipeline->pipeline.setProcessListElement(refresh_pipeline->process_list_entry->getQueryStatus()); + refresh_context->setProcessListElement(refresh_pipeline->process_list_entry->getQueryStatus()); + refresh_executor.emplace(refresh_pipeline->pipeline); } diff --git a/tests/queries/0_stateless/02932_refreshable_materialized_views.sh b/tests/queries/0_stateless/02932_refreshable_materialized_views.sh index 1a3979f3f4b..627703f22a3 100755 --- a/tests/queries/0_stateless/02932_refreshable_materialized_views.sh +++ b/tests/queries/0_stateless/02932_refreshable_materialized_views.sh @@ -18,13 +18,13 @@ $CLICKHOUSE_CLIENT -nq "create view refreshes as select * from system.view_refre # Basic refreshing. $CLICKHOUSE_CLIENT -nq " create materialized view a - refresh after 1 second (x UInt64) + refresh after 1 second engine Memory as select number as x from numbers(2) union all select rand64() as x" $CLICKHOUSE_CLIENT -nq "select '<1: created view>', view, remaining_dependencies, exception, last_refresh_result in ('Unknown', 'Finished') from refreshes"; $CLICKHOUSE_CLIENT -nq "show create a" # Wait for any refresh. (xargs trims the string and turns \t and \n into spaces) -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes" | xargs`" == 'Unknown' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" == 'Unknown' ] do sleep 0.1 done @@ -34,7 +34,7 @@ $CLICKHOUSE_CLIENT -nq "select '<2: refreshed>', count(), sum(x=0), sum(x=1) fro res1="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values'`" while : do - res2="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values'`" + res2="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`" [ "$res2" == "$res1" ] || break sleep 0.1 done @@ -42,7 +42,7 @@ time2="`$CLICKHOUSE_CLIENT -nq "select reinterpret(now64(), 'Int64')"`" # Wait for another change. while : do - res3="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values'`" + res3="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`" [ "$res3" == "$res2" ] || break sleep 0.1 done @@ -63,7 +63,7 @@ $CLICKHOUSE_CLIENT -nq " alter table a modify refresh every 2 year; alter table a modify query select x*2 as x from src;" $CLICKHOUSE_CLIENT -nq "show create a" -while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes" | xargs`" != 'Scheduled 2052-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes -- $LINENO" | xargs`" != 'Scheduled 2052-01-01 00:00:00' ] do sleep 0.1 done @@ -71,7 +71,7 @@ done $CLICKHOUSE_CLIENT -nq " select '<5: no refresh>', count() from a; system test view a set fake time '2052-02-03 04:05:06';" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_time from refreshes" | xargs`" != '2052-02-03 04:05:06' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_time from refreshes -- $LINENO" | xargs`" != '2052-02-03 04:05:06' ] do sleep 0.1 done @@ -85,7 +85,7 @@ $CLICKHOUSE_CLIENT -nq " show create b; system test view b set fake time '2052-11-11 11:11:11'; system refresh view b;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_time from refreshes where view = 'b'" | xargs`" != '2052-11-11 11:11:11' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2052-11-11 11:11:11' ] do sleep 0.1 done @@ -94,7 +94,7 @@ $CLICKHOUSE_CLIENT -nq " select '<8: refreshed>', * from b; select '<9: refreshed>', view, status, last_refresh_result, next_refresh_time from refreshes; system test view b set fake time '2054-01-24 23:22:21';" -while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes where view = 'b'" | xargs`" != 'WaitingForDependencies 2054-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies 2054-01-01 00:00:00' ] do sleep 0.1 done @@ -105,7 +105,7 @@ $CLICKHOUSE_CLIENT -nq " drop table src; create table src (x Int16) engine Memory as select 2; system test view a set fake time '2054-01-01 00:00:01';" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b'" | xargs`" != 'Scheduled' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled' ] do sleep 0.1 done @@ -120,14 +120,14 @@ $CLICKHOUSE_CLIENT -nq " $CLICKHOUSE_CLIENT -nq " system test view b set fake time '2059-01-01 00:00:00'; system refresh view b;" -while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b'" | xargs`" != '2060-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2060-01-01 00:00:00' ] do sleep 0.1 done $CLICKHOUSE_CLIENT -nq " system test view b set fake time '2061-01-01 00:00:00'; system test view a set fake time '2057-01-01 00:00:00';" -while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes" | xargs`" != 'Scheduled 2058-01-01 00:00:00 WaitingForDependencies 2060-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes -- $LINENO" | xargs`" != 'Scheduled 2058-01-01 00:00:00 WaitingForDependencies 2060-01-01 00:00:00' ] do sleep 0.1 done @@ -137,7 +137,7 @@ $CLICKHOUSE_CLIENT -nq " truncate src; insert into src values (3); system test view a set fake time '2060-02-02 02:02:02';" -while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b'" | xargs`" != '2062-01-01 00:00:00' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2062-01-01 00:00:00' ] do sleep 0.1 done @@ -149,13 +149,13 @@ $CLICKHOUSE_CLIENT -nq " # Get to WaitingForDependencies state and remove the depencency. $CLICKHOUSE_CLIENT -nq " system test view b set fake time '2062-03-03 03:03:03'" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b'" | xargs`" != 'WaitingForDependencies' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ] do sleep 0.1 done $CLICKHOUSE_CLIENT -nq " alter table b modify refresh every 2 year" -while [ "`$CLICKHOUSE_CLIENT -nq "select status, last_refresh_time from refreshes where view = 'b'" | xargs`" != 'Scheduled 2062-03-03 03:03:03' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status, last_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled 2062-03-03 03:03:03' ] do sleep 0.1 done @@ -169,7 +169,7 @@ $CLICKHOUSE_CLIENT -nq " drop table b; create materialized view c refresh every 1 second (x Int64) engine Memory as select * from src; drop table src;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes" | xargs`" != 'Exception' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Exception' ] do sleep 0.1 done @@ -178,7 +178,7 @@ $CLICKHOUSE_CLIENT -nq " select '<19: exception>', exception ilike '%table%src%exist%UNKNOWN_TABLE%' from refreshes; create table src (x Int64) engine Memory as select 1; system refresh view c;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes" | xargs`" != 'Finished' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] do sleep 0.1 done @@ -196,7 +196,7 @@ $CLICKHOUSE_CLIENT -nq " truncate src; insert into src values (1) create materialized view e refresh every 1 second (x Int64) engine MergeTree order by x as select x + sleepEachRow(1) as x from src settings max_block_size = 1;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes" | xargs`" != 'Finished' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] do sleep 0.1 done @@ -204,7 +204,7 @@ done $CLICKHOUSE_CLIENT -nq " select '<23: simple refresh>', * from e; system stop view e;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes" | xargs`" != 'Disabled' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Disabled' ] do sleep 0.1 done @@ -213,7 +213,7 @@ done $CLICKHOUSE_CLIENT -nq " insert into src select * from numbers(1000) settings max_block_size=1; system start view e;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes" | xargs`" != 'Running' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] do sleep 0.1 done @@ -228,14 +228,14 @@ sleep 2 # make it likely that at least one row was processed rows_before_pause="`$CLICKHOUSE_CLIENT -nq "select read_rows from refreshes" | xargs`" $CLICKHOUSE_CLIENT -nq " system pause view f;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes" | xargs`" != 'Paused' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Paused' ] do sleep 0.1 done # Resume. $CLICKHOUSE_CLIENT -nq " system resume view f;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes" | xargs`" != 'Running' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] do sleep 0.1 done @@ -244,7 +244,7 @@ $CLICKHOUSE_CLIENT -nq " # Cancel. $CLICKHOUSE_CLIENT -nq " system cancel view f;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes" | xargs`" != 'Canceled' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Canceled' ] do sleep 0.1 done @@ -253,7 +253,7 @@ sleep 1 $CLICKHOUSE_CLIENT -nq " select '<27: canceled>', view, status from refreshes; system refresh view f;" -while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes" | xargs`" != 'Running' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ] do sleep 0.1 done @@ -267,7 +267,7 @@ $CLICKHOUSE_CLIENT -nq " create materialized view g refresh every 1 week offset 3 day 4 hour randomize for 4 day 1 hour (x Int64) engine Memory as select 42; show create g; system test view g set fake time '2050-02-03 15:30:13';" -while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time > '2049-01-01' from refreshes" | xargs`" != '1' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time > '2049-01-01' from refreshes -- $LINENO" | xargs`" != '1' ] do sleep 0.1 done @@ -283,14 +283,14 @@ $CLICKHOUSE_CLIENT -nq " insert into src values (1); create materialized view h refresh every 1 second to dest as select x*10 as x from src; show create h;" -while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes" | xargs`" != 'Finished' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ] do sleep 0.1 done $CLICKHOUSE_CLIENT -nq " select '<30: to existing table>', * from dest; insert into src values (2);" -while [ "`$CLICKHOUSE_CLIENT -nq "select count() from dest" | xargs`" != '2' ] +while [ "`$CLICKHOUSE_CLIENT -nq "select count() from dest -- $LINENO" | xargs`" != '2' ] do sleep 0.1 done