thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-25 00:52:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add to system.process, improve test slightly

Browse Source
This commit is contained in:
Michael Kolupaev 2023-12-06 03:13:59 +00:00
parent bda01ca9db
commit 4d732cdf1e
2 changed files with 41 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/Storages/MaterializedView/RefreshTask.cpp
View File

@ -6,6 +6,7 @@
#include <Interpreters/Context.h>
#include <Interpreters/InterpreterInsertQuery.h>
#include <Interpreters/InterpreterDropQuery.h>
#include <Interpreters/ProcessList.h>
#include <Parsers/ASTCreateQuery.h>
#include <Processors/Executors/ManualPipelineExecutor.h>
@ -419,14 +420,28 @@ bool RefreshTask::executeRefreshUnlocked()
CurrentThread::QueryScope query_scope(refresh_context); // create a thread group for the query
/// TODO: Execute in multiple threads.
/// TODO: interrupt_execution is not enough to interrupt the PipelineExecutor reliably quickly,
/// it doesn't interrupt the epoll for async tasks (if any), see async_task_queue.
/// Either do refresh_executor.cancel() (which won't work for pausing) or add a periodic
/// check of the flag in the epoll wait (wake up every e.g. 500 ms).
if (!refresh_executor)
{
refresh_pipeline = InterpreterInsertQuery(refresh_query, refresh_context).execute();
refresh_pipeline->pipeline.setProgressCallback([this](const Progress & prog)
{
/// TODO: Investigate why most fields are not populated. Change columns in system.view_refreshes as needed, update documentation (docs/en/operations/system-tables/view_refreshes.md).
progress.incrementPiecewiseAtomically(prog);
});
/// Add the query to system.processes and allow it to be killed with KILL QUERY.
String query_for_logging = refresh_query->formatForLogging(
refresh_context->getSettingsRef().log_queries_cut_to_length);
refresh_pipeline->process_list_entry = refresh_context->getProcessList().insert(
query_for_logging, refresh_query.get(), refresh_context, Stopwatch{CLOCK_MONOTONIC}.getStart());
refresh_pipeline->pipeline.setProcessListElement(refresh_pipeline->process_list_entry->getQueryStatus());
refresh_context->setProcessListElement(refresh_pipeline->process_list_entry->getQueryStatus());
refresh_executor.emplace(refresh_pipeline->pipeline);
}

52
tests/queries/0_stateless/02932_refreshable_materialized_views.sh
View File

@ -18,13 +18,13 @@ $CLICKHOUSE_CLIENT -nq "create view refreshes as select * from system.view_refre
# Basic refreshing.
$CLICKHOUSE_CLIENT -nq "
create materialized view a
refresh after 1 second (x UInt64)
refresh after 1 second
engine Memory
as select number as x from numbers(2) union all select rand64() as x"
$CLICKHOUSE_CLIENT -nq "select '<1: created view>', view, remaining_dependencies, exception, last_refresh_result in ('Unknown', 'Finished') from refreshes";
$CLICKHOUSE_CLIENT -nq "show create a"
# Wait for any refresh. (xargs trims the string and turns \t and \n into spaces)
while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes" | xargs`" == 'Unknown' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" == 'Unknown' ]
do
sleep 0.1
done
@ -34,7 +34,7 @@ $CLICKHOUSE_CLIENT -nq "select '<2: refreshed>', count(), sum(x=0), sum(x=1) fro
res1="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values'`"
while :
do
res2="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values'`"
res2="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`"
[ "$res2" == "$res1" ] || break
sleep 0.1
done
@ -42,7 +42,7 @@ time2="`$CLICKHOUSE_CLIENT -nq "select reinterpret(now64(), 'Int64')"`"
# Wait for another change.
while :
do
res3="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values'`"
res3="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`"
[ "$res3" == "$res2" ] || break
sleep 0.1
done
@ -63,7 +63,7 @@ $CLICKHOUSE_CLIENT -nq "
alter table a modify refresh every 2 year;
alter table a modify query select x*2 as x from src;"
$CLICKHOUSE_CLIENT -nq "show create a"
while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes" | xargs`" != 'Scheduled 2052-01-01 00:00:00' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes -- $LINENO" | xargs`" != 'Scheduled 2052-01-01 00:00:00' ]
do
sleep 0.1
done
@ -71,7 +71,7 @@ done
$CLICKHOUSE_CLIENT -nq "
select '<5: no refresh>', count() from a;
system test view a set fake time '2052-02-03 04:05:06';"
while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_time from refreshes" | xargs`" != '2052-02-03 04:05:06' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_time from refreshes -- $LINENO" | xargs`" != '2052-02-03 04:05:06' ]
do
sleep 0.1
done
@ -85,7 +85,7 @@ $CLICKHOUSE_CLIENT -nq "
show create b;
system test view b set fake time '2052-11-11 11:11:11';
system refresh view b;"
while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_time from refreshes where view = 'b'" | xargs`" != '2052-11-11 11:11:11' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2052-11-11 11:11:11' ]
do
sleep 0.1
done
@ -94,7 +94,7 @@ $CLICKHOUSE_CLIENT -nq "
select '<8: refreshed>', * from b;
select '<9: refreshed>', view, status, last_refresh_result, next_refresh_time from refreshes;
system test view b set fake time '2054-01-24 23:22:21';"
while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes where view = 'b'" | xargs`" != 'WaitingForDependencies 2054-01-01 00:00:00' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies 2054-01-01 00:00:00' ]
do
sleep 0.1
done
@ -105,7 +105,7 @@ $CLICKHOUSE_CLIENT -nq "
drop table src;
create table src (x Int16) engine Memory as select 2;
system test view a set fake time '2054-01-01 00:00:01';"
while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b'" | xargs`" != 'Scheduled' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled' ]
do
sleep 0.1
done
@ -120,14 +120,14 @@ $CLICKHOUSE_CLIENT -nq "
$CLICKHOUSE_CLIENT -nq "
system test view b set fake time '2059-01-01 00:00:00';
system refresh view b;"
while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b'" | xargs`" != '2060-01-01 00:00:00' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2060-01-01 00:00:00' ]
do
sleep 0.1
done
$CLICKHOUSE_CLIENT -nq "
system test view b set fake time '2061-01-01 00:00:00';
system test view a set fake time '2057-01-01 00:00:00';"
while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes" | xargs`" != 'Scheduled 2058-01-01 00:00:00 WaitingForDependencies 2060-01-01 00:00:00' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes -- $LINENO" | xargs`" != 'Scheduled 2058-01-01 00:00:00 WaitingForDependencies 2060-01-01 00:00:00' ]
do
sleep 0.1
done
@ -137,7 +137,7 @@ $CLICKHOUSE_CLIENT -nq "
truncate src;
insert into src values (3);
system test view a set fake time '2060-02-02 02:02:02';"
while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b'" | xargs`" != '2062-01-01 00:00:00' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2062-01-01 00:00:00' ]
do
sleep 0.1
done
@ -149,13 +149,13 @@ $CLICKHOUSE_CLIENT -nq "
# Get to WaitingForDependencies state and remove the depencency.
$CLICKHOUSE_CLIENT -nq "
system test view b set fake time '2062-03-03 03:03:03'"
while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b'" | xargs`" != 'WaitingForDependencies' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ]
do
sleep 0.1
done
$CLICKHOUSE_CLIENT -nq "
alter table b modify refresh every 2 year"
while [ "`$CLICKHOUSE_CLIENT -nq "select status, last_refresh_time from refreshes where view = 'b'" | xargs`" != 'Scheduled 2062-03-03 03:03:03' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select status, last_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled 2062-03-03 03:03:03' ]
do
sleep 0.1
done
@ -169,7 +169,7 @@ $CLICKHOUSE_CLIENT -nq "
drop table b;
create materialized view c refresh every 1 second (x Int64) engine Memory as select * from src;
drop table src;"
while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes" | xargs`" != 'Exception' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Exception' ]
do
sleep 0.1
done
@ -178,7 +178,7 @@ $CLICKHOUSE_CLIENT -nq "
select '<19: exception>', exception ilike '%table%src%exist%UNKNOWN_TABLE%' from refreshes;
create table src (x Int64) engine Memory as select 1;
system refresh view c;"
while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes" | xargs`" != 'Finished' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ]
do
sleep 0.1
done
@ -196,7 +196,7 @@ $CLICKHOUSE_CLIENT -nq "
truncate src;
insert into src values (1)
create materialized view e refresh every 1 second (x Int64) engine MergeTree order by x as select x + sleepEachRow(1) as x from src settings max_block_size = 1;"
while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes" | xargs`" != 'Finished' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ]
do
sleep 0.1
done
@ -204,7 +204,7 @@ done
$CLICKHOUSE_CLIENT -nq "
select '<23: simple refresh>', * from e;
system stop view e;"
while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes" | xargs`" != 'Disabled' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Disabled' ]
do
sleep 0.1
done
@ -213,7 +213,7 @@ done
$CLICKHOUSE_CLIENT -nq "
insert into src select * from numbers(1000) settings max_block_size=1;
system start view e;"
while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes" | xargs`" != 'Running' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ]
do
sleep 0.1
done
@ -228,14 +228,14 @@ sleep 2 # make it likely that at least one row was processed
rows_before_pause="`$CLICKHOUSE_CLIENT -nq "select read_rows from refreshes" | xargs`"
$CLICKHOUSE_CLIENT -nq "
system pause view f;"
while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes" | xargs`" != 'Paused' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Paused' ]
do
sleep 0.1
done
# Resume.
$CLICKHOUSE_CLIENT -nq "
system resume view f;"
while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes" | xargs`" != 'Running' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ]
do
sleep 0.1
done
@ -244,7 +244,7 @@ $CLICKHOUSE_CLIENT -nq "
# Cancel.
$CLICKHOUSE_CLIENT -nq "
system cancel view f;"
while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes" | xargs`" != 'Canceled' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Canceled' ]
do
sleep 0.1
done
@ -253,7 +253,7 @@ sleep 1
$CLICKHOUSE_CLIENT -nq "
select '<27: canceled>', view, status from refreshes;
system refresh view f;"
while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes" | xargs`" != 'Running' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ]
do
sleep 0.1
done
@ -267,7 +267,7 @@ $CLICKHOUSE_CLIENT -nq "
create materialized view g refresh every 1 week offset 3 day 4 hour randomize for 4 day 1 hour (x Int64) engine Memory as select 42;
show create g;
system test view g set fake time '2050-02-03 15:30:13';"
while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time > '2049-01-01' from refreshes" | xargs`" != '1' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time > '2049-01-01' from refreshes -- $LINENO" | xargs`" != '1' ]
do
sleep 0.1
done
@ -283,14 +283,14 @@ $CLICKHOUSE_CLIENT -nq "
insert into src values (1);
create materialized view h refresh every 1 second to dest as select x*10 as x from src;
show create h;"
while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes" | xargs`" != 'Finished' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ]
do
sleep 0.1
done
$CLICKHOUSE_CLIENT -nq "
select '<30: to existing table>', * from dest;
insert into src values (2);"
while [ "`$CLICKHOUSE_CLIENT -nq "select count() from dest" | xargs`" != '2' ]
while [ "`$CLICKHOUSE_CLIENT -nq "select count() from dest -- $LINENO" | xargs`" != '2' ]
do
sleep 0.1
done