diff --git a/docs/en/operations/system-tables/view_refreshes.md b/docs/en/operations/system-tables/view_refreshes.md
new file mode 100644
index 00000000000..12377507b39
--- /dev/null
+++ b/docs/en/operations/system-tables/view_refreshes.md
@@ -0,0 +1,43 @@
+---
+slug: /en/operations/system-tables/view_refreshes
+---
+# view_refreshes
+
+Information about [Refreshable Materialized Views](../../sql-reference/statements/create/view.md#refreshable-materialized-view). Contains all refreshable materialized views, regardless of whether there's a refresh in progress or not.
+
+
+Columns:
+
+- `database` ([String](../../sql-reference/data-types/string.md)) — The name of the database the table is in.
+- `view` ([String](../../sql-reference/data-types/string.md)) — Table name.
+- `status` ([String](../../sql-reference/data-types/string.md)) — Current state of the refresh.
+- `last_refresh_result` ([String](../../sql-reference/data-types/string.md)) — Outcome of the latest refresh attempt.
+- `last_refresh_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Time of the last refresh attempt. `NULL` if no refresh attempts happened since server startup or table creation.
+- `last_success_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Time of the last successful refresh. `NULL` if no successful refreshes happened since server startup or table creation.
+- `duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md)) — How long the last refresh attempt took.
+- `next_refresh_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Time at which the next refresh is scheduled to start.
+- `remaining_dependencies` ([Array(String)](../../sql-reference/data-types/array.md)) — If the view has [refresh dependencies](../../sql-reference/statements/create/view.md#refresh-dependencies), this array contains the subset of those dependencies that are not satisfied for the current refresh yet. If `status = 'WaitingForDependencies'`, a refresh is ready to start as soon as these dependencies are fulfilled.
+- `exception` ([String](../../sql-reference/data-types/string.md)) — if `last_refresh_result = 'Exception'`, i.e. the last refresh attempt failed, this column contains the corresponding error message and stack trace.
+- `refresh_count` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number of successful refreshes since last server restart or table creation.
+- `progress` ([Float64](../../sql-reference/data-types/float.md)) — Progress of the current refresh, between 0 and 1.
+- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number of rows read by the current refresh so far.
+- `total_rows` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Estimated total number of rows that need to be read by the current refresh.
+
+(There are additional columns related to current refresh progress, but they are currently unreliable.)
+
+**Example**
+
+```sql
+SELECT
+ database,
+ view,
+ status,
+ last_refresh_result,
+ last_refresh_time,
+ next_refresh_time
+FROM system.view_refreshes
+
+┌─database─┬─view───────────────────────┬─status────┬─last_refresh_result─┬───last_refresh_time─┬───next_refresh_time─┐
+│ default │ hello_documentation_reader │ Scheduled │ Finished │ 2023-12-01 01:24:00 │ 2023-12-01 01:25:00 │
+└──────────┴────────────────────────────┴───────────┴─────────────────────┴─────────────────────┴─────────────────────┘
+```
diff --git a/docs/en/sql-reference/statements/alter/view.md b/docs/en/sql-reference/statements/alter/view.md
index 5c5bf0355f6..517e64e3e5b 100644
--- a/docs/en/sql-reference/statements/alter/view.md
+++ b/docs/en/sql-reference/statements/alter/view.md
@@ -6,28 +6,28 @@ sidebar_label: VIEW
# ALTER TABLE … MODIFY QUERY Statement
-You can modify `SELECT` query that was specified when a [materialized view](../create/view.md#materialized) was created with the `ALTER TABLE … MODIFY QUERY` statement without interrupting ingestion process.
+You can modify `SELECT` query that was specified when a [materialized view](../create/view.md#materialized) was created with the `ALTER TABLE … MODIFY QUERY` statement without interrupting ingestion process.
-The `allow_experimental_alter_materialized_view_structure` setting must be enabled.
+The `allow_experimental_alter_materialized_view_structure` setting must be enabled.
This command is created to change materialized view created with `TO [db.]name` clause. It does not change the structure of the underling storage table and it does not change the columns' definition of the materialized view, because of this the application of this command is very limited for materialized views are created without `TO [db.]name` clause.
**Example with TO table**
```sql
-CREATE TABLE events (ts DateTime, event_type String)
+CREATE TABLE events (ts DateTime, event_type String)
ENGINE = MergeTree ORDER BY (event_type, ts);
-CREATE TABLE events_by_day (ts DateTime, event_type String, events_cnt UInt64)
+CREATE TABLE events_by_day (ts DateTime, event_type String, events_cnt UInt64)
ENGINE = SummingMergeTree ORDER BY (event_type, ts);
-CREATE MATERIALIZED VIEW mv TO events_by_day AS
+CREATE MATERIALIZED VIEW mv TO events_by_day AS
SELECT toStartOfDay(ts) ts, event_type, count() events_cnt
FROM events
-GROUP BY ts, event_type;
+GROUP BY ts, event_type;
-INSERT INTO events
-SELECT Date '2020-01-01' + interval number * 900 second,
+INSERT INTO events
+SELECT Date '2020-01-01' + interval number * 900 second,
['imp', 'click'][number%2+1]
FROM numbers(100);
@@ -43,23 +43,23 @@ ORDER BY ts, event_type;
│ 2020-01-02 00:00:00 │ imp │ 2 │
└─────────────────────┴────────────┴─────────────────┘
--- Let's add the new measurment `cost`
+-- Let's add the new measurment `cost`
-- and the new dimension `browser`.
-ALTER TABLE events
+ALTER TABLE events
ADD COLUMN browser String,
ADD COLUMN cost Float64;
-- Column do not have to match in a materialized view and TO
-- (destination table), so the next alter does not break insertion.
-ALTER TABLE events_by_day
+ALTER TABLE events_by_day
ADD COLUMN cost Float64,
ADD COLUMN browser String after event_type,
MODIFY ORDER BY (event_type, ts, browser);
-INSERT INTO events
-SELECT Date '2020-01-02' + interval number * 900 second,
+INSERT INTO events
+SELECT Date '2020-01-02' + interval number * 900 second,
['imp', 'click'][number%2+1],
['firefox', 'safary', 'chrome'][number%3+1],
10/(number+1)%33
@@ -82,16 +82,16 @@ ORDER BY ts, event_type;
└─────────────────────┴────────────┴─────────┴────────────┴──────┘
SET allow_experimental_alter_materialized_view_structure=1;
-
-ALTER TABLE mv MODIFY QUERY
+
+ALTER TABLE mv MODIFY QUERY
SELECT toStartOfDay(ts) ts, event_type, browser,
count() events_cnt,
sum(cost) cost
FROM events
GROUP BY ts, event_type, browser;
-INSERT INTO events
-SELECT Date '2020-01-03' + interval number * 900 second,
+INSERT INTO events
+SELECT Date '2020-01-03' + interval number * 900 second,
['imp', 'click'][number%2+1],
['firefox', 'safary', 'chrome'][number%3+1],
10/(number+1)%33
@@ -138,7 +138,7 @@ PRIMARY KEY (event_type, ts)
ORDER BY (event_type, ts, browser)
SETTINGS index_granularity = 8192
--- !!! The columns' definition is unchanged but it does not matter, we are not quering
+-- !!! The columns' definition is unchanged but it does not matter, we are not quering
-- MATERIALIZED VIEW, we are quering TO (storage) table.
-- SELECT section is updated.
@@ -169,7 +169,7 @@ The application is very limited because you can only change the `SELECT` section
```sql
CREATE TABLE src_table (`a` UInt32) ENGINE = MergeTree ORDER BY a;
-CREATE MATERIALIZED VIEW mv (`a` UInt32) ENGINE = MergeTree ORDER BY a AS SELECT a FROM src_table;
+CREATE MATERIALIZED VIEW mv (`a` UInt32) ENGINE = MergeTree ORDER BY a AS SELECT a FROM src_table;
INSERT INTO src_table (a) VALUES (1), (2);
SELECT * FROM mv;
```
@@ -199,3 +199,7 @@ SELECT * FROM mv;
## ALTER LIVE VIEW Statement
`ALTER LIVE VIEW ... REFRESH` statement refreshes a [Live view](../create/view.md#live-view). See [Force Live View Refresh](../create/view.md#live-view-alter-refresh).
+
+## ALTER TABLE … MODIFY REFRESH Statement
+
+`ALTER TABLE ... MODIFY REFRESH` statement changes refresh parameters of a [Refreshable Materialized View](../create/view.md#refreshable-materialized-view). See [Changing Refresh Parameters](../create/view.md#changing-refresh-parameters).
diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md
index 56828745048..f6158acd9a4 100644
--- a/docs/en/sql-reference/statements/create/view.md
+++ b/docs/en/sql-reference/statements/create/view.md
@@ -37,6 +37,7 @@ SELECT a, b, c FROM (SELECT ...)
```
## Parameterized View
+
Parametrized views are similar to normal views, but can be created with parameters which are not resolved immediately. These views can be used with table functions, which specify the name of the view as function name and the parameter values as its arguments.
``` sql
@@ -66,7 +67,7 @@ When creating a materialized view with `TO [db].[table]`, you can't also use `PO
A materialized view is implemented as follows: when inserting data to the table specified in `SELECT`, part of the inserted data is converted by this `SELECT` query, and the result is inserted in the view.
-:::note
+:::note
Materialized views in ClickHouse use **column names** instead of column order during insertion into destination table. If some column names are not present in the `SELECT` query result, ClickHouse uses a default value, even if the column is not [Nullable](../../data-types/nullable.md). A safe practice would be to add aliases for every column when using Materialized views.
Materialized views in ClickHouse are implemented more like insert triggers. If there’s some aggregation in the view query, it’s applied only to the batch of freshly inserted data. Any changes to existing data of source table (like update, delete, drop partition, etc.) does not change the materialized view.
@@ -96,9 +97,116 @@ This feature is deprecated and will be removed in the future.
For your convenience, the old documentation is located [here](https://pastila.nl/?00f32652/fdf07272a7b54bda7e13b919264e449f.md)
+## Refreshable Materialized View {#refreshable-materialized-view}
+
+```sql
+CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name
+REFRESH EVERY|AFTER interval [OFFSET interval]
+RANDOMIZE FOR interval
+DEPENDS ON [db.]name [, [db.]name [, ...]]
+[TO[db.]name] [(columns)] [ENGINE = engine] [EMPTY]
+AS SELECT ...
+```
+where `interval` is a sequence of simple intervals:
+```sql
+number SECOND|MINUTE|HOUR|DAY|WEEK|MONTH|YEAR
+```
+
+Periodically runs the corresponding query and stores its result in a table, atomically replacing the table's previous contents.
+
+Differences from regular non-refreshable materialized views:
+ * No insert trigger. I.e. when new data is inserted into the table specified in SELECT, it's *not* automatically pushed to the refreshable materialized view. The periodic refresh runs the entire query and replaces the entire table.
+ * No restrictions on the SELECT query. Table functions (e.g. `url()`), views, UNION, JOIN, are all allowed.
+
+:::note
+Refreshable materialized views are a work in progress. Setting `allow_experimental_refreshable_materialized_view = 1` is required for creating one. Current limitations:
+ * not compatible with Replicated database or table engines,
+ * require [Atomic database engine](../../../engines/database-engines/atomic.md),
+ * no retries for failed refresh - we just skip to the next scheduled refresh time,
+ * no limit on number of concurrent refreshes.
+:::
+
+### Refresh Schedule
+
+Example refresh schedules:
+```sql
+REFRESH EVERY 1 DAY -- every day, at midnight (UTC)
+REFRESH EVERY 1 MONTH -- on 1st day of every month, at midnight
+REFRESH EVERY 1 MONTH OFFSET 5 DAY 2 HOUR -- on 6th day of every month, at 2:00 am
+REFRESH EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE -- every other Saturday, at 3:10 pm
+REFRESH EVERY 30 MINUTE -- at 00:00, 00:30, 01:00, 01:30, etc
+REFRESH AFTER 30 MINUTE -- 30 minutes after the previous refresh completes, no alignment with time of day
+-- REFRESH AFTER 1 HOUR OFFSET 1 MINUTE -- syntax errror, OFFSET is not allowed with AFTER
+```
+
+`RANDOMIZE FOR` randomly adjusts the time of each refresh, e.g.:
+```sql
+REFRESH EVERY 1 DAY OFFSET 2 HOUR RANDOMIZE FOR 1 HOUR -- every day at random time between 01:30 and 02:30
+```
+
+At most one refresh may be running at a time, for a given view. E.g. if a view with `REFRESH EVERY 1 MINUTE` takes 2 minutes to refresh, it'll just be refreshing every 2 minutes. If it then becomes faster and starts refreshing in 10 seconds, it'll go back to refreshing every minute. (In particular, it won't refresh every 10 seconds to catch up with a backlog of missed refreshes - there's no such backlog.)
+
+Additionally, a refresh is started immediately after the materialized view is created, unless `EMPTY` is specified in the `CREATE` query. If `EMPTY` is specified, the first refresh happens according to schedule.
+
+### Dependencies {#refresh-dependencies}
+
+`DEPENDS ON` synchronizes refreshes of different tables. By way of example, suppose there's a chain of two refreshable materialized views:
+```sql
+CREATE MATERIALIZED VIEW source REFRESH EVERY 1 DAY AS SELECT * FROM url(...)
+CREATE MATERIALIZED VIEW destination REFRESH EVERY 1 DAY AS SELECT ... FROM source
+```
+Without `DEPENDS ON`, both views will start a refresh at midnight, and `destination` typically will see yesterday's data in `source`. If we add dependency:
+```
+CREATE MATERIALIZED VIEW destination REFRESH EVERY 1 DAY DEPENDS ON source AS SELECT ... FROM source
+```
+then `destination`'s refresh will start only after `source`'s refresh finished for that day, so `destination` will be based on fresh data.
+
+Alternatively, the same result can be achieved with:
+```
+CREATE MATERIALIZED VIEW destination REFRESH AFTER 1 HOUR DEPENDS ON source AS SELECT ... FROM source
+```
+where `1 HOUR` can be any duration less than `source`'s refresh period. The dependent table won't be refreshed more frequently than any of its dependencies. This is a valid way to set up a chain of refreshable views without specifying the real refresh period more than once.
+
+A few more examples:
+ * `REFRESH EVERY 1 DAY OFFSET 10 MINUTE` (`destination`) depends on `REFRESH EVERY 1 DAY` (`source`)
+ If `source` refresh takes more than 10 minutes, `destination` will wait for it.
+ * `REFRESH EVERY 1 DAY OFFSET 1 HOUR` depends on `REFRESH EVERY 1 DAY OFFSET 23 HOUR`
+ Similar to the above, even though the corresponding refreshes happen on different calendar days.
+ `destination`'s refresh on day X+1 will wait for `source`'s refresh on day X (if it takes more than 2 hours).
+ * `REFRESH EVERY 2 HOUR` depends on `REFRESH EVERY 1 HOUR`
+ The 2 HOUR refresh happens after the 1 HOUR refresh for every other hour, e.g. after the midnight
+ refresh, then after the 2am refresh, etc.
+ * `REFRESH EVERY 1 MINUTE` depends on `REFRESH EVERY 2 HOUR`
+ `REFRESH AFTER 1 MINUTE` depends on `REFRESH EVERY 2 HOUR`
+ `REFRESH AFTER 1 MINUTE` depends on `REFRESH AFTER 2 HOUR`
+ `destination` is refreshed once after every `source` refresh, i.e. every 2 hours. The `1 MINUTE` is effectively ignored.
+ * `REFRESH AFTER 1 HOUR` depends on `REFRESH AFTER 1 HOUR`
+ Currently this is not recommended.
+
+:::note
+`DEPENDS ON` only works between refreshable materialized views. Listing a regular table in the `DEPENDS ON` list will prevent the view from ever refreshing (dependencies can be removed with `ALTER`, see below).
+:::
+
+### Changing Refresh Parameters {#changing-refresh-parameters}
+
+To change refresh parameters:
+```
+ALTER TABLE [db.]name MODIFY REFRESH EVERY|AFTER ... [RANDOMIZE FOR ...] [DEPENDS ON ...]
+```
+
+:::note
+This replaces refresh schedule *and* dependencies. If the table had a `DEPENDS ON`, doing a `MODIFY REFRESH` without `DEPENDS ON` will remove the dependencies.
+:::
+
+### Other operations
+
+The status of all refreshable materialized views is available in table [`system.view_refreshes`](../../../operations/system-tables/view_refreshes.md). In particular, it contains refresh progress (if running), last and next refresh time, exception message if a refresh failed.
+
+To manually stop, start, trigger, or cancel refreshes use [`SYSTEM STOP|START|REFRESH|CANCEL VIEW`](../system.md#refreshable-materialized-views).
+
## Window View [Experimental]
-:::info
+:::info
This is an experimental feature that may change in backwards-incompatible ways in the future releases. Enable usage of window views and `WATCH` query using [allow_experimental_window_view](../../../operations/settings/settings.md#allow-experimental-window-view) setting. Input the command `set allow_experimental_window_view = 1`.
:::
diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md
index 695801983b7..0fdbbeac235 100644
--- a/docs/en/sql-reference/statements/system.md
+++ b/docs/en/sql-reference/statements/system.md
@@ -449,7 +449,7 @@ SYSTEM SYNC FILE CACHE [ON CLUSTER cluster_name]
```
-### SYSTEM STOP LISTEN
+## SYSTEM STOP LISTEN
Closes the socket and gracefully terminates the existing connections to the server on the specified port with the specified protocol.
@@ -464,7 +464,7 @@ SYSTEM STOP LISTEN [ON CLUSTER cluster_name] [QUERIES ALL | QUERIES DEFAULT | QU
- If `QUERIES DEFAULT [EXCEPT .. [,..]]` modifier is specified, all default protocols are stopped, unless specified with `EXCEPT` clause.
- If `QUERIES CUSTOM [EXCEPT .. [,..]]` modifier is specified, all custom protocols are stopped, unless specified with `EXCEPT` clause.
-### SYSTEM START LISTEN
+## SYSTEM START LISTEN
Allows new connections to be established on the specified protocols.
@@ -473,3 +473,47 @@ However, if the server on the specified port and protocol was not stopped using
```sql
SYSTEM START LISTEN [ON CLUSTER cluster_name] [QUERIES ALL | QUERIES DEFAULT | QUERIES CUSTOM | TCP | TCP WITH PROXY | TCP SECURE | HTTP | HTTPS | MYSQL | GRPC | POSTGRESQL | PROMETHEUS | CUSTOM 'protocol']
```
+
+## Managing Refreshable Materialized Views {#refreshable-materialized-views}
+
+Commands to control background tasks performed by [Refreshable Materialized Views](../../sql-reference/statements/create/view.md#refreshable-materialized-view)
+
+Keep an eye on [`system.view_refreshes`](../../operations/system-tables/view_refreshes.md) while using them.
+
+### SYSTEM REFRESH VIEW
+
+Trigger an immediate out-of-schedule refresh of a given view.
+
+```sql
+SYSTEM REFRESH VIEW [db.]name
+```
+
+### SYSTEM STOP VIEW, SYSTEM STOP VIEWS
+
+Disable periodic refreshing of the given view or all refreshable views. If a refresh is in progress, cancel it too.
+
+```sql
+SYSTEM STOP VIEW [db.]name
+```
+```sql
+SYSTEM STOP VIEWS
+```
+
+### SYSTEM START VIEW, SYSTEM START VIEWS
+
+Enable periodic refreshing for the given view or all refreshable views. No immediate refresh is triggered.
+
+```sql
+SYSTEM START VIEW [db.]name
+```
+```sql
+SYSTEM START VIEWS
+```
+
+### SYSTEM CANCEL VIEW
+
+If there's a refresh in progress for the given view, interrupt and cancel it. Otherwise do nothing.
+
+```sql
+SYSTEM CANCEL VIEW [db.]name
+```
diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h
index 45d427a7c55..463be6a3aea 100644
--- a/src/Access/Common/AccessType.h
+++ b/src/Access/Common/AccessType.h
@@ -82,7 +82,8 @@ enum class AccessType
\
M(ALTER_VIEW_REFRESH, "ALTER LIVE VIEW REFRESH, REFRESH VIEW", VIEW, ALTER_VIEW) \
M(ALTER_VIEW_MODIFY_QUERY, "ALTER TABLE MODIFY QUERY", VIEW, ALTER_VIEW) \
- M(ALTER_VIEW, "", GROUP, ALTER) /* allows to execute ALTER VIEW REFRESH, ALTER VIEW MODIFY QUERY;
+ M(ALTER_VIEW_MODIFY_REFRESH, "ALTER TABLE MODIFY QUERY", VIEW, ALTER_VIEW) \
+ M(ALTER_VIEW, "", GROUP, ALTER) /* allows to execute ALTER VIEW REFRESH, ALTER VIEW MODIFY QUERY, ALTER VIEW MODIFY REFRESH;
implicitly enabled by the grant ALTER_TABLE */\
\
M(ALTER, "", GROUP, ALL) /* allows to execute ALTER {TABLE|LIVE VIEW} */\
@@ -177,6 +178,7 @@ enum class AccessType
M(SYSTEM_MOVES, "SYSTEM STOP MOVES, SYSTEM START MOVES, STOP MOVES, START MOVES", TABLE, SYSTEM) \
M(SYSTEM_PULLING_REPLICATION_LOG, "SYSTEM STOP PULLING REPLICATION LOG, SYSTEM START PULLING REPLICATION LOG", TABLE, SYSTEM) \
M(SYSTEM_CLEANUP, "SYSTEM STOP CLEANUP, SYSTEM START CLEANUP", TABLE, SYSTEM) \
+ M(SYSTEM_VIEWS, "SYSTEM REFRESH VIEW, SYSTEM START VIEWS, SYSTEM STOP VIEWS, SYSTEM START VIEW, SYSTEM STOP VIEW, SYSTEM CANCEL VIEW, REFRESH VIEW, START VIEWS, STOP VIEWS, START VIEW, STOP VIEW, CANCEL VIEW", VIEW, SYSTEM) \
M(SYSTEM_DISTRIBUTED_SENDS, "SYSTEM STOP DISTRIBUTED SENDS, SYSTEM START DISTRIBUTED SENDS, STOP DISTRIBUTED SENDS, START DISTRIBUTED SENDS", TABLE, SYSTEM_SENDS) \
M(SYSTEM_REPLICATED_SENDS, "SYSTEM STOP REPLICATED SENDS, SYSTEM START REPLICATED SENDS, STOP REPLICATED SENDS, START REPLICATED SENDS", TABLE, SYSTEM_SENDS) \
M(SYSTEM_SENDS, "SYSTEM STOP SENDS, SYSTEM START SENDS, STOP SENDS, START SENDS", GROUP, SYSTEM) \
diff --git a/src/Access/tests/gtest_access_rights_ops.cpp b/src/Access/tests/gtest_access_rights_ops.cpp
index b5a15513a89..a7594503992 100644
--- a/src/Access/tests/gtest_access_rights_ops.cpp
+++ b/src/Access/tests/gtest_access_rights_ops.cpp
@@ -51,7 +51,7 @@ TEST(AccessRights, Union)
"CREATE DICTIONARY, DROP DATABASE, DROP TABLE, DROP VIEW, DROP DICTIONARY, UNDROP TABLE, "
"TRUNCATE, OPTIMIZE, BACKUP, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, "
"SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, "
- "SYSTEM MOVES, SYSTEM PULLING REPLICATION LOG, SYSTEM CLEANUP, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, "
+ "SYSTEM MOVES, SYSTEM PULLING REPLICATION LOG, SYSTEM CLEANUP, SYSTEM VIEWS, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, "
"SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, "
"SYSTEM RESTORE REPLICA, SYSTEM WAIT LOADING PARTS, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*, GRANT NAMED COLLECTION ADMIN ON db1");
}
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 6063c701708..86cb9acd056 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -226,6 +226,7 @@ add_object_library(clickhouse_storages_statistics Storages/Statistics)
add_object_library(clickhouse_storages_liveview Storages/LiveView)
add_object_library(clickhouse_storages_windowview Storages/WindowView)
add_object_library(clickhouse_storages_s3queue Storages/S3Queue)
+add_object_library(clickhouse_storages_materializedview Storages/MaterializedView)
add_object_library(clickhouse_client Client)
add_object_library(clickhouse_bridge BridgeHelper)
add_object_library(clickhouse_server Server)
diff --git a/src/Common/CalendarTimeInterval.cpp b/src/Common/CalendarTimeInterval.cpp
new file mode 100644
index 00000000000..b218e1d3c7c
--- /dev/null
+++ b/src/Common/CalendarTimeInterval.cpp
@@ -0,0 +1,144 @@
+#include
+
+#include
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+ extern const int BAD_ARGUMENTS;
+}
+
+CalendarTimeInterval::CalendarTimeInterval(const CalendarTimeInterval::Intervals & intervals)
+{
+ for (auto [kind, val] : intervals)
+ {
+ switch (kind.kind)
+ {
+ case IntervalKind::Nanosecond:
+ case IntervalKind::Microsecond:
+ case IntervalKind::Millisecond:
+ throw Exception(ErrorCodes::BAD_ARGUMENTS, "Sub-second intervals are not supported here");
+
+ case IntervalKind::Second:
+ case IntervalKind::Minute:
+ case IntervalKind::Hour:
+ case IntervalKind::Day:
+ case IntervalKind::Week:
+ seconds += val * kind.toAvgSeconds();
+ break;
+
+ case IntervalKind::Month:
+ months += val;
+ break;
+ case IntervalKind::Quarter:
+ months += val * 3;
+ break;
+ case IntervalKind::Year:
+ months += val * 12;
+ break;
+ }
+ }
+}
+
+CalendarTimeInterval::Intervals CalendarTimeInterval::toIntervals() const
+{
+ Intervals res;
+ auto greedy = [&](UInt64 x, std::initializer_list> kinds)
+ {
+ for (auto [kind, count] : kinds)
+ {
+ UInt64 k = x / count;
+ if (k == 0)
+ continue;
+ x -= k * count;
+ res.emplace_back(kind, k);
+ }
+ chassert(x == 0);
+ };
+ greedy(months, {{IntervalKind::Year, 12}, {IntervalKind::Month, 1}});
+ greedy(seconds, {{IntervalKind::Week, 3600*24*7}, {IntervalKind::Day, 3600*24}, {IntervalKind::Hour, 3600}, {IntervalKind::Minute, 60}, {IntervalKind::Second, 1}});
+ return res;
+}
+
+UInt64 CalendarTimeInterval::minSeconds() const
+{
+ return 3600*24 * (months/12 * 365 + months%12 * 28) + seconds;
+}
+
+UInt64 CalendarTimeInterval::maxSeconds() const
+{
+ return 3600*24 * (months/12 * 366 + months%12 * 31) + seconds;
+}
+
+void CalendarTimeInterval::assertSingleUnit() const
+{
+ if (seconds && months)
+ throw Exception(ErrorCodes::BAD_ARGUMENTS, "Interval shouldn't contain both calendar units and clock units (e.g. months and days)");
+}
+
+void CalendarTimeInterval::assertPositive() const
+{
+ if (!seconds && !months)
+ throw Exception(ErrorCodes::BAD_ARGUMENTS, "Interval must be positive");
+}
+
+/// Number of whole months between 1970-01-01 and `t`.
+static Int64 toAbsoluteMonth(std::chrono::system_clock::time_point t)
+{
+ std::chrono::year_month_day ymd(std::chrono::floor(t));
+ return (Int64(int(ymd.year())) - 1970) * 12 + Int64(unsigned(ymd.month()) - 1);
+}
+
+static std::chrono::sys_seconds startOfAbsoluteMonth(Int64 absolute_month)
+{
+ Int64 year = absolute_month >= 0 ? absolute_month/12 : -((-absolute_month+11)/12);
+ Int64 month = absolute_month - year*12;
+ chassert(month >= 0 && month < 12);
+ std::chrono::year_month_day ymd(
+ std::chrono::year(int(year + 1970)),
+ std::chrono::month(unsigned(month + 1)),
+ std::chrono::day(1));
+ return std::chrono::sys_days(ymd);
+}
+
+std::chrono::sys_seconds CalendarTimeInterval::advance(std::chrono::system_clock::time_point tp) const
+{
+ auto t = std::chrono::sys_seconds(std::chrono::floor(tp));
+ if (months)
+ {
+ auto m = toAbsoluteMonth(t);
+ auto s = t - startOfAbsoluteMonth(m);
+ t = startOfAbsoluteMonth(m + Int64(months)) + s;
+ }
+ return t + std::chrono::seconds(Int64(seconds));
+}
+
+std::chrono::sys_seconds CalendarTimeInterval::floor(std::chrono::system_clock::time_point tp) const
+{
+ assertSingleUnit();
+ assertPositive();
+
+ if (months)
+ return startOfAbsoluteMonth(toAbsoluteMonth(tp) / months * months);
+ else
+ {
+ constexpr std::chrono::seconds epoch(-3600*24*3);
+ auto t = std::chrono::sys_seconds(std::chrono::floor(tp));
+ /// We want to align with weeks, but 1970-01-01 is a Thursday, so align with 1969-12-29 instead.
+ return std::chrono::sys_seconds((t.time_since_epoch() - epoch) / seconds * seconds + epoch);
+ }
+}
+
+bool CalendarTimeInterval::operator==(const CalendarTimeInterval & rhs) const
+{
+ return std::tie(months, seconds) == std::tie(rhs.months, rhs.seconds);
+}
+
+bool CalendarTimeInterval::operator!=(const CalendarTimeInterval & rhs) const
+{
+ return !(*this == rhs);
+}
+
+}
diff --git a/src/Common/CalendarTimeInterval.h b/src/Common/CalendarTimeInterval.h
new file mode 100644
index 00000000000..d5acc6ee2f2
--- /dev/null
+++ b/src/Common/CalendarTimeInterval.h
@@ -0,0 +1,63 @@
+#pragma once
+
+#include
+#include
+
+namespace DB
+{
+
+/// Represents a duration of calendar time, e.g.:
+/// * 2 weeks + 5 minutes + and 21 seconds (aka 605121 seconds),
+/// * 1 (calendar) month - not equivalent to any number of seconds!
+/// * 3 years + 2 weeks (aka 36 months + 604800 seconds).
+///
+/// Be careful with calendar arithmetic: it's missing many familiar properties of numbers.
+/// E.g. x + y - y is not always equal to x (October 31 + 1 month - 1 month = November 1).
+struct CalendarTimeInterval
+{
+ UInt64 seconds = 0;
+ UInt64 months = 0;
+
+ using Intervals = std::vector>;
+
+ CalendarTimeInterval() = default;
+
+ /// Year, Quarter, Month are converted to months.
+ /// Week, Day, Hour, Minute, Second are converted to seconds.
+ /// Millisecond, Microsecond, Nanosecond throw exception.
+ explicit CalendarTimeInterval(const Intervals & intervals);
+
+ /// E.g. for {36 months, 604801 seconds} returns {3 years, 2 weeks, 1 second}.
+ Intervals toIntervals() const;
+
+ /// Approximate shortest and longest duration in seconds. E.g. a month is [28, 31] days.
+ UInt64 minSeconds() const;
+ UInt64 maxSeconds() const;
+
+ /// Checks that the interval has only months or only seconds, throws otherwise.
+ void assertSingleUnit() const;
+ void assertPositive() const;
+
+ /// Add this interval to the timestamp. First months, then seconds.
+ /// Gets weird near month boundaries: October 31 + 1 month = December 1.
+ std::chrono::sys_seconds advance(std::chrono::system_clock::time_point t) const;
+
+ /// Rounds the timestamp down to the nearest timestamp "aligned" with this interval.
+ /// The interval must satisfy assertSingleUnit() and assertPositive().
+ /// * For months, rounds to the start of a month whose abosolute index is divisible by `months`.
+ /// The month index is 0-based starting from January 1970.
+ /// E.g. if the interval is 1 month, rounds down to the start of the month.
+ /// * For seconds, rounds to a timestamp x such that (x - December 29 1969 (Monday)) is divisible
+ /// by this interval.
+ /// E.g. if the interval is 1 week, rounds down to the start of the week (Monday).
+ ///
+ /// Guarantees:
+ /// * advance(floor(x)) > x
+ /// * floor(advance(floor(x))) = advance(floor(x))
+ std::chrono::sys_seconds floor(std::chrono::system_clock::time_point t) const;
+
+ bool operator==(const CalendarTimeInterval & rhs) const;
+ bool operator!=(const CalendarTimeInterval & rhs) const;
+};
+
+}
diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp
index 38b14e4b0b4..2613e9ec116 100644
--- a/src/Common/CurrentMetrics.cpp
+++ b/src/Common/CurrentMetrics.cpp
@@ -253,6 +253,8 @@
M(MergeTreeAllRangesAnnouncementsSent, "The current number of announcement being sent in flight from the remote server to the initiator server about the set of data parts (for MergeTree tables). Measured on the remote server side.") \
M(CreatedTimersInQueryProfiler, "Number of Created thread local timers in QueryProfiler") \
M(ActiveTimersInQueryProfiler, "Number of Active thread local timers in QueryProfiler") \
+ M(RefreshableViews, "Number materialized views with periodic refreshing (REFRESH)") \
+ M(RefreshingViews, "Number of materialized views currently executing a refresh") \
#ifdef APPLY_FOR_EXTERNAL_METRICS
#define APPLY_FOR_METRICS(M) APPLY_FOR_BUILTIN_METRICS(M) APPLY_FOR_EXTERNAL_METRICS(M)
diff --git a/src/Common/IntervalKind.h b/src/Common/IntervalKind.h
index 6893286f196..0f45d0ac169 100644
--- a/src/Common/IntervalKind.h
+++ b/src/Common/IntervalKind.h
@@ -71,6 +71,8 @@ struct IntervalKind
/// Returns false if the conversion did not succeed.
/// For example, `IntervalKind::tryParseString('second', result)` returns `result` equals `IntervalKind::Kind::Second`.
static bool tryParseString(const std::string & kind, IntervalKind::Kind & result);
+
+ auto operator<=>(const IntervalKind & other) const { return kind <=> other.kind; }
};
/// NOLINTNEXTLINE
diff --git a/src/Core/BackgroundSchedulePool.cpp b/src/Core/BackgroundSchedulePool.cpp
index ec1ae047d05..fa892bc3c84 100644
--- a/src/Core/BackgroundSchedulePool.cpp
+++ b/src/Core/BackgroundSchedulePool.cpp
@@ -31,7 +31,7 @@ bool BackgroundSchedulePoolTaskInfo::schedule()
return true;
}
-bool BackgroundSchedulePoolTaskInfo::scheduleAfter(size_t milliseconds, bool overwrite)
+bool BackgroundSchedulePoolTaskInfo::scheduleAfter(size_t milliseconds, bool overwrite, bool only_if_scheduled)
{
std::lock_guard lock(schedule_mutex);
@@ -39,6 +39,8 @@ bool BackgroundSchedulePoolTaskInfo::scheduleAfter(size_t milliseconds, bool ove
return false;
if (delayed && !overwrite)
return false;
+ if (!delayed && only_if_scheduled)
+ return false;
pool.scheduleDelayedTask(shared_from_this(), milliseconds, lock);
return true;
diff --git a/src/Core/BackgroundSchedulePool.h b/src/Core/BackgroundSchedulePool.h
index e97b02e976f..eca93353283 100644
--- a/src/Core/BackgroundSchedulePool.h
+++ b/src/Core/BackgroundSchedulePool.h
@@ -106,8 +106,10 @@ public:
bool schedule();
/// Schedule for execution after specified delay.
- /// If overwrite is set then the task will be re-scheduled (if it was already scheduled, i.e. delayed == true).
- bool scheduleAfter(size_t milliseconds, bool overwrite = true);
+ /// If overwrite is set, and the task is already scheduled with a delay (delayed == true),
+ /// the task will be re-scheduled with the new delay.
+ /// If only_if_scheduled is set, don't do anything unless the task is already scheduled with a delay.
+ bool scheduleAfter(size_t milliseconds, bool overwrite = true, bool only_if_scheduled = false);
/// Further attempts to schedule become no-op. Will wait till the end of the current execution of the task.
void deactivate();
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index d96b1b9fc10..988c4f357e0 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -584,6 +584,8 @@ class IColumn;
M(Bool, enable_early_constant_folding, true, "Enable query optimization where we analyze function and subqueries results and rewrite query if there're constants there", 0) \
M(Bool, deduplicate_blocks_in_dependent_materialized_views, false, "Should deduplicate blocks for materialized views if the block is not a duplicate for the table. Use true to always deduplicate in dependent tables.", 0) \
M(Bool, materialized_views_ignore_errors, false, "Allows to ignore errors for MATERIALIZED VIEW, and deliver original block to the table regardless of MVs", 0) \
+ M(Bool, allow_experimental_refreshable_materialized_view, false, "Allow refreshable materialized views (CREATE MATERIALIZED VIEW REFRESH ...).", 0) \
+ M(Bool, stop_refreshable_materialized_views_on_startup, false, "On server startup, prevent scheduling of refreshable materialized views, as if with SYSTEM STOP VIEWS. You can manually start them with SYSTEM START VIEWS or SYSTEM START VIEW afterwards. Also applies to newly created views. Has no effect on non-refreshable materialized views.", 0) \
M(Bool, use_compact_format_in_distributed_parts_names, true, "Changes format of directories names for distributed table insert parts.", 0) \
M(Bool, validate_polygons, true, "Throw exception if polygon is invalid in function pointInPolygon (e.g. self-tangent, self-intersecting). If the setting is false, the function will accept invalid polygons but may silently return wrong result.", 0) \
M(UInt64, max_parser_depth, DBMS_DEFAULT_MAX_PARSER_DEPTH, "Maximum parser depth (recursion depth of recursive descend parser).", 0) \
diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp
index afc09fbe62a..bda48737621 100644
--- a/src/Databases/DatabasesCommon.cpp
+++ b/src/Databases/DatabasesCommon.cpp
@@ -65,6 +65,11 @@ void applyMetadataChangesToCreateQuery(const ASTPtr & query, const StorageInMemo
query->replace(ast_create_query.select, metadata.select.select_query);
}
+ if (metadata.refresh)
+ {
+ query->replace(ast_create_query.refresh_strategy, metadata.refresh);
+ }
+
/// MaterializedView, Dictionary are types of CREATE query without storage.
if (ast_create_query.storage)
{
diff --git a/src/Databases/TablesDependencyGraph.h b/src/Databases/TablesDependencyGraph.h
index e71d5ecc5fc..50be3bbf969 100644
--- a/src/Databases/TablesDependencyGraph.h
+++ b/src/Databases/TablesDependencyGraph.h
@@ -60,7 +60,7 @@ public:
/// Removes all dependencies of "table_id", returns those dependencies.
std::vector removeDependencies(const StorageID & table_id, bool remove_isolated_tables = false);
- /// Removes a table from the graph and removes all references to in from the graph (both from its dependencies and dependents).
+ /// Removes a table from the graph and removes all references to it from the graph (both from its dependencies and dependents).
bool removeTable(const StorageID & table_id);
/// Removes tables from the graph by a specified filter.
diff --git a/src/Interpreters/ActionLocksManager.cpp b/src/Interpreters/ActionLocksManager.cpp
index fb5ef4b98ae..65f13ebd66c 100644
--- a/src/Interpreters/ActionLocksManager.cpp
+++ b/src/Interpreters/ActionLocksManager.cpp
@@ -18,6 +18,7 @@ namespace ActionLocks
extern const StorageActionBlockType PartsMove = 7;
extern const StorageActionBlockType PullReplicationLog = 8;
extern const StorageActionBlockType Cleanup = 9;
+ extern const StorageActionBlockType ViewRefresh = 10;
}
diff --git a/src/Interpreters/AddDefaultDatabaseVisitor.h b/src/Interpreters/AddDefaultDatabaseVisitor.h
index 27639c4b813..b977a73d461 100644
--- a/src/Interpreters/AddDefaultDatabaseVisitor.h
+++ b/src/Interpreters/AddDefaultDatabaseVisitor.h
@@ -5,6 +5,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -87,6 +88,12 @@ public:
visit(child);
}
+ void visit(ASTRefreshStrategy & refresh) const
+ {
+ ASTPtr unused;
+ visit(refresh, unused);
+ }
+
private:
ContextPtr context;
@@ -229,6 +236,13 @@ private:
}
}
+ void visit(ASTRefreshStrategy & refresh, ASTPtr &) const
+ {
+ if (refresh.dependencies)
+ for (auto & table : refresh.dependencies->children)
+ tryVisit(table);
+ }
+
void visitChildren(IAST & ast) const
{
for (auto & child : ast.children)
diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index 25146ebc10d..fda22e4075e 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -95,6 +95,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -289,6 +290,7 @@ struct ContextSharedPart : boost::noncopyable
MergeList merge_list; /// The list of executable merge (for (Replicated)?MergeTree)
MovesList moves_list; /// The list of executing moves (for (Replicated)?MergeTree)
ReplicatedFetchList replicated_fetch_list;
+ RefreshSet refresh_set; /// The list of active refreshes (for MaterializedView)
ConfigurationPtr users_config TSA_GUARDED_BY(mutex); /// Config with the users, profiles and quotas sections.
InterserverIOHandler interserver_io_handler; /// Handler for interserver communication.
@@ -825,6 +827,8 @@ MovesList & Context::getMovesList() { return shared->moves_list; }
const MovesList & Context::getMovesList() const { return shared->moves_list; }
ReplicatedFetchList & Context::getReplicatedFetchList() { return shared->replicated_fetch_list; }
const ReplicatedFetchList & Context::getReplicatedFetchList() const { return shared->replicated_fetch_list; }
+RefreshSet & Context::getRefreshSet() { return shared->refresh_set; }
+const RefreshSet & Context::getRefreshSet() const { return shared->refresh_set; }
String Context::resolveDatabase(const String & database_name) const
{
diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h
index 39d2212ce80..b09eeb8ca2d 100644
--- a/src/Interpreters/Context.h
+++ b/src/Interpreters/Context.h
@@ -74,6 +74,7 @@ class BackgroundSchedulePool;
class MergeList;
class MovesList;
class ReplicatedFetchList;
+class RefreshSet;
class Cluster;
class Compiler;
class MarkCache;
@@ -922,6 +923,9 @@ public:
ReplicatedFetchList & getReplicatedFetchList();
const ReplicatedFetchList & getReplicatedFetchList() const;
+ RefreshSet & getRefreshSet();
+ const RefreshSet & getRefreshSet() const;
+
/// If the current session is expired at the time of the call, synchronously creates and returns a new session with the startNewSession() call.
/// If no ZooKeeper configured, throws an exception.
std::shared_ptr getZooKeeper() const;
diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp
index db93467c0a4..2a34932d950 100644
--- a/src/Interpreters/InterpreterAlterQuery.cpp
+++ b/src/Interpreters/InterpreterAlterQuery.cpp
@@ -460,6 +460,11 @@ AccessRightsElements InterpreterAlterQuery::getRequiredAccessForCommand(const AS
required_access.emplace_back(AccessType::ALTER_VIEW_MODIFY_QUERY, database, table);
break;
}
+ case ASTAlterCommand::MODIFY_REFRESH:
+ {
+ required_access.emplace_back(AccessType::ALTER_VIEW_MODIFY_REFRESH, database, table);
+ break;
+ }
case ASTAlterCommand::LIVE_VIEW_REFRESH:
{
required_access.emplace_back(AccessType::ALTER_VIEW_REFRESH, database, table);
diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index 1eadb325e95..68f52b11e05 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -1089,6 +1089,13 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data
"{} UUID specified, but engine of database {} is not Atomic", kind, create.getDatabase());
}
+ if (create.refresh_strategy && database->getEngineName() != "Atomic")
+ throw Exception(ErrorCodes::INCORRECT_QUERY,
+ "Refreshable materialized view requires Atomic database engine, but database {} has engine {}", create.getDatabase(), database->getEngineName());
+ /// TODO: Support Replicated databases, only with Shared/ReplicatedMergeTree.
+ /// Figure out how to make the refreshed data appear all at once on other
+ /// replicas; maybe a replicated SYSTEM SYNC REPLICA query before the rename?
+
/// The database doesn't support UUID so we'll ignore it. The UUID could be set here because of either
/// a) the initiator of `ON CLUSTER` query generated it to ensure the same UUIDs are used on different hosts; or
/// b) `RESTORE from backup` query generated it to ensure the same UUIDs are used on different hosts.
@@ -1210,6 +1217,16 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
visitor.visit(*create.select);
}
+ if (create.refresh_strategy)
+ {
+ if (!getContext()->getSettingsRef().allow_experimental_refreshable_materialized_view)
+ throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
+ "Refreshable materialized views are experimental. Enable allow_experimental_refreshable_materialized_view to use.");
+
+ AddDefaultDatabaseVisitor visitor(getContext(), current_database);
+ visitor.visit(*create.refresh_strategy);
+ }
+
if (create.columns_list)
{
AddDefaultDatabaseVisitor visitor(getContext(), current_database);
diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp
index fc040e2af04..db02ee13a4f 100644
--- a/src/Interpreters/InterpreterSystemQuery.cpp
+++ b/src/Interpreters/InterpreterSystemQuery.cpp
@@ -54,6 +54,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -108,6 +109,7 @@ namespace ActionLocks
extern const StorageActionBlockType PartsMove;
extern const StorageActionBlockType PullReplicationLog;
extern const StorageActionBlockType Cleanup;
+ extern const StorageActionBlockType ViewRefresh;
}
@@ -165,6 +167,8 @@ AccessType getRequiredAccessType(StorageActionBlockType action_type)
return AccessType::SYSTEM_PULLING_REPLICATION_LOG;
else if (action_type == ActionLocks::Cleanup)
return AccessType::SYSTEM_CLEANUP;
+ else if (action_type == ActionLocks::ViewRefresh)
+ return AccessType::SYSTEM_VIEWS;
else
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown action type: {}", std::to_string(action_type));
}
@@ -605,6 +609,23 @@ BlockIO InterpreterSystemQuery::execute()
case Type::START_CLEANUP:
startStopAction(ActionLocks::Cleanup, true);
break;
+ case Type::START_VIEW:
+ case Type::START_VIEWS:
+ startStopAction(ActionLocks::ViewRefresh, true);
+ break;
+ case Type::STOP_VIEW:
+ case Type::STOP_VIEWS:
+ startStopAction(ActionLocks::ViewRefresh, false);
+ break;
+ case Type::REFRESH_VIEW:
+ getRefreshTask()->run();
+ break;
+ case Type::CANCEL_VIEW:
+ getRefreshTask()->cancel();
+ break;
+ case Type::TEST_VIEW:
+ getRefreshTask()->setFakeTime(query.fake_time_for_view);
+ break;
case Type::DROP_REPLICA:
dropReplica(query);
break;
@@ -1092,6 +1113,17 @@ void InterpreterSystemQuery::flushDistributed(ASTSystemQuery &)
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "SYSTEM RESTART DISK is not supported");
}
+RefreshTaskHolder InterpreterSystemQuery::getRefreshTask()
+{
+ auto ctx = getContext();
+ ctx->checkAccess(AccessType::SYSTEM_VIEWS);
+ auto task = ctx->getRefreshSet().getTask(table_id);
+ if (!task)
+ throw Exception(
+ ErrorCodes::BAD_ARGUMENTS, "Refreshable view {} doesn't exist", table_id.getNameForLogs());
+ return task;
+}
+
AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster() const
{
@@ -1241,6 +1273,20 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster()
required_access.emplace_back(AccessType::SYSTEM_REPLICATION_QUEUES, query.getDatabase(), query.getTable());
break;
}
+ case Type::REFRESH_VIEW:
+ case Type::START_VIEW:
+ case Type::START_VIEWS:
+ case Type::STOP_VIEW:
+ case Type::STOP_VIEWS:
+ case Type::CANCEL_VIEW:
+ case Type::TEST_VIEW:
+ {
+ if (!query.table)
+ required_access.emplace_back(AccessType::SYSTEM_VIEWS);
+ else
+ required_access.emplace_back(AccessType::SYSTEM_VIEWS, query.getDatabase(), query.getTable());
+ break;
+ }
case Type::DROP_REPLICA:
case Type::DROP_DATABASE_REPLICA:
{
diff --git a/src/Interpreters/InterpreterSystemQuery.h b/src/Interpreters/InterpreterSystemQuery.h
index 462449623d0..89de7402b4d 100644
--- a/src/Interpreters/InterpreterSystemQuery.h
+++ b/src/Interpreters/InterpreterSystemQuery.h
@@ -3,6 +3,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -72,6 +73,8 @@ private:
void flushDistributed(ASTSystemQuery & query);
[[noreturn]] void restartDisk(String & name);
+ RefreshTaskHolder getRefreshTask();
+
AccessRightsElements getRequiredAccessForDDLOnCluster() const;
void startStopAction(StorageActionBlockType action_type, bool start);
};
diff --git a/src/Parsers/ASTAlterQuery.cpp b/src/Parsers/ASTAlterQuery.cpp
index ed9de6a46eb..84355817b2c 100644
--- a/src/Parsers/ASTAlterQuery.cpp
+++ b/src/Parsers/ASTAlterQuery.cpp
@@ -453,6 +453,12 @@ void ASTAlterCommand::formatImpl(const FormatSettings & settings, FormatState &
<< (settings.hilite ? hilite_none : "");
select->formatImpl(settings, state, frame);
}
+ else if (type == ASTAlterCommand::MODIFY_REFRESH)
+ {
+ settings.ostr << (settings.hilite ? hilite_keyword : "") << "MODIFY REFRESH " << settings.nl_or_ws
+ << (settings.hilite ? hilite_none : "");
+ refresh->formatImpl(settings, state, frame);
+ }
else if (type == ASTAlterCommand::LIVE_VIEW_REFRESH)
{
settings.ostr << (settings.hilite ? hilite_keyword : "") << "REFRESH " << (settings.hilite ? hilite_none : "");
diff --git a/src/Parsers/ASTAlterQuery.h b/src/Parsers/ASTAlterQuery.h
index 77c540aed33..0b115537a6d 100644
--- a/src/Parsers/ASTAlterQuery.h
+++ b/src/Parsers/ASTAlterQuery.h
@@ -40,6 +40,7 @@ public:
MODIFY_SETTING,
RESET_SETTING,
MODIFY_QUERY,
+ MODIFY_REFRESH,
REMOVE_TTL,
REMOVE_SAMPLE_BY,
@@ -166,6 +167,9 @@ public:
*/
ASTPtr values;
+ /// For MODIFY REFRESH
+ ASTPtr refresh;
+
bool detach = false; /// true for DETACH PARTITION
bool part = false; /// true for ATTACH PART, DROP DETACHED PART and MOVE
diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp
index 1562586bd93..9d5f0bcddbd 100644
--- a/src/Parsers/ASTCreateQuery.cpp
+++ b/src/Parsers/ASTCreateQuery.cpp
@@ -2,7 +2,6 @@
#include
#include
#include
-#include
#include
#include
#include
@@ -340,6 +339,12 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat
formatOnCluster(settings);
}
+ if (refresh_strategy)
+ {
+ settings.ostr << settings.nl_or_ws;
+ refresh_strategy->formatImpl(settings, state, frame);
+ }
+
if (to_table_id)
{
assert((is_materialized_view || is_window_view) && to_inner_uuid == UUIDHelpers::Nil);
diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h
index 28f5e05802b..49a0140625c 100644
--- a/src/Parsers/ASTCreateQuery.h
+++ b/src/Parsers/ASTCreateQuery.h
@@ -5,6 +5,7 @@
#include
#include
#include
+#include
#include
namespace DB
@@ -116,6 +117,7 @@ public:
ASTExpressionList * dictionary_attributes_list = nullptr; /// attributes of
ASTDictionary * dictionary = nullptr; /// dictionary definition (layout, primary key, etc.)
+ ASTRefreshStrategy * refresh_strategy = nullptr; // For CREATE MATERIALIZED VIEW ... REFRESH ...
std::optional live_view_periodic_refresh; /// For CREATE LIVE VIEW ... WITH [PERIODIC] REFRESH ...
bool is_watermark_strictly_ascending{false}; /// STRICTLY ASCENDING WATERMARK STRATEGY FOR WINDOW VIEW
diff --git a/src/Parsers/ASTRefreshStrategy.cpp b/src/Parsers/ASTRefreshStrategy.cpp
new file mode 100644
index 00000000000..2e0c6ee4638
--- /dev/null
+++ b/src/Parsers/ASTRefreshStrategy.cpp
@@ -0,0 +1,71 @@
+#include
+
+#include
+
+namespace DB
+{
+
+ASTPtr ASTRefreshStrategy::clone() const
+{
+ auto res = std::make_shared(*this);
+ res->children.clear();
+
+ if (period)
+ res->set(res->period, period->clone());
+ if (offset)
+ res->set(res->offset, offset->clone());
+ if (spread)
+ res->set(res->spread, spread->clone());
+ if (settings)
+ res->set(res->settings, settings->clone());
+ if (dependencies)
+ res->set(res->dependencies, dependencies->clone());
+ res->schedule_kind = schedule_kind;
+ return res;
+}
+
+void ASTRefreshStrategy::formatImpl(
+ const IAST::FormatSettings & f_settings, IAST::FormatState & state, IAST::FormatStateStacked frame) const
+{
+ frame.need_parens = false;
+
+ f_settings.ostr << (f_settings.hilite ? hilite_keyword : "") << "REFRESH " << (f_settings.hilite ? hilite_none : "");
+ using enum RefreshScheduleKind;
+ switch (schedule_kind)
+ {
+ case AFTER:
+ f_settings.ostr << "AFTER " << (f_settings.hilite ? hilite_none : "");
+ period->formatImpl(f_settings, state, frame);
+ break;
+ case EVERY:
+ f_settings.ostr << "EVERY " << (f_settings.hilite ? hilite_none : "");
+ period->formatImpl(f_settings, state, frame);
+ if (offset)
+ {
+ f_settings.ostr << (f_settings.hilite ? hilite_keyword : "") << " OFFSET " << (f_settings.hilite ? hilite_none : "");
+ offset->formatImpl(f_settings, state, frame);
+ }
+ break;
+ default:
+ f_settings.ostr << (f_settings.hilite ? hilite_none : "");
+ break;
+ }
+
+ if (spread)
+ {
+ f_settings.ostr << (f_settings.hilite ? hilite_keyword : "") << " RANDOMIZE FOR " << (f_settings.hilite ? hilite_none : "");
+ spread->formatImpl(f_settings, state, frame);
+ }
+ if (dependencies)
+ {
+ f_settings.ostr << (f_settings.hilite ? hilite_keyword : "") << " DEPENDS ON " << (f_settings.hilite ? hilite_none : "");
+ dependencies->formatImpl(f_settings, state, frame);
+ }
+ if (settings)
+ {
+ f_settings.ostr << (f_settings.hilite ? hilite_keyword : "") << " SETTINGS " << (f_settings.hilite ? hilite_none : "");
+ settings->formatImpl(f_settings, state, frame);
+ }
+}
+
+}
diff --git a/src/Parsers/ASTRefreshStrategy.h b/src/Parsers/ASTRefreshStrategy.h
new file mode 100644
index 00000000000..ca248b76b40
--- /dev/null
+++ b/src/Parsers/ASTRefreshStrategy.h
@@ -0,0 +1,35 @@
+#pragma once
+
+#include
+#include
+#include
+
+namespace DB
+{
+
+enum class RefreshScheduleKind : UInt8
+{
+ UNKNOWN = 0,
+ AFTER,
+ EVERY
+};
+
+/// Strategy for MATERIALIZED VIEW ... REFRESH ..
+class ASTRefreshStrategy : public IAST
+{
+public:
+ ASTSetQuery * settings = nullptr;
+ ASTExpressionList * dependencies = nullptr;
+ ASTTimeInterval * period = nullptr;
+ ASTTimeInterval * offset = nullptr;
+ ASTTimeInterval * spread = nullptr;
+ RefreshScheduleKind schedule_kind{RefreshScheduleKind::UNKNOWN};
+
+ String getID(char) const override { return "Refresh strategy definition"; }
+
+ ASTPtr clone() const override;
+
+ void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override;
+};
+
+}
diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h
index 8e6100fe7b4..fc26f5dee1c 100644
--- a/src/Parsers/ASTSystemQuery.h
+++ b/src/Parsers/ASTSystemQuery.h
@@ -90,6 +90,13 @@ public:
STOP_CLEANUP,
START_CLEANUP,
RESET_COVERAGE,
+ REFRESH_VIEW,
+ START_VIEW,
+ START_VIEWS,
+ STOP_VIEW,
+ STOP_VIEWS,
+ CANCEL_VIEW,
+ TEST_VIEW,
END
};
@@ -133,6 +140,10 @@ public:
ServerType server_type;
+ /// For SYSTEM TEST VIEW (SET FAKE TIME