ALTER <materialized view name> MODIFY QUERY <select_query>

Trying to resurrect https://github.com/ClickHouse/ClickHouse/pull/7533.
I'd like to get this PR in if we have an agreement on syntax and general
direction, after that I'll rebase actual alter functionality from above mentioned PR.

dbms/src/Core/Settings.h

@@ -388,6 +388,7 @@ struct Settings : public SettingsCollection<Settings>
     M(SettingBool, optimize_trivial_count_query, true, "Process trivial 'SELECT count() FROM table' query from metadata.", 0) \
     M(SettingUInt64, mutations_sync, 0, "Wait for synchronous execution of ALTER TABLE UPDATE/DELETE queries (mutations). 0 - execute asynchronously. 1 - wait current server. 2 - wait all replicas if they exist.", 0) \
     M(SettingBool, optimize_if_chain_to_miltiif, false, "Replace if(cond1, then1, if(cond2, ...)) chains to multiIf. Currently it's not beneficial for numeric types.", 0) \
+    M(SettingBool, allow_experimental_alter_materialized_view_structure, false, "Allow atomic alter on Materialized views. Work in progress.", 0) \
     \
     /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \
     \

dbms/src/Databases/DatabaseOrdinary.cpp

@@ -237,7 +237,7 @@ void DatabaseOrdinary::alterTable(
     ParserCreateQuery parser;
     ASTPtr ast = parseQuery(parser, statement.data(), statement.data() + statement.size(), "in file " + table_metadata_path, 0);
 
-    const auto & ast_create_query = ast->as<ASTCreateQuery &>();
+    auto & ast_create_query = ast->as<ASTCreateQuery &>();
 
     ASTPtr new_columns = InterpreterCreateQuery::formatColumns(metadata.columns);
     ASTPtr new_indices = InterpreterCreateQuery::formatIndices(metadata.indices);
@@ -247,6 +247,11 @@ void DatabaseOrdinary::alterTable(
     ast_create_query.columns_list->setOrReplace(ast_create_query.columns_list->indices, new_indices);
     ast_create_query.columns_list->setOrReplace(ast_create_query.columns_list->constraints, new_constraints);
 
+    if (metadata.select)
+    {
+        ast->replace(ast_create_query.select, metadata.select);
+    }
+
     ASTStorage & storage_ast = *ast_create_query.storage;
     /// ORDER BY may change, but cannot appear, it's required construction
     if (metadata.order_by_ast && storage_ast.order_by)

dbms/src/Parsers/ASTAlterQuery.cpp

@@ -266,6 +266,11 @@ void ASTAlterCommand::formatImpl(
         settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "MODIFY SETTING " << (settings.hilite ? hilite_none : "");
         settings_changes->formatImpl(settings, state, frame);
     }
+    else if (type == ASTAlterCommand::MODIFY_QUERY)
+    {
+        settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "MODIFY QUERY " << settings.nl_or_ws << (settings.hilite ? hilite_none : "");
+        select->formatImpl(settings, state, frame);
+    }
     else if (type == ASTAlterCommand::LIVE_VIEW_REFRESH)
     {
         settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "REFRESH " << (settings.hilite ? hilite_none : "");

dbms/src/Parsers/ASTAlterQuery.h

@@ -32,6 +32,7 @@ public:
         MODIFY_ORDER_BY,
         MODIFY_TTL,
         MODIFY_SETTING,
+        MODIFY_QUERY,
 
         ADD_INDEX,
         DROP_INDEX,
@@ -113,6 +114,9 @@ public:
     /// FOR MODIFY_SETTING
     ASTPtr settings_changes;
 
+    /// For MODIFY_QUERY
+    ASTPtr select;
+
     /** In ALTER CHANNEL, ADD, DROP, SUSPEND, RESUME, REFRESH, MODIFY queries, the list of live views is stored here
      */
     ASTPtr values;

dbms/src/Parsers/ASTCreateQuery.h

@@ -4,6 +4,7 @@
 #include <Parsers/ASTQueryWithOnCluster.h>
 #include <Parsers/ASTDictionary.h>
 #include <Parsers/ASTDictionaryAttributeDeclaration.h>
+#include <Parsers/ASTSelectWithUnionQuery.h>
 
 
 namespace DB
@@ -48,8 +49,6 @@ public:
 };
 
 
-class ASTSelectWithUnionQuery;
-
 /// CREATE TABLE or ATTACH TABLE query
 class ASTCreateQuery : public ASTQueryWithTableAndOutput, public ASTQueryWithOnCluster
 {

dbms/src/Parsers/ParserAlterQuery.cpp

@@ -5,6 +5,7 @@
 #include <Parsers/ExpressionListParsers.h>
 #include <Parsers/ParserCreateQuery.h>
 #include <Parsers/ParserPartition.h>
+#include <Parsers/ParserSelectWithUnionQuery.h>
 #include <Parsers/ParserSetQuery.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTIndexDeclaration.h>
@@ -30,6 +31,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
     ParserKeyword s_modify_order_by("MODIFY ORDER BY");
     ParserKeyword s_modify_ttl("MODIFY TTL");
     ParserKeyword s_modify_setting("MODIFY SETTING");
+    ParserKeyword s_modify_query("MODIFY QUERY");
 
     ParserKeyword s_add_index("ADD INDEX");
     ParserKeyword s_drop_index("DROP INDEX");
@@ -88,6 +90,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
         /* allow_empty = */ false);
     ParserSetQuery parser_settings(true);
     ParserNameList values_p;
+    ParserSelectWithUnionQuery select_p;
     ParserTTLExpressionList parser_ttl_list;
 
     if (is_live_view)
@@ -461,6 +464,12 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected
                 return false;
             command->type = ASTAlterCommand::MODIFY_SETTING;
         }
+        else if (s_modify_query.ignore(pos, expected))
+        {
+            if (!select_p.parse(pos, command->select, expected))
+                return false;
+            command->type = ASTAlterCommand::MODIFY_QUERY;
+        }
         else
             return false;
 

dbms/src/Storages/AlterCommands.cpp

@@ -209,6 +209,13 @@ std::optional<AlterCommand> AlterCommand::parse(const ASTAlterCommand * command_
         command.settings_changes = command_ast->settings_changes->as<ASTSetQuery &>().changes;
         return command;
     }
+    else if (command_ast->type == ASTAlterCommand::MODIFY_QUERY)
+    {
+        AlterCommand command;
+        command.type = AlterCommand::MODIFY_QUERY;
+        command.select = command_ast->select;
+        return command;
+    }
     else
         return {};
 }
@@ -389,6 +396,10 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata) const
     {
         metadata.ttl_for_table_ast = ttl;
     }
+    else if (type == MODIFY_QUERY)
+    {
+        metadata.select = select;
+    }
     else if (type == MODIFY_SETTING)
     {
         auto & settings_from_storage = metadata.settings_ast->as<ASTSetQuery &>().changes;
@@ -467,6 +478,8 @@ String alterTypeToString(const AlterCommand::Type type)
         return "MODIFY TTL";
     case AlterCommand::Type::MODIFY_SETTING:
         return "MODIFY SETTING";
+    case AlterCommand::Type::MODIFY_QUERY:
+        return "MODIFY QUERY";
     }
     __builtin_unreachable();
 }

dbms/src/Storages/AlterCommands.h

@@ -31,6 +31,7 @@ struct AlterCommand
         DROP_CONSTRAINT,
         MODIFY_TTL,
         MODIFY_SETTING,
+        MODIFY_QUERY,
     };
 
     Type type;
@@ -86,6 +87,9 @@ struct AlterCommand
     /// For MODIFY SETTING
     SettingsChanges settings_changes;
 
+    /// For MODIFY_QUERY
+    ASTPtr select = nullptr;
+
     static std::optional<AlterCommand> parse(const ASTAlterCommand * command);
 
     void apply(StorageInMemoryMetadata & metadata) const;

dbms/src/Storages/StorageInMemoryMetadata.h

@@ -33,6 +33,8 @@ struct StorageInMemoryMetadata
     ASTPtr sample_by_ast = nullptr;
     /// SETTINGS expression. Supported for MergeTree, Buffer and Kafka.
     ASTPtr settings_ast = nullptr;
+    /// SELECT QUERY. Supported for MaterializedView only.
+    ASTPtr select = nullptr;
 };
 
 }

dbms/src/Storages/StorageMaterializedView.cpp

@@ -14,6 +14,7 @@
 #include <DataStreams/IBlockInputStream.h>
 #include <DataStreams/IBlockOutputStream.h>
 
+#include <Storages/AlterCommands.h>
 #include <Storages/StorageFactory.h>
 #include <Storages/ReadInOrderOptimizer.h>
 
@@ -112,6 +113,7 @@ StorageMaterializedView::StorageMaterializedView(
     if (query.select->list_of_selects->children.size() != 1)
         throw Exception("UNION is not supported for MATERIALIZED VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_MATERIALIZED_VIEW);
 
+    select = query.select->clone();
     inner_query = query.select->list_of_selects->children.at(0);
 
     auto & select_query = inner_query->as<ASTSelectQuery &>();
@@ -159,6 +161,17 @@ bool StorageMaterializedView::hasColumn(const String & column_name) const
     return getTargetTable()->hasColumn(column_name);
 }
 
+StorageInMemoryMetadata StorageMaterializedView::getInMemoryMetadata() const
+{
+    return
+        {
+            .columns = getColumns(),
+            .indices = getIndices(),
+            .constraints = getConstraints(),
+            .select = getSelectQuery(),
+        };
+}
+
 QueryProcessingStage::Enum StorageMaterializedView::getQueryProcessingStage(const Context & context) const
 {
     return getTargetTable()->getQueryProcessingStage(context);
@@ -239,6 +252,38 @@ bool StorageMaterializedView::optimize(const ASTPtr & query, const ASTPtr & part
     return getTargetTable()->optimize(query, partition, final, deduplicate, context);
 }
 
+void StorageMaterializedView::alter(
+    const AlterCommands & params,
+    const Context & context,
+    TableStructureWriteLockHolder & table_lock_holder)
+{
+    lockStructureExclusively(table_lock_holder, context.getCurrentQueryId());
+    auto table_id = getStorageID();
+    StorageInMemoryMetadata metadata = getInMemoryMetadata();
+    params.apply(metadata);
+    context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata);
+    setColumns(std::move(metadata.columns));
+}
+
+
+void StorageMaterializedView::checkAlterIsPossible(const AlterCommands & commands, const Settings & settings)
+{
+    if (settings.allow_experimental_alter_materialized_view_structure)
+    {
+        throw Exception("work in progress", ErrorCodes::NOT_IMPLEMENTED);
+    }
+    else
+    {
+        for (const auto & command : commands)
+        {
+            if (!command.isCommentAlter())
+                throw Exception(
+                    "Alter of type '" + alterTypeToString(command.type) + "' is not supported by storage " + getName(),
+                    ErrorCodes::NOT_IMPLEMENTED);
+        }
+    }
+}
+
 void StorageMaterializedView::alterPartition(const ASTPtr & query, const PartitionCommands &commands, const Context &context)
 {
     checkStatementCanBeForwarded();

dbms/src/Storages/StorageMaterializedView.h

@@ -3,7 +3,10 @@
 #include <ext/shared_ptr_helper.h>
 
 #include <Parsers/IAST_fwd.h>
+#include <Parsers/ASTSelectWithUnionQuery.h>
+
 #include <Storages/IStorage.h>
+#include <Storages/StorageInMemoryMetadata.h>
 
 
 namespace DB
@@ -15,11 +18,14 @@ class StorageMaterializedView : public ext::shared_ptr_helper<StorageMaterialize
 public:
     std::string getName() const override { return "MaterializedView"; }
 
+    ASTPtr getSelectQuery() const { return select->clone(); }
     ASTPtr getInnerQuery() const { return inner_query->clone(); }
 
     NameAndTypePair getColumn(const String & column_name) const override;
     bool hasColumn(const String & column_name) const override;
 
+    StorageInMemoryMetadata getInMemoryMetadata() const override;
+
     bool supportsSampling() const override { return getTargetTable()->supportsSampling(); }
     bool supportsPrewhere() const override { return getTargetTable()->supportsPrewhere(); }
     bool supportsFinal() const override { return getTargetTable()->supportsFinal(); }
@@ -37,6 +43,10 @@ public:
 
     bool optimize(const ASTPtr & query, const ASTPtr & partition, bool final, bool deduplicate, const Context & context) override;
 
+    void alter(const AlterCommands & params, const Context & context, TableStructureWriteLockHolder & table_lock_holder) override;
+
+    void checkAlterIsPossible(const AlterCommands & commands, const Settings & settings) override;
+
     void alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & context) override;
 
     void mutate(const MutationCommands & commands, const Context & context) override;
@@ -71,7 +81,9 @@ private:
     /// Will be initialized in constructor
     StorageID target_table_id = StorageID::createEmpty();
 
+    ASTPtr select;
     ASTPtr inner_query;
+
     Context & global_context;
     bool has_inner_table = false;
 

dbms/tests/queries/0_stateless/01019_alter_materialized_view_query.reference

@@ -0,0 +1,13 @@
+1
+1
+2
+2
+3
+3
+1	0
+1	0
+2	0
+2	0
+3	0
+3	0
+42	0

dbms/tests/queries/0_stateless/01019_alter_materialized_view_query.sql

@@ -0,0 +1,37 @@
+-- Just testing syntax for now.
+
+DROP TABLE IF EXISTS src;
+DROP TABLE IF EXISTS dest;
+DROP TABLE IF EXISTS pipe;
+
+CREATE TABLE src(v UInt64) ENGINE = Null;
+CREATE TABLE dest(v UInt64) Engine = MergeTree() ORDER BY v;
+
+CREATE MATERIALIZED VIEW pipe TO dest AS
+SELECT v FROM src;
+
+INSERT INTO src VALUES (1), (2), (3);
+
+SET allow_experimental_alter_materialized_view_structure = 1;
+
+-- Live alter which changes query logic and adds an extra column.
+ALTER TABLE pipe
+    MODIFY QUERY
+    SELECT
+        v * 2 as v,
+        1 as v2
+    FROM src; -- { serverError 48 }
+
+INSERT INTO src VALUES (1), (2), (3);
+
+SELECT * FROM dest ORDER BY v;
+
+ALTER TABLE dest
+    ADD COLUMN v2 UInt64;
+
+INSERT INTO src VALUES (42);
+SELECT * FROM dest ORDER BY v;
+
+DROP TABLE src;
+DROP TABLE dest;
+DROP TABLE pipe;