materialize TTL after its modifying

src/Core/Settings.h

@@ -422,6 +422,7 @@ struct Settings : public SettingsCollection<Settings>
     M(SettingBool, transform_null_in, false, "If enabled, NULL values will be matched with 'IN' operator as if they are considered equal.", 0) \
     M(SettingBool, allow_nondeterministic_mutations, false, "Allow non-deterministic functions in ALTER UPDATE/ALTER DELETE statements", 0) \
     M(SettingSeconds, lock_acquire_timeout, DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC, "How long locking request should wait before failing", 0) \
+    M(SettingBool, materialize_ttl_after_modify, true, "Apply TTL for old data, after ALTER MODIFY TTL query", 0) \
     \
     /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \
     \

src/Storages/AlterCommands.cpp

@@ -594,6 +594,27 @@ bool AlterCommand::isCommentAlter() const
     return false;
 }
 
+bool AlterCommand::isTTLAlter(const StorageInMemoryMetadata & metadata) const
+{
+    if (type == MODIFY_TTL)
+        return true;
+
+    if (!ttl || type != MODIFY_COLUMN)
+        return false;
+
+    bool ttl_changed = true;
+    for (const auto & [name, ttl_ast] : metadata.columns.getColumnTTLs())
+    {
+        if (name == column_name && queryToString(*ttl) == queryToString(*ttl_ast))
+        {
+            ttl_changed = false;
+            break;
+        }
+    }
+
+    return ttl_changed;
+}
+
 std::optional<MutationCommand> AlterCommand::tryConvertToMutationCommand(const StorageInMemoryMetadata & metadata) const
 {
     if (!isRequireMutationStage(metadata))
@@ -922,13 +943,35 @@ bool AlterCommands::isCommentAlter() const
     return std::all_of(begin(), end(), [](const AlterCommand & c) { return c.isCommentAlter(); });
 }
 
+static MutationCommand createMaterializeTTLCommand()
+{
+    MutationCommand command;
+    auto ast = std::make_shared<ASTAlterCommand>();
+    ast->type = ASTAlterCommand::MATERIALIZE_TTL;
+    command.type = MutationCommand::MATERIALIZE_TTL;
+    command.ast = std::move(ast);
+    return command;
+}
 
-MutationCommands AlterCommands::getMutationCommands(const StorageInMemoryMetadata & metadata) const
+MutationCommands AlterCommands::getMutationCommands(const StorageInMemoryMetadata & metadata, bool materialize_ttl) const
 {
     MutationCommands result;
     for (const auto & alter_cmd : *this)
         if (auto mutation_cmd = alter_cmd.tryConvertToMutationCommand(metadata); mutation_cmd)
             result.push_back(*mutation_cmd);
+
+    if (materialize_ttl)
+    {
+        for (const auto & alter_cmd : *this)
+        {
+            if (alter_cmd.isTTLAlter(metadata))
+            {
+                result.push_back(createMaterializeTTLCommand());
+                break;
+            }
+        }
+    }
+
     return result;
 }
 

src/Storages/AlterCommands.h

@@ -118,6 +118,9 @@ struct AlterCommand
     /// Checks that only comment changed by alter
     bool isCommentAlter() const;
 
+    /// Checks that any TTL changed by alter
+    bool isTTLAlter(const StorageInMemoryMetadata & metadata) const;
+
     /// If possible, convert alter command to mutation command. In other case
     /// return empty optional. Some storages may execute mutations after
     /// metadata changes.
@@ -162,7 +165,7 @@ public:
     /// Return mutation commands which some storages may execute as part of
     /// alter. If alter can be performed is pure metadata update, than result is
     /// empty.
-    MutationCommands getMutationCommands(const StorageInMemoryMetadata & metadata) const;
+    MutationCommands getMutationCommands(const StorageInMemoryMetadata & metadata, bool materialize_ttl) const;
 };
 
 }

src/Storages/StorageMergeTree.cpp

@@ -220,7 +220,7 @@ void StorageMergeTree::alter(
     auto table_id = getStorageID();
 
     StorageInMemoryMetadata metadata = getInMemoryMetadata();
-    auto maybe_mutation_commands = commands.getMutationCommands(metadata);
+    auto maybe_mutation_commands = commands.getMutationCommands(metadata, context.getSettingsRef().materialize_ttl_after_modify);
     commands.apply(metadata);
 
     /// This alter can be performed at metadata level only

src/Storages/StorageReplicatedMergeTree.cpp

@@ -3388,7 +3388,7 @@ void StorageReplicatedMergeTree::alter(
         alter_entry->alter_version = new_metadata_version;
         alter_entry->create_time = time(nullptr);
 
-        auto maybe_mutation_commands = params.getMutationCommands(current_metadata);
+        auto maybe_mutation_commands = params.getMutationCommands(current_metadata, query_context.getSettingsRef().materialize_ttl_after_modify);
         alter_entry->have_mutation = !maybe_mutation_commands.empty();
 
         ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", alter_entry->toString(), zkutil::CreateMode::PersistentSequential));

tests/queries/0_stateless/01070_materialize_ttl.reference

@@ -1,8 +1,16 @@
+2000-10-10	1
+2000-10-10	2
+2100-10-10	3
+2100-10-10	4
 2100-10-10	3
 2100-10-10	4
 1	a
 3	c
 1	a
+2	b
+3	c
+4	d
+1	a
 2	
 3	c
 4	

tests/queries/0_stateless/01070_materialize_ttl.sql

@@ -6,9 +6,14 @@ insert into ttl values (toDateTime('2000-10-10 00:00:00'), 2);
 insert into ttl values (toDateTime('2100-10-10 00:00:00'), 3);
 insert into ttl values (toDateTime('2100-10-10 00:00:00'), 4);
 
+set materialize_ttl_after_modify = 0;
+
 alter table ttl materialize ttl; -- { serverError 80 }
 
 alter table ttl modify ttl d + interval 1 day;
+-- TTL should not be applied
+select * from ttl order by a;
+
 alter table ttl materialize ttl settings mutations_sync=2;
 select * from ttl order by a;
 
@@ -31,6 +36,9 @@ create table ttl (i Int, s String) engine = MergeTree order by i;
 insert into ttl values (1, 'a') (2, 'b') (3, 'c') (4, 'd');
 
 alter table ttl modify column s String ttl i % 2 = 0 ? today() - 10 : toDate('2100-01-01');
+-- TTL should not be applied
+select * from ttl order by i;
+
 alter table ttl materialize ttl settings mutations_sync=2;
 select * from ttl order by i;
 

tests/queries/0_stateless/01070_modify_ttl.reference

@@ -0,0 +1,32 @@
+2100-10-10	3
+2100-10-10	4
+=============
+1	a
+3	c
+=============
+=============
+1	a
+2	
+3	c
+4	
+=============
+1	
+2	
+3	
+4	
+=============
+1	a
+2	b
+4	d
+=============
+1	
+2	
+4	d
+=============
+1	a	
+2	b	bb
+3		cc
+4	d	
+1
+=============
+0

tests/queries/0_stateless/01070_modify_ttl.sql

@@ -0,0 +1,74 @@
+drop table if exists ttl;
+
+create table ttl (d Date, a Int) engine = MergeTree order by a partition by toDayOfMonth(d);
+insert into ttl values (toDateTime('2000-10-10 00:00:00'), 1);
+insert into ttl values (toDateTime('2000-10-10 00:00:00'), 2);
+insert into ttl values (toDateTime('2100-10-10 00:00:00'), 3);
+insert into ttl values (toDateTime('2100-10-10 00:00:00'), 4);
+
+set mutations_sync = 2;
+
+alter table ttl modify ttl d + interval 1 day;
+select * from ttl order by a;
+select '=============';
+
+drop table if exists ttl;
+
+create table ttl (i Int, s String) engine = MergeTree order by i;
+insert into ttl values (1, 'a') (2, 'b') (3, 'c') (4, 'd');
+
+alter table ttl modify ttl i % 2 = 0 ? today() - 10 : toDate('2100-01-01');
+select * from ttl order by i;
+select '=============';
+
+alter table ttl modify ttl toDate('2000-01-01');
+select * from ttl order by i;
+select '=============';
+
+drop table if exists ttl;
+
+create table ttl (i Int, s String) engine = MergeTree order by i;
+insert into ttl values (1, 'a') (2, 'b') (3, 'c') (4, 'd');
+
+alter table ttl modify column s String ttl i % 2 = 0 ? today() - 10 : toDate('2100-01-01');
+select * from ttl order by i;
+select '=============';
+
+alter table ttl modify column s String ttl toDate('2000-01-01');
+select * from ttl order by i;
+select '=============';
+
+drop table if exists ttl;
+
+create table ttl (d Date, i Int, s String) engine = MergeTree order by i;
+insert into ttl values (toDate('2000-01-02'), 1, 'a') (toDate('2000-01-03'), 2, 'b') (toDate('2080-01-01'), 3, 'c') (toDate('2080-01-03'), 4, 'd');
+
+alter table ttl modify ttl i % 3 = 0 ? today() - 10 : toDate('2100-01-01');
+select i, s from ttl order by i;
+select '=============';
+
+alter table ttl modify column s String ttl d + interval 1 month;
+select i, s from ttl order by i;
+select '=============';
+
+drop table if exists ttl;
+
+create table ttl (i Int, s String, t String) engine = MergeTree order by i;
+insert into ttl values (1, 'a', 'aa') (2, 'b', 'bb') (3, 'c', 'cc') (4, 'd', 'dd');
+
+alter table ttl modify column s String ttl i % 3 = 0 ? today() - 10 : toDate('2100-01-01'),
+                modify column t String ttl i % 3 = 1 ? today() - 10 : toDate('2100-01-01');
+
+select i, s, t from ttl order by i;
+-- MATERIALIZE TTL ran only once
+select count() from system.mutations where table = 'ttl' and is_done;
+select '=============';
+
+drop table if exists ttl;
+
+-- Nothing changed, don't run mutation
+create table ttl (i Int, s String ttl toDate('2000-01-02')) engine = MergeTree order by i;
+alter table ttl modify column s String ttl toDate('2000-01-02');
+select count() from system.mutations where table = 'ttl' and is_done;
+
+drop table if exists ttl;