diff --git a/src/Core/Settings.h b/src/Core/Settings.h index ff151e24a99..f33a8a4da35 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -422,6 +422,7 @@ struct Settings : public SettingsCollection M(SettingBool, transform_null_in, false, "If enabled, NULL values will be matched with 'IN' operator as if they are considered equal.", 0) \ M(SettingBool, allow_nondeterministic_mutations, false, "Allow non-deterministic functions in ALTER UPDATE/ALTER DELETE statements", 0) \ M(SettingSeconds, lock_acquire_timeout, DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC, "How long locking request should wait before failing", 0) \ + M(SettingBool, materialize_ttl_after_modify, true, "Apply TTL for old data, after ALTER MODIFY TTL query", 0) \ \ /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \ \ diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 67bd88d10a8..e34a39d8f46 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -594,6 +594,27 @@ bool AlterCommand::isCommentAlter() const return false; } +bool AlterCommand::isTTLAlter(const StorageInMemoryMetadata & metadata) const +{ + if (type == MODIFY_TTL) + return true; + + if (!ttl || type != MODIFY_COLUMN) + return false; + + bool ttl_changed = true; + for (const auto & [name, ttl_ast] : metadata.columns.getColumnTTLs()) + { + if (name == column_name && queryToString(*ttl) == queryToString(*ttl_ast)) + { + ttl_changed = false; + break; + } + } + + return ttl_changed; +} + std::optional AlterCommand::tryConvertToMutationCommand(const StorageInMemoryMetadata & metadata) const { if (!isRequireMutationStage(metadata)) @@ -922,13 +943,35 @@ bool AlterCommands::isCommentAlter() const return std::all_of(begin(), end(), [](const AlterCommand & c) { return c.isCommentAlter(); }); } +static MutationCommand createMaterializeTTLCommand() +{ + MutationCommand command; + auto ast = std::make_shared(); + ast->type = ASTAlterCommand::MATERIALIZE_TTL; + command.type = MutationCommand::MATERIALIZE_TTL; + command.ast = std::move(ast); + return command; +} -MutationCommands AlterCommands::getMutationCommands(const StorageInMemoryMetadata & metadata) const +MutationCommands AlterCommands::getMutationCommands(const StorageInMemoryMetadata & metadata, bool materialize_ttl) const { MutationCommands result; for (const auto & alter_cmd : *this) if (auto mutation_cmd = alter_cmd.tryConvertToMutationCommand(metadata); mutation_cmd) result.push_back(*mutation_cmd); + + if (materialize_ttl) + { + for (const auto & alter_cmd : *this) + { + if (alter_cmd.isTTLAlter(metadata)) + { + result.push_back(createMaterializeTTLCommand()); + break; + } + } + } + return result; } diff --git a/src/Storages/AlterCommands.h b/src/Storages/AlterCommands.h index c1c913dad73..d52a4ff2ed5 100644 --- a/src/Storages/AlterCommands.h +++ b/src/Storages/AlterCommands.h @@ -118,6 +118,9 @@ struct AlterCommand /// Checks that only comment changed by alter bool isCommentAlter() const; + /// Checks that any TTL changed by alter + bool isTTLAlter(const StorageInMemoryMetadata & metadata) const; + /// If possible, convert alter command to mutation command. In other case /// return empty optional. Some storages may execute mutations after /// metadata changes. @@ -162,7 +165,7 @@ public: /// Return mutation commands which some storages may execute as part of /// alter. If alter can be performed is pure metadata update, than result is /// empty. - MutationCommands getMutationCommands(const StorageInMemoryMetadata & metadata) const; + MutationCommands getMutationCommands(const StorageInMemoryMetadata & metadata, bool materialize_ttl) const; }; } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 68d468233a8..e78438210ec 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -220,7 +220,7 @@ void StorageMergeTree::alter( auto table_id = getStorageID(); StorageInMemoryMetadata metadata = getInMemoryMetadata(); - auto maybe_mutation_commands = commands.getMutationCommands(metadata); + auto maybe_mutation_commands = commands.getMutationCommands(metadata, context.getSettingsRef().materialize_ttl_after_modify); commands.apply(metadata); /// This alter can be performed at metadata level only diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 1317b05d9fe..add21251acc 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3388,7 +3388,7 @@ void StorageReplicatedMergeTree::alter( alter_entry->alter_version = new_metadata_version; alter_entry->create_time = time(nullptr); - auto maybe_mutation_commands = params.getMutationCommands(current_metadata); + auto maybe_mutation_commands = params.getMutationCommands(current_metadata, query_context.getSettingsRef().materialize_ttl_after_modify); alter_entry->have_mutation = !maybe_mutation_commands.empty(); ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", alter_entry->toString(), zkutil::CreateMode::PersistentSequential)); diff --git a/tests/queries/0_stateless/01070_materialize_ttl.reference b/tests/queries/0_stateless/01070_materialize_ttl.reference index b4a9947a521..af1b3a4459b 100644 --- a/tests/queries/0_stateless/01070_materialize_ttl.reference +++ b/tests/queries/0_stateless/01070_materialize_ttl.reference @@ -1,8 +1,16 @@ +2000-10-10 1 +2000-10-10 2 +2100-10-10 3 +2100-10-10 4 2100-10-10 3 2100-10-10 4 1 a 3 c 1 a +2 b +3 c +4 d +1 a 2 3 c 4 diff --git a/tests/queries/0_stateless/01070_materialize_ttl.sql b/tests/queries/0_stateless/01070_materialize_ttl.sql index 6696fbc980a..2521ae35edf 100755 --- a/tests/queries/0_stateless/01070_materialize_ttl.sql +++ b/tests/queries/0_stateless/01070_materialize_ttl.sql @@ -6,9 +6,14 @@ insert into ttl values (toDateTime('2000-10-10 00:00:00'), 2); insert into ttl values (toDateTime('2100-10-10 00:00:00'), 3); insert into ttl values (toDateTime('2100-10-10 00:00:00'), 4); +set materialize_ttl_after_modify = 0; + alter table ttl materialize ttl; -- { serverError 80 } alter table ttl modify ttl d + interval 1 day; +-- TTL should not be applied +select * from ttl order by a; + alter table ttl materialize ttl settings mutations_sync=2; select * from ttl order by a; @@ -31,6 +36,9 @@ create table ttl (i Int, s String) engine = MergeTree order by i; insert into ttl values (1, 'a') (2, 'b') (3, 'c') (4, 'd'); alter table ttl modify column s String ttl i % 2 = 0 ? today() - 10 : toDate('2100-01-01'); +-- TTL should not be applied +select * from ttl order by i; + alter table ttl materialize ttl settings mutations_sync=2; select * from ttl order by i; diff --git a/tests/queries/0_stateless/01070_modify_ttl.reference b/tests/queries/0_stateless/01070_modify_ttl.reference new file mode 100644 index 00000000000..d64c1a4edc2 --- /dev/null +++ b/tests/queries/0_stateless/01070_modify_ttl.reference @@ -0,0 +1,32 @@ +2100-10-10 3 +2100-10-10 4 +============= +1 a +3 c +============= +============= +1 a +2 +3 c +4 +============= +1 +2 +3 +4 +============= +1 a +2 b +4 d +============= +1 +2 +4 d +============= +1 a +2 b bb +3 cc +4 d +1 +============= +0 diff --git a/tests/queries/0_stateless/01070_modify_ttl.sql b/tests/queries/0_stateless/01070_modify_ttl.sql new file mode 100644 index 00000000000..4e842948afe --- /dev/null +++ b/tests/queries/0_stateless/01070_modify_ttl.sql @@ -0,0 +1,74 @@ +drop table if exists ttl; + +create table ttl (d Date, a Int) engine = MergeTree order by a partition by toDayOfMonth(d); +insert into ttl values (toDateTime('2000-10-10 00:00:00'), 1); +insert into ttl values (toDateTime('2000-10-10 00:00:00'), 2); +insert into ttl values (toDateTime('2100-10-10 00:00:00'), 3); +insert into ttl values (toDateTime('2100-10-10 00:00:00'), 4); + +set mutations_sync = 2; + +alter table ttl modify ttl d + interval 1 day; +select * from ttl order by a; +select '============='; + +drop table if exists ttl; + +create table ttl (i Int, s String) engine = MergeTree order by i; +insert into ttl values (1, 'a') (2, 'b') (3, 'c') (4, 'd'); + +alter table ttl modify ttl i % 2 = 0 ? today() - 10 : toDate('2100-01-01'); +select * from ttl order by i; +select '============='; + +alter table ttl modify ttl toDate('2000-01-01'); +select * from ttl order by i; +select '============='; + +drop table if exists ttl; + +create table ttl (i Int, s String) engine = MergeTree order by i; +insert into ttl values (1, 'a') (2, 'b') (3, 'c') (4, 'd'); + +alter table ttl modify column s String ttl i % 2 = 0 ? today() - 10 : toDate('2100-01-01'); +select * from ttl order by i; +select '============='; + +alter table ttl modify column s String ttl toDate('2000-01-01'); +select * from ttl order by i; +select '============='; + +drop table if exists ttl; + +create table ttl (d Date, i Int, s String) engine = MergeTree order by i; +insert into ttl values (toDate('2000-01-02'), 1, 'a') (toDate('2000-01-03'), 2, 'b') (toDate('2080-01-01'), 3, 'c') (toDate('2080-01-03'), 4, 'd'); + +alter table ttl modify ttl i % 3 = 0 ? today() - 10 : toDate('2100-01-01'); +select i, s from ttl order by i; +select '============='; + +alter table ttl modify column s String ttl d + interval 1 month; +select i, s from ttl order by i; +select '============='; + +drop table if exists ttl; + +create table ttl (i Int, s String, t String) engine = MergeTree order by i; +insert into ttl values (1, 'a', 'aa') (2, 'b', 'bb') (3, 'c', 'cc') (4, 'd', 'dd'); + +alter table ttl modify column s String ttl i % 3 = 0 ? today() - 10 : toDate('2100-01-01'), + modify column t String ttl i % 3 = 1 ? today() - 10 : toDate('2100-01-01'); + +select i, s, t from ttl order by i; +-- MATERIALIZE TTL ran only once +select count() from system.mutations where table = 'ttl' and is_done; +select '============='; + +drop table if exists ttl; + +-- Nothing changed, don't run mutation +create table ttl (i Int, s String ttl toDate('2000-01-02')) engine = MergeTree order by i; +alter table ttl modify column s String ttl toDate('2000-01-02'); +select count() from system.mutations where table = 'ttl' and is_done; + +drop table if exists ttl;