materialize ttl recalculate only optional

tests/integration/test_ttl_replicated/test.py

@@ -161,7 +161,7 @@ def test_modify_ttl(started_cluster):
             '''
                 CREATE TABLE test_ttl(d DateTime, id UInt32)
                 ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_modify', '{replica}')
-                ORDER BY id SETTINGS merge_with_ttl_timeout=0;
+                ORDER BY id
             '''.format(replica=node.name))
 
     node1.query(
@@ -169,15 +169,12 @@ def test_modify_ttl(started_cluster):
     node2.query("SYSTEM SYNC REPLICA test_ttl", timeout=20)
 
     node1.query("ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 4 HOUR SETTINGS mutations_sync = 2")
-    time.sleep(6)  # TTL merges shall happen.
     assert node2.query("SELECT id FROM test_ttl") == "2\n3\n"
 
     node2.query("ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 2 HOUR SETTINGS mutations_sync = 2")
-    time.sleep(6)  # TTL merges shall happen.
     assert node1.query("SELECT id FROM test_ttl") == "3\n"
 
     node1.query("ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 30 MINUTE SETTINGS mutations_sync = 2")
-    time.sleep(6)  # TTL merges shall happen.
     assert node2.query("SELECT id FROM test_ttl") == ""
 
 
@@ -188,7 +185,7 @@ def test_modify_column_ttl(started_cluster):
             '''
                 CREATE TABLE test_ttl(d DateTime, id UInt32 DEFAULT 42)
                 ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_column', '{replica}')
-                ORDER BY d SETTINGS merge_with_ttl_timeout=0;
+                ORDER BY d
             '''.format(replica=node.name))
 
     node1.query(
@@ -196,15 +193,12 @@ def test_modify_column_ttl(started_cluster):
     node2.query("SYSTEM SYNC REPLICA test_ttl", timeout=20)
 
     node1.query("ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 4 HOUR SETTINGS mutations_sync = 2")
-    time.sleep(6)  # TTL merges shall happen.
     assert node2.query("SELECT id FROM test_ttl") == "42\n2\n3\n"
 
     node1.query("ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 2 HOUR SETTINGS mutations_sync = 2")
-    time.sleep(6)  # TTL merges shall happen.
     assert node1.query("SELECT id FROM test_ttl") == "42\n42\n3\n"
 
     node1.query("ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 30 MINUTE SETTINGS mutations_sync = 2")
-    time.sleep(6)  # TTL merges shall happen.
     assert node2.query("SELECT id FROM test_ttl") == "42\n42\n42\n"
 
 
@@ -304,9 +298,7 @@ def test_ttl_empty_parts(started_cluster):
             SETTINGS max_bytes_to_merge_at_min_space_in_pool = 1, max_bytes_to_merge_at_max_space_in_pool = 1,
                 cleanup_delay_period = 1, cleanup_delay_period_random_add = 0
         '''.format(replica=node.name))
-        node.query("SYSTEM STOP TTL MERGES test_ttl_empty_parts")
 
-    
     for i in range (1, 7):
         node1.query("INSERT INTO test_ttl_empty_parts SELECT '2{}00-01-0{}', number FROM numbers(1000)".format(i % 2, i))
 
@@ -316,21 +308,18 @@ def test_ttl_empty_parts(started_cluster):
 
     node1.query("ALTER TABLE test_ttl_empty_parts MODIFY TTL date")
 
-    assert node1.query("SELECT count() FROM test_ttl_empty_parts") == "6000\n"
+    assert node1.query("SELECT count() FROM test_ttl_empty_parts") == "3000\n"
 
     time.sleep(3) # Wait for cleanup thread
     assert node1.query("SELECT name FROM system.parts WHERE table = 'test_ttl_empty_parts' AND active ORDER BY name") == \
-        "all_0_0_0_6\nall_1_1_0_6\nall_2_2_0_6\nall_3_3_0_6\nall_4_4_0_6\nall_5_5_0_6\n"
+        "all_0_0_0_6\nall_2_2_0_6\nall_4_4_0_6\n"
 
     for node in [node1, node2]:
         node.query("ALTER TABLE test_ttl_empty_parts MODIFY SETTING max_bytes_to_merge_at_min_space_in_pool = 1000000000")
         node.query("ALTER TABLE test_ttl_empty_parts MODIFY SETTING max_bytes_to_merge_at_max_space_in_pool = 1000000000")
-        node.query("SYSTEM START TTL MERGES test_ttl_empty_parts")
 
     optimize_with_retry(node1, 'test_ttl_empty_parts')
-    assert node1.query("SELECT name FROM system.parts WHERE table = 'test_ttl_empty_parts' AND active ORDER BY name") == "all_0_5_1_6\n"
+    assert node1.query("SELECT name FROM system.parts WHERE table = 'test_ttl_empty_parts' AND active ORDER BY name") == "all_0_4_1_6\n"
-
-    assert node1.query("SELECT count() FROM test_ttl_empty_parts") == "3000\n"
 
     # Check that after removing empty parts mutations and merges works
     node1.query("INSERT INTO test_ttl_empty_parts SELECT '2100-01-20', number FROM numbers(1000)")

tests/queries/0_stateless/01070_materialize_ttl.reference

@@ -2,33 +2,15 @@
 2000-10-10	2
 2100-10-10	3
 2100-10-10	4
-2000-10-10	1
-2000-10-10	2
-2100-10-10	3
-2100-10-10	4
 2100-10-10	3
 2100-10-10	4
 1	a
-2	b
-3	c
-4	d
-1	a
-3	c
-1	a
 3	c
 1	a
 2	b
 3	c
 4	d
 1	a
-2	b
-3	c
-4	d
-1	a
-2	
-3	c
-4	
-1	a
 2	
 3	c
 4	
@@ -38,13 +20,6 @@
 4	
 1	a
 2	b
-3	c
-4	d
-1	a
-2	b
-4	d
-1	a
-2	b
 4	d
 1	
 2	

tests/queries/0_stateless/01070_materialize_ttl.sql

@@ -1,92 +1,62 @@
-set mutations_sync = 2;
-set materialize_ttl_after_modify = 0;
-
 drop table if exists ttl;
 
-create table ttl (d Date, a Int) engine = MergeTree order by a partition by toDayOfMonth(d)
+create table ttl (d Date, a Int) engine = MergeTree order by a partition by toDayOfMonth(d);
-SETTINGS max_number_of_merges_with_ttl_in_pool=0;
-
 insert into ttl values (toDateTime('2000-10-10 00:00:00'), 1);
 insert into ttl values (toDateTime('2000-10-10 00:00:00'), 2);
 insert into ttl values (toDateTime('2100-10-10 00:00:00'), 3);
 insert into ttl values (toDateTime('2100-10-10 00:00:00'), 4);
 
+set materialize_ttl_after_modify = 0;
+
 alter table ttl materialize ttl; -- { serverError 80 }
 
 alter table ttl modify ttl d + interval 1 day;
 -- TTL should not be applied
 select * from ttl order by a;
 
-alter table ttl materialize ttl;
+alter table ttl materialize ttl settings mutations_sync=2;
-select * from ttl order by a;
-
-optimize table ttl final;
 select * from ttl order by a;
 
 drop table if exists ttl;
 
-create table ttl (i Int, s String) engine = MergeTree order by i
+create table ttl (i Int, s String) engine = MergeTree order by i;
-SETTINGS max_number_of_merges_with_ttl_in_pool=0;
-
 insert into ttl values (1, 'a') (2, 'b') (3, 'c') (4, 'd');
 
 alter table ttl modify ttl i % 2 = 0 ? today() - 10 : toDate('2100-01-01');
-alter table ttl materialize ttl;
+alter table ttl materialize ttl settings mutations_sync=2;
-select * from ttl order by i;
-
-optimize table ttl final;
 select * from ttl order by i;
 
 alter table ttl modify ttl toDate('2000-01-01');
-alter table ttl materialize ttl;
+alter table ttl materialize ttl settings mutations_sync=2;
-select * from ttl order by i;
-
-optimize table ttl final;
 select * from ttl order by i;
 
 drop table if exists ttl;
 
-create table ttl (i Int, s String) engine = MergeTree order by i
+create table ttl (i Int, s String) engine = MergeTree order by i;
-SETTINGS max_number_of_merges_with_ttl_in_pool=0;
-
 insert into ttl values (1, 'a') (2, 'b') (3, 'c') (4, 'd');
 
 alter table ttl modify column s String ttl i % 2 = 0 ? today() - 10 : toDate('2100-01-01');
 -- TTL should not be applied
 select * from ttl order by i;
 
-alter table ttl materialize ttl;
+alter table ttl materialize ttl settings mutations_sync=2;
-select * from ttl order by i;
-
-optimize table ttl final;
 select * from ttl order by i;
 
 alter table ttl modify column s String ttl toDate('2000-01-01');
-alter table ttl materialize ttl;
+alter table ttl materialize ttl settings mutations_sync=2;
-select * from ttl order by i;
-
-optimize table ttl final;
 select * from ttl order by i;
 
 drop table if exists ttl;
 
-create table ttl (d Date, i Int, s String) engine = MergeTree order by i
+create table ttl (d Date, i Int, s String) engine = MergeTree order by i;
-SETTINGS max_number_of_merges_with_ttl_in_pool=0;
-
 insert into ttl values (toDate('2000-01-02'), 1, 'a') (toDate('2000-01-03'), 2, 'b') (toDate('2080-01-01'), 3, 'c') (toDate('2080-01-03'), 4, 'd');
 
 alter table ttl modify ttl i % 3 = 0 ? today() - 10 : toDate('2100-01-01');
-alter table ttl materialize ttl;
+alter table ttl materialize ttl settings mutations_sync=2;
-select i, s from ttl order by i;
-
-optimize table ttl final;
 select i, s from ttl order by i;
 
 alter table ttl modify column s String ttl d + interval 1 month;
-alter table ttl materialize ttl;
+alter table ttl materialize ttl settings mutations_sync=2;
-select i, s from ttl order by i;
-
-optimize table ttl final;
 select i, s from ttl order by i;
 
 drop table if exists ttl;

tests/queries/0_stateless/01070_modify_ttl.reference

@@ -1,33 +1,15 @@
-2000-10-10	1
-2000-10-10	2
-2100-10-10	3
-2100-10-10	4
 2100-10-10	3
 2100-10-10	4
 =============
 1	a
-2	b
-3	c
-4	d
-1	a
 3	c
 =============
-1	a
-3	c
 =============
 1	a
-2	b
-3	c
-4	d
-1	a
 2	
 3	c
 4	
 =============
-1	a
-2	
-3	c
-4	
 1	
 2	
 3	
@@ -35,23 +17,12 @@
 =============
 1	a
 2	b
-3	c
-4	d
-1	a
-2	b
 4	d
 =============
-1	a
-2	b
-4	d
 1	
 2	
 4	d
 =============
-1	a	aa
-2	b	bb
-3	c	cc
-4	d	dd
 1	a	
 2	b	bb
 3		cc

tests/queries/0_stateless/01070_modify_ttl.sql

@@ -1,98 +1,65 @@
 drop table if exists ttl;
-set mutations_sync = 2;
-
-create table ttl (d Date, a Int) engine = MergeTree order by a partition by toDayOfMonth(d)
-SETTINGS max_number_of_merges_with_ttl_in_pool=0;
 
+create table ttl (d Date, a Int) engine = MergeTree order by a partition by toDayOfMonth(d);
 insert into ttl values (toDateTime('2000-10-10 00:00:00'), 1);
 insert into ttl values (toDateTime('2000-10-10 00:00:00'), 2);
 insert into ttl values (toDateTime('2100-10-10 00:00:00'), 3);
 insert into ttl values (toDateTime('2100-10-10 00:00:00'), 4);
 
-alter table ttl modify ttl d + interval 1 day;
+set mutations_sync = 2;
-select * from ttl order by a;
 
-optimize table ttl final;
+alter table ttl modify ttl d + interval 1 day;
 select * from ttl order by a;
 select '=============';
 
 drop table if exists ttl;
 
-create table ttl (i Int, s String) engine = MergeTree order by i
+create table ttl (i Int, s String) engine = MergeTree order by i;
-SETTINGS max_number_of_merges_with_ttl_in_pool=0;
-
 insert into ttl values (1, 'a') (2, 'b') (3, 'c') (4, 'd');
 
 alter table ttl modify ttl i % 2 = 0 ? today() - 10 : toDate('2100-01-01');
 select * from ttl order by i;
-
-optimize table ttl final;
-select * from ttl order by i;
 select '=============';
 
 alter table ttl modify ttl toDate('2000-01-01');
 select * from ttl order by i;
-
-optimize table ttl final;
-select * from ttl order by i;
 select '=============';
 
 drop table if exists ttl;
 
-create table ttl (i Int, s String) engine = MergeTree order by i
+create table ttl (i Int, s String) engine = MergeTree order by i;
-SETTINGS max_number_of_merges_with_ttl_in_pool=0;
-
 insert into ttl values (1, 'a') (2, 'b') (3, 'c') (4, 'd');
 
 alter table ttl modify column s String ttl i % 2 = 0 ? today() - 10 : toDate('2100-01-01');
 select * from ttl order by i;
-
-optimize table ttl final;
-select * from ttl order by i;
 select '=============';
 
 alter table ttl modify column s String ttl toDate('2000-01-01');
 select * from ttl order by i;
-
-optimize table ttl final;
-select * from ttl order by i;
 select '=============';
 
 drop table if exists ttl;
 
-create table ttl (d Date, i Int, s String) engine = MergeTree order by i
+create table ttl (d Date, i Int, s String) engine = MergeTree order by i;
-SETTINGS max_number_of_merges_with_ttl_in_pool=0;
-
 insert into ttl values (toDate('2000-01-02'), 1, 'a') (toDate('2000-01-03'), 2, 'b') (toDate('2080-01-01'), 3, 'c') (toDate('2080-01-03'), 4, 'd');
 
 alter table ttl modify ttl i % 3 = 0 ? today() - 10 : toDate('2100-01-01');
 select i, s from ttl order by i;
-
-optimize table ttl final;
-select i, s from ttl order by i;
 select '=============';
 
 alter table ttl modify column s String ttl d + interval 1 month;
 select i, s from ttl order by i;
-
-optimize table ttl final;
-select i, s from ttl order by i;
 select '=============';
 
 drop table if exists ttl;
 
-create table ttl (i Int, s String, t String) engine = MergeTree order by i
+create table ttl (i Int, s String, t String) engine = MergeTree order by i;
-SETTINGS max_number_of_merges_with_ttl_in_pool=0;
-
 insert into ttl values (1, 'a', 'aa') (2, 'b', 'bb') (3, 'c', 'cc') (4, 'd', 'dd');
 
 alter table ttl modify column s String ttl i % 3 = 0 ? today() - 10 : toDate('2100-01-01'),
                 modify column t String ttl i % 3 = 1 ? today() - 10 : toDate('2100-01-01');
-select i, s, t from ttl order by i;
 
-optimize table ttl final;
 select i, s, t from ttl order by i;
-
 -- MATERIALIZE TTL ran only once
 select count() from system.mutations where table = 'ttl' and is_done;
 select '=============';
@@ -100,9 +67,7 @@ select '=============';
 drop table if exists ttl;
 
 -- Nothing changed, don't run mutation
-create table ttl (i Int, s String ttl toDate('2000-01-02')) engine = MergeTree order by i
+create table ttl (i Int, s String ttl toDate('2000-01-02')) engine = MergeTree order by i;
-SETTINGS max_number_of_merges_with_ttl_in_pool=0;
-
 alter table ttl modify column s String ttl toDate('2000-01-02');
 select count() from system.mutations where table = 'ttl' and is_done;
 

tests/queries/0_stateless/01070_mutations_with_dependencies.reference

@@ -3,20 +3,9 @@
 3	1	c
 4	1	d
 1	1	a
-2	1	b
-3	1	c
-4	1	d
-1	1	a
-2	0	b
-3	1	c
-4	0	d
-1	1	a
 3	1	c
 ===================
 4
-4
-0
 2
 2
-1	2	3	2000-01-01
 1	2	42	2000-01-01

tests/queries/0_stateless/01070_mutations_with_dependencies.sql

@@ -2,62 +2,43 @@ drop table if exists ttl;
 set mutations_sync = 2;
 
 -- check that ttl info was updated after mutation.
-create table ttl (i Int, a Int, s String) engine = MergeTree order by i
+create table ttl (i Int, a Int, s String) engine = MergeTree order by i;
-SETTINGS max_number_of_merges_with_ttl_in_pool=0;
-
 insert into ttl values (1, 1, 'a') (2, 1, 'b') (3, 1, 'c') (4, 1, 'd');
 
 alter table ttl modify ttl a % 2 = 0 ? today() - 10 : toDate('2100-01-01');
 alter table ttl materialize ttl;
-select * from ttl order by i;
 
-optimize table ttl final;
 select * from ttl order by i;
-
 alter table ttl update a = 0 where i % 2 = 0;
 select * from ttl order by i;
 
-optimize table ttl final;
-select * from ttl order by i;
-
 drop table ttl;
 
 select '===================';
 
 -- check that skip index is updated after column was modified by ttl.
 create table ttl (i Int, a Int, s String default 'b' ttl a % 2 = 0 ? today() - 10 : toDate('2100-01-01'),
-    index ind_s (s) type set(1) granularity 1) engine = MergeTree order by i
+    index ind_s (s) type set(1) granularity 1) engine = MergeTree order by i;
-    SETTINGS max_number_of_merges_with_ttl_in_pool=0;
-
 insert into ttl values (1, 1, 'a') (2, 1, 'a') (3, 1, 'a') (4, 1, 'a');
 
 select count() from ttl where s = 'a';
 
 alter table ttl update a = 0 where i % 2 = 0;
-select count() from ttl where s = 'a';
-select count() from ttl where s = 'b';
 
-optimize table ttl final;
 select count() from ttl where s = 'a';
 select count() from ttl where s = 'b';
 
 drop table ttl;
 
 -- check only that it doesn't throw exceptions.
-create table ttl (i Int, s String) engine = MergeTree order by i ttl toDate('2000-01-01') TO DISK 'default'
+create table ttl (i Int, s String) engine = MergeTree order by i ttl toDate('2000-01-01') TO DISK 'default';
-SETTINGS max_number_of_merges_with_ttl_in_pool=0;
-
 alter table ttl materialize ttl;
 drop table ttl;
 
 create table ttl (a Int, b Int, c Int default 42 ttl d, d Date, index ind (b * c) type minmax granularity 1)
-engine = MergeTree order by a SETTINGS max_number_of_merges_with_ttl_in_pool=0;
+engine = MergeTree order by a;
-
 insert into ttl values (1, 2, 3, '2100-01-01');
 alter table ttl update d = '2000-01-01' where 1;
 alter table ttl materialize ttl;
 select * from ttl;
-
-optimize table ttl final;
-select * from ttl;
 drop table ttl;

tests/queries/0_stateless/01378_alter_rename_with_ttl_zookeeper.reference

@@ -1,4 +1,3 @@
 9
 9
-9
 0

tests/queries/0_stateless/01378_alter_rename_with_ttl_zookeeper.sql

@@ -1,7 +1,4 @@
 DROP TABLE IF EXISTS table_rename_with_ttl;
-SET replication_alter_partitions_sync = 2;
-SET mutations_sync = 2;
-SET materialize_ttl_after_modify = 0;
 
 CREATE TABLE table_rename_with_ttl
 (
@@ -9,25 +6,20 @@ CREATE TABLE table_rename_with_ttl
   value1 String
 )
 ENGINE = ReplicatedMergeTree('/clickhouse/test/table_rename_with_ttl_01378', '1')
-ORDER BY tuple()
+ORDER BY tuple();
-SETTINGS max_number_of_merges_with_ttl_in_pool=0;
 
 INSERT INTO table_rename_with_ttl SELECT toDate('2018-10-01') + number % 3, toString(number) from numbers(9);
 
 SELECT count() FROM table_rename_with_ttl;
 
+SET materialize_ttl_after_modify = 0;
 ALTER TABLE table_rename_with_ttl MODIFY TTL date1 + INTERVAL 1 MONTH;
 
 SELECT count() FROM table_rename_with_ttl;
 
 ALTER TABLE table_rename_with_ttl RENAME COLUMN date1 TO renamed_date1;
 
-ALTER TABLE table_rename_with_ttl materialize TTL;
+ALTER TABLE table_rename_with_ttl materialize TTL settings mutations_sync=2;
-
-SELECT count() FROM table_rename_with_ttl;
-
-ALTER TABLE table_rename_with_ttl modify setting max_number_of_merges_with_ttl_in_pool=2;
-optimize table table_rename_with_ttl FINAL;
 
 SELECT count() FROM table_rename_with_ttl;
 

tests/queries/0_stateless/01560_ttl_remove_empty_parts.sql

@@ -10,9 +10,6 @@ SELECT count() FROM system.parts WHERE table = 'ttl_empty_parts' AND database =
 
 ALTER TABLE ttl_empty_parts MODIFY TTL d;
 
-SELECT sleep(3) format Null;
-SELECT sleep(3) format Null;
-
 -- To be sure, that task, which clears outdated parts executed.
 DETACH TABLE ttl_empty_parts;
 ATTACH TABLE ttl_empty_parts;