thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 07:01:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

materialize ttl recalculate only optional

Browse Source
This commit is contained in:
terrylin 2021-08-04 22:15:11 +08:00
parent ffe9eab1d0
commit 8a6455116a
10 changed files with 33 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
tests/integration/test_ttl_replicated/test.py
View File

@ -161,7 +161,7 @@ def test_modify_ttl(started_cluster):
'''
CREATE TABLE test_ttl(d DateTime, id UInt32)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_modify', '{replica}')
ORDER BY id SETTINGS merge_with_ttl_timeout=0;
ORDER BY id
'''.format(replica=node.name))
node1.query(
@ -169,15 +169,12 @@ def test_modify_ttl(started_cluster):
node2.query("SYSTEM SYNC REPLICA test_ttl", timeout=20)
node1.query("ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 4 HOUR SETTINGS mutations_sync = 2")
time.sleep(6) # TTL merges shall happen.
assert node2.query("SELECT id FROM test_ttl") == "2\n3\n"
node2.query("ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 2 HOUR SETTINGS mutations_sync = 2")
time.sleep(6) # TTL merges shall happen.
assert node1.query("SELECT id FROM test_ttl") == "3\n"
node1.query("ALTER TABLE test_ttl MODIFY TTL d + INTERVAL 30 MINUTE SETTINGS mutations_sync = 2")
time.sleep(6) # TTL merges shall happen.
assert node2.query("SELECT id FROM test_ttl") == ""
@ -188,7 +185,7 @@ def test_modify_column_ttl(started_cluster):
'''
CREATE TABLE test_ttl(d DateTime, id UInt32 DEFAULT 42)
ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/test_ttl_column', '{replica}')
ORDER BY d SETTINGS merge_with_ttl_timeout=0;
ORDER BY d
'''.format(replica=node.name))
node1.query(
@ -196,15 +193,12 @@ def test_modify_column_ttl(started_cluster):
node2.query("SYSTEM SYNC REPLICA test_ttl", timeout=20)
node1.query("ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 4 HOUR SETTINGS mutations_sync = 2")
time.sleep(6) # TTL merges shall happen.
assert node2.query("SELECT id FROM test_ttl") == "42\n2\n3\n"
node1.query("ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 2 HOUR SETTINGS mutations_sync = 2")
time.sleep(6) # TTL merges shall happen.
assert node1.query("SELECT id FROM test_ttl") == "42\n42\n3\n"
node1.query("ALTER TABLE test_ttl MODIFY COLUMN id UInt32 TTL d + INTERVAL 30 MINUTE SETTINGS mutations_sync = 2")
time.sleep(6) # TTL merges shall happen.
assert node2.query("SELECT id FROM test_ttl") == "42\n42\n42\n"
@ -304,9 +298,7 @@ def test_ttl_empty_parts(started_cluster):
SETTINGS max_bytes_to_merge_at_min_space_in_pool = 1, max_bytes_to_merge_at_max_space_in_pool = 1,
cleanup_delay_period = 1, cleanup_delay_period_random_add = 0
'''.format(replica=node.name))
node.query("SYSTEM STOP TTL MERGES test_ttl_empty_parts")
for i in range (1, 7):
node1.query("INSERT INTO test_ttl_empty_parts SELECT '2{}00-01-0{}', number FROM numbers(1000)".format(i % 2, i))
@ -316,21 +308,18 @@ def test_ttl_empty_parts(started_cluster):
node1.query("ALTER TABLE test_ttl_empty_parts MODIFY TTL date")
assert node1.query("SELECT count() FROM test_ttl_empty_parts") == "6000\n"
assert node1.query("SELECT count() FROM test_ttl_empty_parts") == "3000\n"
time.sleep(3) # Wait for cleanup thread
assert node1.query("SELECT name FROM system.parts WHERE table = 'test_ttl_empty_parts' AND active ORDER BY name") == \
"all_0_0_0_6\nall_1_1_0_6\nall_2_2_0_6\nall_3_3_0_6\nall_4_4_0_6\nall_5_5_0_6\n"
"all_0_0_0_6\nall_2_2_0_6\nall_4_4_0_6\n"
for node in [node1, node2]:
node.query("ALTER TABLE test_ttl_empty_parts MODIFY SETTING max_bytes_to_merge_at_min_space_in_pool = 1000000000")
node.query("ALTER TABLE test_ttl_empty_parts MODIFY SETTING max_bytes_to_merge_at_max_space_in_pool = 1000000000")
node.query("SYSTEM START TTL MERGES test_ttl_empty_parts")
optimize_with_retry(node1, 'test_ttl_empty_parts')
assert node1.query("SELECT name FROM system.parts WHERE table = 'test_ttl_empty_parts' AND active ORDER BY name") == "all_0_5_1_6\n"
assert node1.query("SELECT count() FROM test_ttl_empty_parts") == "3000\n"
assert node1.query("SELECT name FROM system.parts WHERE table = 'test_ttl_empty_parts' AND active ORDER BY name") == "all_0_4_1_6\n"
# Check that after removing empty parts mutations and merges works
node1.query("INSERT INTO test_ttl_empty_parts SELECT '2100-01-20', number FROM numbers(1000)")

25
tests/queries/0_stateless/01070_materialize_ttl.reference
View File

@ -2,33 +2,15 @@
2000-10-10 2
2100-10-10 3
2100-10-10 4
2000-10-10 1
2000-10-10 2
2100-10-10 3
2100-10-10 4
2100-10-10 3
2100-10-10 4
1 a
2 b
3 c
4 d
1 a
3 c
1 a
3 c
1 a
2 b
3 c
4 d
1 a
2 b
3 c
4 d
1 a
2
3 c
4
1 a
2
3 c
4
@ -38,13 +20,6 @@
4
1 a
2 b
3 c
4 d
1 a
2 b
4 d
1 a
2 b
4 d
1
2

56
tests/queries/0_stateless/01070_materialize_ttl.sql
View File

@ -1,92 +1,62 @@
set mutations_sync = 2;
set materialize_ttl_after_modify = 0;
drop table if exists ttl;
create table ttl (d Date, a Int) engine = MergeTree order by a partition by toDayOfMonth(d)
SETTINGS max_number_of_merges_with_ttl_in_pool=0;
create table ttl (d Date, a Int) engine = MergeTree order by a partition by toDayOfMonth(d);
insert into ttl values (toDateTime('2000-10-10 00:00:00'), 1);
insert into ttl values (toDateTime('2000-10-10 00:00:00'), 2);
insert into ttl values (toDateTime('2100-10-10 00:00:00'), 3);
insert into ttl values (toDateTime('2100-10-10 00:00:00'), 4);
set materialize_ttl_after_modify = 0;
alter table ttl materialize ttl; -- { serverError 80 }
alter table ttl modify ttl d + interval 1 day;
-- TTL should not be applied
select * from ttl order by a;
alter table ttl materialize ttl;
select * from ttl order by a;
optimize table ttl final;
alter table ttl materialize ttl settings mutations_sync=2;
select * from ttl order by a;
drop table if exists ttl;
create table ttl (i Int, s String) engine = MergeTree order by i
SETTINGS max_number_of_merges_with_ttl_in_pool=0;
create table ttl (i Int, s String) engine = MergeTree order by i;
insert into ttl values (1, 'a') (2, 'b') (3, 'c') (4, 'd');
alter table ttl modify ttl i % 2 = 0 ? today() - 10 : toDate('2100-01-01');
alter table ttl materialize ttl;
select * from ttl order by i;
optimize table ttl final;
alter table ttl materialize ttl settings mutations_sync=2;
select * from ttl order by i;
alter table ttl modify ttl toDate('2000-01-01');
alter table ttl materialize ttl;
select * from ttl order by i;
optimize table ttl final;
alter table ttl materialize ttl settings mutations_sync=2;
select * from ttl order by i;
drop table if exists ttl;
create table ttl (i Int, s String) engine = MergeTree order by i
SETTINGS max_number_of_merges_with_ttl_in_pool=0;
create table ttl (i Int, s String) engine = MergeTree order by i;
insert into ttl values (1, 'a') (2, 'b') (3, 'c') (4, 'd');
alter table ttl modify column s String ttl i % 2 = 0 ? today() - 10 : toDate('2100-01-01');
-- TTL should not be applied
select * from ttl order by i;
alter table ttl materialize ttl;
select * from ttl order by i;
optimize table ttl final;
alter table ttl materialize ttl settings mutations_sync=2;
select * from ttl order by i;
alter table ttl modify column s String ttl toDate('2000-01-01');
alter table ttl materialize ttl;
select * from ttl order by i;
optimize table ttl final;
alter table ttl materialize ttl settings mutations_sync=2;
select * from ttl order by i;
drop table if exists ttl;
create table ttl (d Date, i Int, s String) engine = MergeTree order by i
SETTINGS max_number_of_merges_with_ttl_in_pool=0;
create table ttl (d Date, i Int, s String) engine = MergeTree order by i;
insert into ttl values (toDate('2000-01-02'), 1, 'a') (toDate('2000-01-03'), 2, 'b') (toDate('2080-01-01'), 3, 'c') (toDate('2080-01-03'), 4, 'd');
alter table ttl modify ttl i % 3 = 0 ? today() - 10 : toDate('2100-01-01');
alter table ttl materialize ttl;
select i, s from ttl order by i;
optimize table ttl final;
alter table ttl materialize ttl settings mutations_sync=2;
select i, s from ttl order by i;
alter table ttl modify column s String ttl d + interval 1 month;
alter table ttl materialize ttl;
select i, s from ttl order by i;
optimize table ttl final;
alter table ttl materialize ttl settings mutations_sync=2;
select i, s from ttl order by i;
drop table if exists ttl;

29
tests/queries/0_stateless/01070_modify_ttl.reference
View File

@ -1,33 +1,15 @@
2000-10-10 1
2000-10-10 2
2100-10-10 3
2100-10-10 4
2100-10-10 3
2100-10-10 4
=============
1 a
2 b
3 c
4 d
1 a
3 c
=============
1 a
3 c
=============
1 a
2 b
3 c
4 d
1 a
2
3 c
4
=============
1 a
2
3 c
4
1
2
3
@ -35,23 +17,12 @@
=============
1 a
2 b
3 c
4 d
1 a
2 b
4 d
=============
1 a
2 b
4 d
1
2
4 d
=============
1 a aa
2 b bb
3 c cc
4 d dd
1 a
2 b bb
3 cc

51
tests/queries/0_stateless/01070_modify_ttl.sql
View File

@ -1,98 +1,65 @@
drop table if exists ttl;
set mutations_sync = 2;
create table ttl (d Date, a Int) engine = MergeTree order by a partition by toDayOfMonth(d)
SETTINGS max_number_of_merges_with_ttl_in_pool=0;
create table ttl (d Date, a Int) engine = MergeTree order by a partition by toDayOfMonth(d);
insert into ttl values (toDateTime('2000-10-10 00:00:00'), 1);
insert into ttl values (toDateTime('2000-10-10 00:00:00'), 2);
insert into ttl values (toDateTime('2100-10-10 00:00:00'), 3);
insert into ttl values (toDateTime('2100-10-10 00:00:00'), 4);
alter table ttl modify ttl d + interval 1 day;
select * from ttl order by a;
set mutations_sync = 2;
optimize table ttl final;
alter table ttl modify ttl d + interval 1 day;
select * from ttl order by a;
select '=============';
drop table if exists ttl;
create table ttl (i Int, s String) engine = MergeTree order by i
SETTINGS max_number_of_merges_with_ttl_in_pool=0;
create table ttl (i Int, s String) engine = MergeTree order by i;
insert into ttl values (1, 'a') (2, 'b') (3, 'c') (4, 'd');
alter table ttl modify ttl i % 2 = 0 ? today() - 10 : toDate('2100-01-01');
select * from ttl order by i;
optimize table ttl final;
select * from ttl order by i;
select '=============';
alter table ttl modify ttl toDate('2000-01-01');
select * from ttl order by i;
optimize table ttl final;
select * from ttl order by i;
select '=============';
drop table if exists ttl;
create table ttl (i Int, s String) engine = MergeTree order by i
SETTINGS max_number_of_merges_with_ttl_in_pool=0;
create table ttl (i Int, s String) engine = MergeTree order by i;
insert into ttl values (1, 'a') (2, 'b') (3, 'c') (4, 'd');
alter table ttl modify column s String ttl i % 2 = 0 ? today() - 10 : toDate('2100-01-01');
select * from ttl order by i;
optimize table ttl final;
select * from ttl order by i;
select '=============';
alter table ttl modify column s String ttl toDate('2000-01-01');
select * from ttl order by i;
optimize table ttl final;
select * from ttl order by i;
select '=============';
drop table if exists ttl;
create table ttl (d Date, i Int, s String) engine = MergeTree order by i
SETTINGS max_number_of_merges_with_ttl_in_pool=0;
create table ttl (d Date, i Int, s String) engine = MergeTree order by i;
insert into ttl values (toDate('2000-01-02'), 1, 'a') (toDate('2000-01-03'), 2, 'b') (toDate('2080-01-01'), 3, 'c') (toDate('2080-01-03'), 4, 'd');
alter table ttl modify ttl i % 3 = 0 ? today() - 10 : toDate('2100-01-01');
select i, s from ttl order by i;
optimize table ttl final;
select i, s from ttl order by i;
select '=============';
alter table ttl modify column s String ttl d + interval 1 month;
select i, s from ttl order by i;
optimize table ttl final;
select i, s from ttl order by i;
select '=============';
drop table if exists ttl;
create table ttl (i Int, s String, t String) engine = MergeTree order by i
SETTINGS max_number_of_merges_with_ttl_in_pool=0;
create table ttl (i Int, s String, t String) engine = MergeTree order by i;
insert into ttl values (1, 'a', 'aa') (2, 'b', 'bb') (3, 'c', 'cc') (4, 'd', 'dd');
alter table ttl modify column s String ttl i % 3 = 0 ? today() - 10 : toDate('2100-01-01'),
modify column t String ttl i % 3 = 1 ? today() - 10 : toDate('2100-01-01');
select i, s, t from ttl order by i;
optimize table ttl final;
select i, s, t from ttl order by i;
-- MATERIALIZE TTL ran only once
select count() from system.mutations where table = 'ttl' and is_done;
select '=============';
@ -100,9 +67,7 @@ select '=============';
drop table if exists ttl;
-- Nothing changed, don't run mutation
create table ttl (i Int, s String ttl toDate('2000-01-02')) engine = MergeTree order by i
SETTINGS max_number_of_merges_with_ttl_in_pool=0;
create table ttl (i Int, s String ttl toDate('2000-01-02')) engine = MergeTree order by i;
alter table ttl modify column s String ttl toDate('2000-01-02');
select count() from system.mutations where table = 'ttl' and is_done;

11
tests/queries/0_stateless/01070_mutations_with_dependencies.reference
View File

@ -3,20 +3,9 @@
3 1 c
4 1 d
1 1 a
2 1 b
3 1 c
4 1 d
1 1 a
2 0 b
3 1 c
4 0 d
1 1 a
3 1 c
===================
4
4
0
2
2
1 2 3 2000-01-01
1 2 42 2000-01-01

27
tests/queries/0_stateless/01070_mutations_with_dependencies.sql
View File

@ -2,62 +2,43 @@ drop table if exists ttl;
set mutations_sync = 2;
-- check that ttl info was updated after mutation.
create table ttl (i Int, a Int, s String) engine = MergeTree order by i
SETTINGS max_number_of_merges_with_ttl_in_pool=0;
create table ttl (i Int, a Int, s String) engine = MergeTree order by i;
insert into ttl values (1, 1, 'a') (2, 1, 'b') (3, 1, 'c') (4, 1, 'd');
alter table ttl modify ttl a % 2 = 0 ? today() - 10 : toDate('2100-01-01');
alter table ttl materialize ttl;
select * from ttl order by i;
optimize table ttl final;
select * from ttl order by i;
alter table ttl update a = 0 where i % 2 = 0;
select * from ttl order by i;
optimize table ttl final;
select * from ttl order by i;
drop table ttl;
select '===================';
-- check that skip index is updated after column was modified by ttl.
create table ttl (i Int, a Int, s String default 'b' ttl a % 2 = 0 ? today() - 10 : toDate('2100-01-01'),
index ind_s (s) type set(1) granularity 1) engine = MergeTree order by i
SETTINGS max_number_of_merges_with_ttl_in_pool=0;
index ind_s (s) type set(1) granularity 1) engine = MergeTree order by i;
insert into ttl values (1, 1, 'a') (2, 1, 'a') (3, 1, 'a') (4, 1, 'a');
select count() from ttl where s = 'a';
alter table ttl update a = 0 where i % 2 = 0;
select count() from ttl where s = 'a';
select count() from ttl where s = 'b';
optimize table ttl final;
select count() from ttl where s = 'a';
select count() from ttl where s = 'b';
drop table ttl;
-- check only that it doesn't throw exceptions.
create table ttl (i Int, s String) engine = MergeTree order by i ttl toDate('2000-01-01') TO DISK 'default'
SETTINGS max_number_of_merges_with_ttl_in_pool=0;
create table ttl (i Int, s String) engine = MergeTree order by i ttl toDate('2000-01-01') TO DISK 'default';
alter table ttl materialize ttl;
drop table ttl;
create table ttl (a Int, b Int, c Int default 42 ttl d, d Date, index ind (b * c) type minmax granularity 1)
engine = MergeTree order by a SETTINGS max_number_of_merges_with_ttl_in_pool=0;
engine = MergeTree order by a;
insert into ttl values (1, 2, 3, '2100-01-01');
alter table ttl update d = '2000-01-01' where 1;
alter table ttl materialize ttl;
select * from ttl;
optimize table ttl final;
select * from ttl;
drop table ttl;

1
tests/queries/0_stateless/01378_alter_rename_with_ttl_zookeeper.reference
View File

@ -1,4 +1,3 @@
9
9
9
0

14
tests/queries/0_stateless/01378_alter_rename_with_ttl_zookeeper.sql
View File

@ -1,7 +1,4 @@
DROP TABLE IF EXISTS table_rename_with_ttl;
SET replication_alter_partitions_sync = 2;
SET mutations_sync = 2;
SET materialize_ttl_after_modify = 0;
CREATE TABLE table_rename_with_ttl
(
@ -9,25 +6,20 @@ CREATE TABLE table_rename_with_ttl
value1 String
)
ENGINE = ReplicatedMergeTree('/clickhouse/test/table_rename_with_ttl_01378', '1')
ORDER BY tuple()
SETTINGS max_number_of_merges_with_ttl_in_pool=0;
ORDER BY tuple();
INSERT INTO table_rename_with_ttl SELECT toDate('2018-10-01') + number % 3, toString(number) from numbers(9);
SELECT count() FROM table_rename_with_ttl;
SET materialize_ttl_after_modify = 0;
ALTER TABLE table_rename_with_ttl MODIFY TTL date1 + INTERVAL 1 MONTH;
SELECT count() FROM table_rename_with_ttl;
ALTER TABLE table_rename_with_ttl RENAME COLUMN date1 TO renamed_date1;
ALTER TABLE table_rename_with_ttl materialize TTL;
SELECT count() FROM table_rename_with_ttl;
ALTER TABLE table_rename_with_ttl modify setting max_number_of_merges_with_ttl_in_pool=2;
optimize table table_rename_with_ttl FINAL;
ALTER TABLE table_rename_with_ttl materialize TTL settings mutations_sync=2;
SELECT count() FROM table_rename_with_ttl;

3
tests/queries/0_stateless/01560_ttl_remove_empty_parts.sql
View File

@ -10,9 +10,6 @@ SELECT count() FROM system.parts WHERE table = 'ttl_empty_parts' AND database =
ALTER TABLE ttl_empty_parts MODIFY TTL d;
SELECT sleep(3) format Null;
SELECT sleep(3) format Null;
-- To be sure, that task, which clears outdated parts executed.
DETACH TABLE ttl_empty_parts;
ATTACH TABLE ttl_empty_parts;