thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-24 16:42:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix fsync_part_directory for horizontal merge

Browse Source
This commit is contained in:
Azat Khuzhin 2021-03-11 21:41:27 +03:00
parent b30a0c68da
commit 69b2b2a159
3 changed files with 20 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp
View File

@ -757,6 +757,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor
std::unique_ptr<WriteBuffer> rows_sources_write_buf; std::unique_ptr<WriteBuffer> rows_sources_write_buf;
std::optional<ColumnSizeEstimator> column_sizes; std::optional<ColumnSizeEstimator> column_sizes;
SyncGuardPtr sync_guard;
if (chosen_merge_algorithm == MergeAlgorithm::Vertical) if (chosen_merge_algorithm == MergeAlgorithm::Vertical)
{ {
tmp_disk->createDirectories(new_part_tmp_path); tmp_disk->createDirectories(new_part_tmp_path);
@ -769,6 +771,9 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor
part->accumulateColumnSizes(merged_column_to_size); part->accumulateColumnSizes(merged_column_to_size);
column_sizes = ColumnSizeEstimator(merged_column_to_size, merging_column_names, gathering_column_names); column_sizes = ColumnSizeEstimator(merged_column_to_size, merging_column_names, gathering_column_names);
if (data.getSettings()->fsync_part_directory)
sync_guard = disk->getDirectorySyncGuard(new_part_tmp_path);
} }
else else
{ {
@ -778,10 +783,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor
gathering_column_names.clear(); gathering_column_names.clear();
} }
SyncGuardPtr sync_guard;
if (data.getSettings()->fsync_part_directory)
sync_guard = disk->getDirectorySyncGuard(new_part_tmp_path);
/** Read from all parts, merge and write into a new one. /** Read from all parts, merge and write into a new one.
* In passing, we calculate expression for sorting. * In passing, we calculate expression for sorting.
*/ */

2
tests/queries/0_stateless/01643_merge_tree_fsync_smoke.reference
View File

@ -10,3 +10,5 @@ wide fsync_after_insert,fsync_part_directory
1 1
memory in_memory_parts_insert_sync memory in_memory_parts_insert_sync
1 1
wide fsync_part_directory,vertical
1

13
tests/queries/0_stateless/01643_merge_tree_fsync_smoke.sql
View File

@ -4,34 +4,47 @@ select 'default';
create table data_01643 (key Int) engine=MergeTree() order by key; create table data_01643 (key Int) engine=MergeTree() order by key;
insert into data_01643 values (1); insert into data_01643 values (1);
select * from data_01643; select * from data_01643;
optimize table data_01643 final;
drop table data_01643; drop table data_01643;
select 'compact fsync_after_insert'; select 'compact fsync_after_insert';
create table data_01643 (key Int) engine=MergeTree() order by key settings min_rows_for_wide_part=2, fsync_after_insert=1; create table data_01643 (key Int) engine=MergeTree() order by key settings min_rows_for_wide_part=2, fsync_after_insert=1;
insert into data_01643 values (1); insert into data_01643 values (1);
select * from data_01643; select * from data_01643;
optimize table data_01643 final;
drop table data_01643; drop table data_01643;
select 'compact fsync_after_insert,fsync_part_directory'; select 'compact fsync_after_insert,fsync_part_directory';
create table data_01643 (key Int) engine=MergeTree() order by key settings min_rows_for_wide_part=2, fsync_after_insert=1, fsync_part_directory=1; create table data_01643 (key Int) engine=MergeTree() order by key settings min_rows_for_wide_part=2, fsync_after_insert=1, fsync_part_directory=1;
insert into data_01643 values (1); insert into data_01643 values (1);
select * from data_01643; select * from data_01643;
optimize table data_01643 final;
drop table data_01643; drop table data_01643;
select 'wide fsync_after_insert'; select 'wide fsync_after_insert';
create table data_01643 (key Int) engine=MergeTree() order by key settings min_bytes_for_wide_part=0, fsync_after_insert=1; create table data_01643 (key Int) engine=MergeTree() order by key settings min_bytes_for_wide_part=0, fsync_after_insert=1;
insert into data_01643 values (1); insert into data_01643 values (1);
select * from data_01643; select * from data_01643;
optimize table data_01643 final;
drop table data_01643; drop table data_01643;
select 'wide fsync_after_insert,fsync_part_directory'; select 'wide fsync_after_insert,fsync_part_directory';
create table data_01643 (key Int) engine=MergeTree() order by key settings min_bytes_for_wide_part=0, fsync_after_insert=1, fsync_part_directory=1; create table data_01643 (key Int) engine=MergeTree() order by key settings min_bytes_for_wide_part=0, fsync_after_insert=1, fsync_part_directory=1;
insert into data_01643 values (1); insert into data_01643 values (1);
select * from data_01643; select * from data_01643;
optimize table data_01643 final;
drop table data_01643; drop table data_01643;
select 'memory in_memory_parts_insert_sync'; select 'memory in_memory_parts_insert_sync';
create table data_01643 (key Int) engine=MergeTree() order by key settings min_rows_for_compact_part=2, in_memory_parts_insert_sync=1, fsync_after_insert=1, fsync_part_directory=1; create table data_01643 (key Int) engine=MergeTree() order by key settings min_rows_for_compact_part=2, in_memory_parts_insert_sync=1, fsync_after_insert=1, fsync_part_directory=1;
insert into data_01643 values (1); insert into data_01643 values (1);
select * from data_01643; select * from data_01643;
optimize table data_01643 final;
drop table data_01643;
select 'wide fsync_part_directory,vertical';
create table data_01643 (key Int) engine=MergeTree() order by key settings min_bytes_for_wide_part=0, fsync_part_directory=1, enable_vertical_merge_algorithm=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;
insert into data_01643 values (1);
select * from data_01643;
optimize table data_01643 final;
drop table data_01643; drop table data_01643;