thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-19 06:01:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

bad size of mark skip idx

Browse Source
This commit is contained in:
王智博 2023-08-20 22:57:53 +08:00
parent 89c365803f
commit 8bb0f18019
4 changed files with 55 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
View File

@ -981,9 +981,9 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd
const auto & index_and_condition = skip_indexes.useful_indices[idx];
auto & stat = useful_indices_stat[idx];
stat.total_parts.fetch_add(1, std::memory_order_relaxed);
stat.total_granules.fetch_add(ranges.ranges.getNumberOfMarks(), std::memory_order_relaxed);
size_t total_granules = ranges.ranges.getNumberOfMarks();
stat.total_granules.fetch_add(total_granules, std::memory_order_relaxed);
size_t granules_dropped = 0;
ranges.ranges = filterMarksUsingIndex(
index_and_condition.index,
index_and_condition.condition,
@ -991,12 +991,11 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd
ranges.ranges,
settings,
reader_settings,
granules_dropped,
mark_cache.get(),
uncompressed_cache.get(),
log);
stat.granules_dropped.fetch_add(granules_dropped, std::memory_order_relaxed);
stat.granules_dropped.fetch_add(total_granules - ranges.ranges.getNumberOfMarks(), std::memory_order_relaxed);
if (ranges.ranges.empty())
stat.parts_dropped.fetch_add(1, std::memory_order_relaxed);
}
@ -1010,17 +1009,15 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd
auto & stat = merged_indices_stat[idx];
stat.total_parts.fetch_add(1, std::memory_order_relaxed);
size_t total_granules = 0;
size_t granules_dropped = 0;
size_t total_granules = ranges.ranges.getNumberOfMarks();
ranges.ranges = filterMarksUsingMergedIndex(
indices_and_condition.indices, indices_and_condition.condition,
part, ranges.ranges,
settings, reader_settings,
total_granules, granules_dropped,
mark_cache.get(), uncompressed_cache.get(), log);
stat.total_granules.fetch_add(total_granules, std::memory_order_relaxed);
stat.granules_dropped.fetch_add(granules_dropped, std::memory_order_relaxed);
stat.granules_dropped.fetch_add(total_granules - ranges.ranges.getNumberOfMarks(), std::memory_order_relaxed);
if (ranges.ranges.empty())
stat.parts_dropped.fetch_add(1, std::memory_order_relaxed);
@ -1574,7 +1571,6 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex(
const MarkRanges & ranges,
const Settings & settings,
const MergeTreeReaderSettings & reader_settings,
size_t & granules_dropped,
MarkCache * mark_cache,
UncompressedCache * uncompressed_cache,
Poco::Logger * log)
@ -1644,8 +1640,6 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex(
{
// vector of indexes of useful ranges
auto result = ann_condition->getUsefulRanges(granule);
if (result.empty())
++granules_dropped;
for (auto range : result)
{
@ -1670,10 +1664,7 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex(
result = cache_in_store.store ? gin_filter_condition->mayBeTrueOnGranuleInPart(granule, cache_in_store) : true;
if (!result)
{
++granules_dropped;
continue;
}
MarkRange data_range(
std::max(ranges[i].begin, index_mark * index_granularity),
@ -1698,8 +1689,6 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingMergedIndex(
const MarkRanges & ranges,
const Settings & settings,
const MergeTreeReaderSettings & reader_settings,
size_t & total_granules,
size_t & granules_dropped,
MarkCache * mark_cache,
UncompressedCache * uncompressed_cache,
Poco::Logger * log)
@ -1756,8 +1745,6 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingMergedIndex(
for (auto & reader : readers)
reader->seek(index_range.begin);
total_granules += index_range.end - index_range.begin;
for (size_t index_mark = index_range.begin; index_mark < index_range.end; ++index_mark)
{
if (index_mark != index_range.begin || !granules_filled || last_index_mark != index_range.begin)
@ -1770,10 +1757,7 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingMergedIndex(
}
if (!condition->mayBeTrueOnGranule(granules))
{
++granules_dropped;
continue;
}
MarkRange data_range(
std::max(range.begin, index_mark * index_granularity),

3
src/Storages/MergeTree/MergeTreeDataSelectExecutor.h
View File

@ -93,7 +93,6 @@ private:
const MarkRanges & ranges,
const Settings & settings,
const MergeTreeReaderSettings & reader_settings,
size_t & granules_dropped,
MarkCache * mark_cache,
UncompressedCache * uncompressed_cache,
Poco::Logger * log);
@ -105,8 +104,6 @@ private:
const MarkRanges & ranges,
const Settings & settings,
const MergeTreeReaderSettings & reader_settings,
size_t & total_granules,
size_t & granules_dropped,
MarkCache * mark_cache,
UncompressedCache * uncompressed_cache,
Poco::Logger * log);

29
tests/queries/0_stateless/02865_size_of_marks_skip_idx_explain.reference Normal file
View File

@ -0,0 +1,29 @@
Expression ((Projection + Before ORDER BY))
Filter (WHERE)
ReadFromMergeTree (default.test_skip_idx)
Indexes:
Skip
Name: name_idx_g2
Description: minmax GRANULARITY 2
Parts: 1/1
Granules: 2/5
Skip
Name: name_idx_g1
Description: minmax GRANULARITY 1
Parts: 1/1
Granules: 1/2
--
Expression ((Projection + Before ORDER BY))
Filter (WHERE)
ReadFromMergeTree (default.test_skip_idx)
Indexes:
Skip
Name: name_idx_g2
Description: minmax GRANULARITY 2
Parts: 1/1
Granules: 2/5
Skip
Name: name_idx_g1
Description: minmax GRANULARITY 1
Parts: 1/1
Granules: 2/2

21
tests/queries/0_stateless/02865_size_of_marks_skip_idx_explain.sql Normal file
View File

@ -0,0 +1,21 @@
DROP TABLE IF EXISTS test_skip_idx;
CREATE TABLE test_skip_idx
(
`id` UInt32,
INDEX name_idx_g2 id TYPE minmax GRANULARITY 2,
INDEX name_idx_g1 id TYPE minmax GRANULARITY 1
)
ENGINE = MergeTree
ORDER BY tuple()
SETTINGS index_granularity = 1, index_granularity_bytes = 0;
INSERT INTO test_skip_idx SELECT number FROM system.numbers LIMIT 5 OFFSET 1;
EXPLAIN indexes = 1 SELECT * FROM test_skip_idx WHERE id < 2;
SELECT '--';
EXPLAIN indexes = 1 SELECT * FROM test_skip_idx WHERE id < 3;
DROP TABLE IF EXISTS test_skip_idx;