thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

replace std::vector to std::deque for MarkRanges

Browse Source
This commit is contained in:
CurtizJ 2020-02-10 15:36:01 +03:00
parent b9fc9b4569
commit 852772364e
7 changed files with 17 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dbms/src/Storages/MergeTree/MarkRange.h
View File

@ -1,7 +1,7 @@
#pragma once
#include <cstddef>
#include <vector>
#include <deque>
namespace DB
@ -19,7 +19,7 @@ struct MarkRange
MarkRange(const size_t begin_, const size_t end_) : begin{begin_}, end{end_} {}
};
using MarkRanges = std::vector<MarkRange>;
using MarkRanges = std::deque<MarkRange>;
}

2
dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
View File

@ -74,8 +74,6 @@ MergeTreeReadTask::MergeTreeReadTask(
ordered_names{ordered_names_}, column_name_set{column_name_set_}, columns{columns_}, pre_columns{pre_columns_},
remove_prewhere_column{remove_prewhere_column_}, should_reorder{should_reorder_}, size_predictor{std::move(size_predictor_)}
{
/// We need to save marks in reverse order.
std::reverse(mark_ranges.begin(), mark_ranges.end());
}
MergeTreeReadTask::~MergeTreeReadTask() = default;

3
dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h
View File

@ -30,8 +30,7 @@ struct MergeTreeReadTask
{
/// data part which should be read while performing this task
MergeTreeData::DataPartPtr data_part;
/** Ranges to read from `data_part`.
* Specified in reverse order for MergeTreeThreadSelectBlockInputStream's convenience of calling .pop_back(). */
/// Ranges to read from `data_part`.
MarkRanges mark_ranges;
/// for virtual `part_index` virtual column
size_t part_index_in_query;

19
dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
View File

@ -823,9 +823,6 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder(
sum_marks_in_parts[i] = parts[i].getMarksCount();
sum_marks += sum_marks_in_parts[i];
/// Let the ranges be listed from right to left so that the leftmost range can be dropped using `pop_back()`.
std::reverse(parts[i].ranges.begin(), parts[i].ranges.end());
if (parts[i].data_part->index_granularity_info.is_adaptive)
adaptive_parts++;
}
@ -888,15 +885,12 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder(
auto range = *it;
while (range.begin + marks_in_range < range.end)
{
new_ranges.emplace_back(range.end - marks_in_range, range.end);
new_ranges.emplace_front(range.end - marks_in_range, range.end);
range.end -= marks_in_range;
marks_in_range = std::min(marks_in_range * 2, max_marks_in_range);
}
new_ranges.emplace_back(range.begin, range.end);
new_ranges.emplace_front(range.begin, range.end);
}
/// Restore left-to-right order.
std::reverse(new_ranges.begin(), new_ranges.end());
}
return new_ranges;
@ -918,7 +912,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder(
RangesInDataPart part = parts.back();
parts.pop_back();
size_t & marks_in_part = sum_marks_in_parts.back();
size_t & marks_in_part = sum_marks_in_parts.front();
/// We will not take too few rows from a part.
if (marks_in_part >= min_marks_for_concurrent_read &&
@ -935,9 +929,6 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder(
/// We take the whole part if it is small enough.
if (marks_in_part <= need_marks)
{
/// Restore the order of segments.
std::reverse(part.ranges.begin(), part.ranges.end());
ranges_to_get_from_part = part.ranges;
need_marks -= marks_in_part;
@ -951,7 +942,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder(
if (part.ranges.empty())
throw Exception("Unexpected end of ranges while spreading marks among streams", ErrorCodes::LOGICAL_ERROR);
MarkRange & range = part.ranges.back();
MarkRange & range = part.ranges.front();
const size_t marks_in_range = range.end - range.begin;
const size_t marks_to_get_from_range = std::min(marks_in_range, need_marks);
@ -961,7 +952,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder(
marks_in_part -= marks_to_get_from_range;
need_marks -= marks_to_get_from_range;
if (range.begin == range.end)
part.ranges.pop_back();
part.ranges.pop_front();
}
parts.emplace_back(part);
}

6
dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp
View File

@ -690,9 +690,9 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::startReadingChain(size_t
if (stream.isFinished())
{
result.addRows(stream.finalize(result.columns));
stream = Stream(ranges.back().begin, ranges.back().end, merge_tree_reader);
result.addRange(ranges.back());
ranges.pop_back();
stream = Stream(ranges.front().begin, ranges.front().end, merge_tree_reader);
result.addRange(ranges.front());
ranges.pop_front();
}
auto rows_to_read = std::min(space_left, stream.numPendingRowsInCurrentGranule());

24
dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp
View File

@ -73,10 +73,7 @@ MergeTreeReadTaskPtr MergeTreeReadPool::getTask(const size_t min_marks_to_read,
if (marks_in_part <= need_marks)
{
const auto marks_to_get_from_range = marks_in_part;
/// Ranges are in right-to-left order, because 'reverse' was done in 'fillPerThreadInfo'.
ranges_to_get_from_part = thread_task.ranges;
std::reverse(ranges_to_get_from_part.begin(), ranges_to_get_from_part.end());
marks_in_part -= marks_to_get_from_range;
@ -91,7 +88,7 @@ MergeTreeReadTaskPtr MergeTreeReadPool::getTask(const size_t min_marks_to_read,
/// Loop through part ranges.
while (need_marks > 0 && !thread_task.ranges.empty())
{
auto & range = thread_task.ranges.back();
auto & range = thread_task.ranges.front();
const size_t marks_in_range = range.end - range.begin;
const size_t marks_to_get_from_range = std::min(marks_in_range, need_marks);
@ -100,14 +97,13 @@ MergeTreeReadTaskPtr MergeTreeReadPool::getTask(const size_t min_marks_to_read,
range.begin += marks_to_get_from_range;
if (range.begin == range.end)
{
std::swap(range, thread_task.ranges.back());
thread_task.ranges.pop_back();
std::swap(range, thread_task.ranges.front());
thread_task.ranges.pop_front();
}
marks_in_part -= marks_to_get_from_range;
need_marks -= marks_to_get_from_range;
}
/// Order of ranges was changed to left-to-right due to .pop_back() above.
}
auto curr_task_size_predictor = !per_part_size_predictor[part_idx] ? nullptr
@ -240,10 +236,6 @@ void MergeTreeReadPool::fillPerThreadInfo(
{
threads_tasks.resize(threads);
/// Let the ranges be listed from right to left so that the leftmost range can be dropped using `pop_back()`.
for (auto & part : parts)
std::reverse(part.ranges.begin(), part.ranges.end());
const size_t min_marks_per_thread = (sum_marks - 1) / threads + 1;
for (size_t i = 0; i < threads && !parts.empty(); ++i)
@ -272,7 +264,6 @@ void MergeTreeReadPool::fillPerThreadInfo(
/// Get whole part to read if it is small enough.
if (marks_in_part <= need_marks)
{
/// Leave ranges in right-to-left order for convenience to use .pop_back() in .getTask()
ranges_to_get_from_part = part.ranges;
marks_in_ranges = marks_in_part;
@ -288,7 +279,7 @@ void MergeTreeReadPool::fillPerThreadInfo(
if (part.ranges.empty())
throw Exception("Unexpected end of ranges while spreading marks among threads", ErrorCodes::LOGICAL_ERROR);
MarkRange & range = part.ranges.back();
MarkRange & range = part.ranges.front();
const size_t marks_in_range = range.end - range.begin;
const size_t marks_to_get_from_range = std::min(marks_in_range, need_marks);
@ -298,13 +289,8 @@ void MergeTreeReadPool::fillPerThreadInfo(
marks_in_part -= marks_to_get_from_range;
need_marks -= marks_to_get_from_range;
if (range.begin == range.end)
part.ranges.pop_back();
part.ranges.pop_front();
}
/** Change order to right-to-left, for getTask() to get ranges with .pop_back()
* (order was changed to left-to-right due to .pop_back() above).
*/
std::reverse(std::begin(ranges_to_get_from_part), std::end(ranges_to_get_from_part));
}
threads_tasks[i].parts_and_ranges.push_back({ part_idx, ranges_to_get_from_part });

2
dbms/src/Storages/MergeTree/MergeTreeReader.h
View File

@ -57,7 +57,7 @@ public:
size_t getFirstMarkToRead() const
{
return all_mark_ranges.back().begin;
return all_mark_ranges.front().begin;
}
private:
using FileStreams = std::map<std::string, std::unique_ptr<MergeTreeReaderStream>>;