2019-10-10 16:30:30 +00:00
|
|
|
#include <Storages/MergeTree/IMergeTreeReader.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Storages/MergeTree/MergeTreeReadPool.h>
|
2021-07-26 16:48:25 +00:00
|
|
|
#include <Storages/MergeTree/MergeTreeThreadSelectProcessor.h>
|
2020-05-20 20:16:32 +00:00
|
|
|
#include <Interpreters/Context.h>
|
2016-11-20 12:43:20 +00:00
|
|
|
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2021-12-09 10:39:28 +00:00
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
|
|
|
extern const int LOGICAL_ERROR;
|
|
|
|
}
|
2016-11-20 12:43:20 +00:00
|
|
|
|
2021-07-26 16:48:25 +00:00
|
|
|
MergeTreeThreadSelectProcessor::MergeTreeThreadSelectProcessor(
|
2022-03-13 12:23:51 +00:00
|
|
|
size_t thread_,
|
2019-08-03 11:02:40 +00:00
|
|
|
const MergeTreeReadPoolPtr & pool_,
|
2022-03-13 12:23:51 +00:00
|
|
|
size_t min_marks_to_read_,
|
|
|
|
UInt64 max_block_size_rows_,
|
2019-01-07 10:40:58 +00:00
|
|
|
size_t preferred_block_size_bytes_,
|
|
|
|
size_t preferred_max_column_in_block_size_bytes_,
|
|
|
|
const MergeTreeData & storage_,
|
2021-07-09 03:15:41 +00:00
|
|
|
const StorageSnapshotPtr & storage_snapshot_,
|
2022-03-13 12:23:51 +00:00
|
|
|
bool use_uncompressed_cache_,
|
2019-01-07 10:40:58 +00:00
|
|
|
const PrewhereInfoPtr & prewhere_info_,
|
2021-06-25 14:49:28 +00:00
|
|
|
ExpressionActionsSettings actions_settings,
|
2019-12-18 15:54:45 +00:00
|
|
|
const MergeTreeReaderSettings & reader_settings_,
|
2021-12-09 10:39:28 +00:00
|
|
|
const Names & virt_column_names_,
|
|
|
|
std::optional<ParallelReadingExtension> extension_)
|
2017-03-24 13:52:50 +00:00
|
|
|
:
|
2020-06-16 14:25:08 +00:00
|
|
|
MergeTreeBaseSelectProcessor{
|
2022-06-07 11:53:10 +00:00
|
|
|
pool_->getHeader(), storage_, storage_snapshot_, prewhere_info_, std::move(actions_settings), max_block_size_rows_,
|
2020-01-04 05:46:50 +00:00
|
|
|
preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_,
|
2021-12-09 10:39:28 +00:00
|
|
|
reader_settings_, use_uncompressed_cache_, virt_column_names_, extension_},
|
2019-08-03 11:02:40 +00:00
|
|
|
thread{thread_},
|
|
|
|
pool{pool_}
|
2017-03-24 13:52:50 +00:00
|
|
|
{
|
|
|
|
/// round min_marks_to_read up to nearest multiple of block_size expressed in marks
|
2019-04-01 10:34:22 +00:00
|
|
|
/// If granularity is adaptive it doesn't make sense
|
2020-08-08 00:47:03 +00:00
|
|
|
/// Maybe it will make sense to add settings `max_block_size_bytes`
|
2019-06-19 14:46:06 +00:00
|
|
|
if (max_block_size_rows && !storage.canUseAdaptiveGranularity())
|
2017-03-24 13:52:50 +00:00
|
|
|
{
|
2019-08-26 14:24:29 +00:00
|
|
|
size_t fixed_index_granularity = storage.getSettings()->index_granularity;
|
2019-04-01 10:34:22 +00:00
|
|
|
min_marks_to_read = (min_marks_to_read_ * fixed_index_granularity + max_block_size_rows - 1)
|
|
|
|
/ max_block_size_rows * max_block_size_rows / fixed_index_granularity;
|
2017-03-24 13:52:50 +00:00
|
|
|
}
|
2021-12-09 10:39:28 +00:00
|
|
|
else if (extension.has_value())
|
|
|
|
{
|
|
|
|
/// Parallel reading from replicas is enabled.
|
|
|
|
/// We try to estimate the average number of bytes in a granule
|
|
|
|
/// to make one request over the network per one gigabyte of data
|
|
|
|
/// Actually we will ask MergeTreeReadPool to provide us heavier tasks to read
|
|
|
|
/// because the most part of each task will be postponed
|
|
|
|
/// (due to using consistent hash for better cache affinity)
|
|
|
|
const size_t amount_of_read_bytes_per_one_request = 1024 * 1024 * 1024; // 1GiB
|
|
|
|
/// In case of reading from compact parts (for which we can't estimate the average size of marks)
|
|
|
|
/// we will use this value
|
|
|
|
const size_t empirical_size_of_mark = 1024 * 1024 * 10; // 10 MiB
|
|
|
|
|
|
|
|
if (extension->colums_to_read.empty())
|
|
|
|
throw Exception(ErrorCodes::LOGICAL_ERROR, "A set of column to read is empty. It is a bug");
|
|
|
|
|
|
|
|
size_t sum_average_marks_size = 0;
|
|
|
|
auto column_sizes = storage.getColumnSizes();
|
|
|
|
for (const auto & name : extension->colums_to_read)
|
|
|
|
{
|
|
|
|
auto it = column_sizes.find(name);
|
|
|
|
if (it == column_sizes.end())
|
|
|
|
continue;
|
|
|
|
auto size = it->second;
|
|
|
|
|
|
|
|
if (size.data_compressed == 0 || size.data_uncompressed == 0 || size.marks == 0)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
sum_average_marks_size += size.data_uncompressed / size.marks;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (sum_average_marks_size == 0)
|
|
|
|
sum_average_marks_size = empirical_size_of_mark * extension->colums_to_read.size();
|
|
|
|
|
|
|
|
min_marks_to_read = extension->count_participating_replicas * amount_of_read_bytes_per_one_request / sum_average_marks_size;
|
|
|
|
}
|
2017-03-24 13:52:50 +00:00
|
|
|
else
|
2021-12-09 10:39:28 +00:00
|
|
|
{
|
2017-03-24 13:52:50 +00:00
|
|
|
min_marks_to_read = min_marks_to_read_;
|
2021-12-09 10:39:28 +00:00
|
|
|
}
|
|
|
|
|
2018-04-19 15:18:26 +00:00
|
|
|
|
2019-09-13 15:41:09 +00:00
|
|
|
ordered_names = getPort().getHeader().getNames();
|
2017-03-24 13:52:50 +00:00
|
|
|
}
|
2016-11-20 12:43:20 +00:00
|
|
|
|
|
|
|
/// Requests read task from MergeTreeReadPool and signals whether it got one
|
2021-12-09 10:39:28 +00:00
|
|
|
bool MergeTreeThreadSelectProcessor::getNewTaskImpl()
|
2016-11-20 12:43:20 +00:00
|
|
|
{
|
2018-04-19 15:18:26 +00:00
|
|
|
task = pool->getTask(min_marks_to_read, thread, ordered_names);
|
2021-12-09 10:39:28 +00:00
|
|
|
return static_cast<bool>(task);
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
|
|
|
|
2021-12-09 10:39:28 +00:00
|
|
|
void MergeTreeThreadSelectProcessor::finalizeNewTask()
|
|
|
|
{
|
2021-02-10 14:12:49 +00:00
|
|
|
const std::string part_name = task->data_part->isProjectionPart() ? task->data_part->getParentPart()->name : task->data_part->name;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2017-03-13 18:01:46 +00:00
|
|
|
/// Allows pool to reduce number of threads in case of too slow reads.
|
2019-08-03 11:02:40 +00:00
|
|
|
auto profile_callback = [this](ReadBufferFromFileBase::ProfileInfo info_) { pool->profileFeedback(info_); };
|
2021-07-09 03:15:41 +00:00
|
|
|
const auto & metadata_snapshot = storage_snapshot->metadata;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2022-07-17 18:41:17 +00:00
|
|
|
IMergeTreeReader::ValueSizeMap value_size_map;
|
|
|
|
|
2016-11-20 12:43:20 +00:00
|
|
|
if (!reader)
|
|
|
|
{
|
|
|
|
if (use_uncompressed_cache)
|
2021-04-10 23:33:54 +00:00
|
|
|
owned_uncompressed_cache = storage.getContext()->getUncompressedCache();
|
|
|
|
owned_mark_cache = storage.getContext()->getMarkCache();
|
2016-11-20 12:43:20 +00:00
|
|
|
}
|
2022-07-17 18:41:17 +00:00
|
|
|
else if (part_name != last_readed_part_name)
|
2016-11-20 12:43:20 +00:00
|
|
|
{
|
2022-07-17 18:41:17 +00:00
|
|
|
value_size_map = reader->getAvgValueSizeHints();
|
|
|
|
}
|
|
|
|
|
|
|
|
const bool init_new_readers = !reader || part_name != last_readed_part_name;
|
|
|
|
if (init_new_readers)
|
|
|
|
{
|
|
|
|
initializeMergeTreeReadersForPart(task->data_part, task->task_columns, metadata_snapshot,
|
|
|
|
task->mark_ranges, value_size_map, profile_callback);
|
2016-11-20 12:43:20 +00:00
|
|
|
}
|
2019-10-10 16:30:30 +00:00
|
|
|
|
2020-04-14 19:47:19 +00:00
|
|
|
last_readed_part_name = part_name;
|
2021-12-09 10:39:28 +00:00
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2021-12-09 10:39:28 +00:00
|
|
|
|
|
|
|
void MergeTreeThreadSelectProcessor::finish()
|
|
|
|
{
|
|
|
|
reader.reset();
|
2022-06-07 07:03:11 +00:00
|
|
|
pre_reader_for_step.clear();
|
2016-11-20 12:43:20 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2021-07-26 16:48:25 +00:00
|
|
|
MergeTreeThreadSelectProcessor::~MergeTreeThreadSelectProcessor() = default;
|
2017-03-24 13:52:50 +00:00
|
|
|
|
2016-11-20 12:43:20 +00:00
|
|
|
}
|