This commit is contained in:
Igor Nikonov 2024-11-21 13:44:15 +00:00 committed by GitHub
commit 9c6cdcde78
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 24 additions and 2 deletions

View File

@ -5652,6 +5652,9 @@ Parts virtually divided into segments to be distributed between replicas for par
)", BETA) \ )", BETA) \
DECLARE(Bool, parallel_replicas_local_plan, true, R"( DECLARE(Bool, parallel_replicas_local_plan, true, R"(
Build local plan for local replica Build local plan for local replica
)", BETA) \
DECLARE(Bool, parallel_replicas_skip_index_analysis_on_workers, true, R"(
Skip index analysis on workers. Effective only with enabled parallel_replicas_local_plan
)", BETA) \ )", BETA) \
\ \
DECLARE(Bool, allow_experimental_analyzer, true, R"( DECLARE(Bool, allow_experimental_analyzer, true, R"(

View File

@ -90,6 +90,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
{"s3_skip_empty_files", false, true, "We hope it will provide better UX"}, {"s3_skip_empty_files", false, true, "We hope it will provide better UX"},
{"filesystem_cache_boundary_alignment", 0, 0, "New setting"}, {"filesystem_cache_boundary_alignment", 0, 0, "New setting"},
{"push_external_roles_in_interserver_queries", false, false, "New setting."}, {"push_external_roles_in_interserver_queries", false, false, "New setting."},
{"parallel_replicas_skip_index_analysis_on_workers", false, true, "Skip index analysis on workers. Effective only with enabled parallel_replicas_local_plan"},
} }
}, },
{"24.10", {"24.10",

View File

@ -80,6 +80,8 @@ namespace Setting
extern const SettingsUInt64 parallel_replica_offset; extern const SettingsUInt64 parallel_replica_offset;
extern const SettingsUInt64 parallel_replicas_count; extern const SettingsUInt64 parallel_replicas_count;
extern const SettingsParallelReplicasMode parallel_replicas_mode; extern const SettingsParallelReplicasMode parallel_replicas_mode;
extern const SettingsBool parallel_replicas_local_plan;
extern const SettingsBool parallel_replicas_skip_index_analysis_on_workers;
} }
namespace MergeTreeSetting namespace MergeTreeSetting
@ -631,10 +633,23 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd
bool use_skip_indexes, bool use_skip_indexes,
bool find_exact_ranges) bool find_exact_ranges)
{ {
RangesInDataParts parts_with_ranges;
parts_with_ranges.resize(parts.size());
const Settings & settings = context->getSettingsRef(); const Settings & settings = context->getSettingsRef();
if (context->canUseParallelReplicasOnFollower() && settings[Setting::parallel_replicas_local_plan]
&& settings[Setting::parallel_replicas_skip_index_analysis_on_workers])
{
// Skip index analysis and return parts with all marks
// The coordinator will chose ranges to read for workers based on index analysis on its side
RangesInDataParts parts_with_ranges;
parts_with_ranges.reserve(parts.size());
for (size_t part_index = 0; part_index < parts.size(); ++part_index)
{
const auto & part = parts[part_index];
parts_with_ranges.emplace_back(part, part_index, MarkRanges{{0, part->getMarksCount()}});
}
return parts_with_ranges;
}
if (use_skip_indexes && settings[Setting::force_data_skipping_indices].changed) if (use_skip_indexes && settings[Setting::force_data_skipping_indices].changed)
{ {
const auto & indices_str = settings[Setting::force_data_skipping_indices].toString(); const auto & indices_str = settings[Setting::force_data_skipping_indices].toString();
@ -673,6 +688,8 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd
std::atomic<size_t> sum_marks_pk = 0; std::atomic<size_t> sum_marks_pk = 0;
std::atomic<size_t> sum_parts_pk = 0; std::atomic<size_t> sum_parts_pk = 0;
RangesInDataParts parts_with_ranges(parts.size());
/// Let's find what range to read from each part. /// Let's find what range to read from each part.
{ {
auto mark_cache = context->getIndexMarkCache(); auto mark_cache = context->getIndexMarkCache();

View File

@ -71,6 +71,7 @@ def _get_result_with_parallel_replicas(
"cluster_for_parallel_replicas": f"{cluster_name}", "cluster_for_parallel_replicas": f"{cluster_name}",
"parallel_replicas_mark_segment_size": parallel_replicas_mark_segment_size, "parallel_replicas_mark_segment_size": parallel_replicas_mark_segment_size,
"query_id": query_id, "query_id": query_id,
"parallel_replicas_skip_index_analysis_on_workers": False,
}, },
) )