From fa8aafa94222da68791492561aaa0ee70e395249 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 22 May 2024 21:28:33 +0000 Subject: [PATCH 001/215] Local plan for parallel replicas: save --- .../ClusterProxy/executeQuery.cpp | 83 +++++++++++++++---- src/Interpreters/ClusterProxy/executeQuery.h | 9 +- src/Planner/PlannerJoinTree.cpp | 4 +- .../QueryPlan/DistributedCreateLocalPlan.cpp | 2 - .../QueryPlan/DistributedCreateLocalPlan.h | 5 -- .../QueryPlan/ParallelReplicasLocalPlan.cpp | 78 +++++++++++++++++ .../QueryPlan/ParallelReplicasLocalPlan.h | 19 +++++ src/Processors/QueryPlan/ReadFromRemote.cpp | 7 +- src/Processors/QueryPlan/ReadFromRemote.h | 4 +- 9 files changed, 182 insertions(+), 29 deletions(-) create mode 100644 src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp create mode 100644 src/Processors/QueryPlan/ParallelReplicasLocalPlan.h diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index d1701d268f1..71912fa1081 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -25,6 +25,7 @@ #include #include #include +#include namespace DB { @@ -403,7 +404,8 @@ void executeQueryWithParallelReplicas( QueryProcessingStage::Enum processed_stage, const ASTPtr & query_ast, ContextPtr context, - std::shared_ptr storage_limits) + std::shared_ptr storage_limits, + QueryPlanStepPtr read_from_merge_tree) { const auto & settings = context->getSettingsRef(); @@ -486,21 +488,66 @@ void executeQueryWithParallelReplicas( auto coordinator = std::make_shared( new_cluster->getShardsInfo().begin()->getAllNodeCount(), settings.parallel_replicas_mark_segment_size); auto external_tables = new_context->getExternalTables(); - auto read_from_remote = std::make_unique( - query_ast, - new_cluster, - storage_id, - std::move(coordinator), - header, - processed_stage, - new_context, - getThrottler(new_context), - std::move(scalars), - std::move(external_tables), - getLogger("ReadFromParallelRemoteReplicasStep"), - std::move(storage_limits)); - query_plan.addStep(std::move(read_from_remote)); + if (settings.allow_experimental_analyzer) + { + auto read_from_remote = std::make_unique( + query_ast, + new_cluster, + storage_id, + std::move(coordinator), + header, + processed_stage, + new_context, + getThrottler(new_context), + std::move(scalars), + std::move(external_tables), + getLogger("ReadFromParallelRemoteReplicasStep"), + std::move(storage_limits), + /*exclude_local_replica*/ true); + + auto remote_plan = std::make_unique(); + remote_plan->addStep(std::move(read_from_remote)); + + auto local_plan = createLocalPlanForParallelReplicas( + query_ast, + header, + new_context, + processed_stage, + coordinator, + std::move(read_from_merge_tree), + /*has_missing_objects=*/false); + + DataStreams input_streams; + input_streams.reserve(2); + input_streams.emplace_back(local_plan->getCurrentDataStream()); + input_streams.emplace_back(remote_plan->getCurrentDataStream()); + + std::vector plans; + plans.emplace_back(std::move(local_plan)); + plans.emplace_back(std::move(remote_plan)); + + auto union_step = std::make_unique(std::move(input_streams)); + query_plan.unitePlans(std::move(union_step), std::move(plans)); + } + else { + auto read_from_remote = std::make_unique( + query_ast, + new_cluster, + storage_id, + std::move(coordinator), + header, + processed_stage, + new_context, + getThrottler(new_context), + std::move(scalars), + std::move(external_tables), + getLogger("ReadFromParallelRemoteReplicasStep"), + std::move(storage_limits), + /*exclude_local_replica*/ false); + + query_plan.addStep(std::move(read_from_remote)); + } } void executeQueryWithParallelReplicas( @@ -510,7 +557,8 @@ void executeQueryWithParallelReplicas( const QueryTreeNodePtr & query_tree, const PlannerContextPtr & planner_context, ContextPtr context, - std::shared_ptr storage_limits) + std::shared_ptr storage_limits, + QueryPlanStepPtr read_from_merge_tree) { QueryTreeNodePtr modified_query_tree = query_tree->clone(); rewriteJoinToGlobalJoin(modified_query_tree, context); @@ -520,7 +568,8 @@ void executeQueryWithParallelReplicas( = InterpreterSelectQueryAnalyzer::getSampleBlock(modified_query_tree, context, SelectQueryOptions(processed_stage).analyze()); auto modified_query_ast = queryNodeToDistributedSelectQuery(modified_query_tree); - executeQueryWithParallelReplicas(query_plan, storage_id, header, processed_stage, modified_query_ast, context, storage_limits); + executeQueryWithParallelReplicas( + query_plan, storage_id, header, processed_stage, modified_query_ast, context, storage_limits, std::move(read_from_merge_tree)); } void executeQueryWithParallelReplicas( diff --git a/src/Interpreters/ClusterProxy/executeQuery.h b/src/Interpreters/ClusterProxy/executeQuery.h index 6548edf8939..1b38d1921b1 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.h +++ b/src/Interpreters/ClusterProxy/executeQuery.h @@ -30,6 +30,9 @@ using QueryTreeNodePtr = std::shared_ptr; class PlannerContext; using PlannerContextPtr = std::shared_ptr; +class IQueryPlanStep; +using QueryPlanStepPtr = std::unique_ptr; + namespace ClusterProxy { @@ -73,7 +76,8 @@ void executeQueryWithParallelReplicas( QueryProcessingStage::Enum processed_stage, const ASTPtr & query_ast, ContextPtr context, - std::shared_ptr storage_limits); + std::shared_ptr storage_limits, + QueryPlanStepPtr read_from_merge_tree = nullptr); void executeQueryWithParallelReplicas( QueryPlan & query_plan, @@ -90,7 +94,8 @@ void executeQueryWithParallelReplicas( const QueryTreeNodePtr & query_tree, const PlannerContextPtr & planner_context, ContextPtr context, - std::shared_ptr storage_limits); + std::shared_ptr storage_limits, + QueryPlanStepPtr read_from_merge_tree); } } diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index a6e4a8ebcde..275461fa109 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -934,6 +934,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres { from_stage = QueryProcessingStage::WithMergeableState; QueryPlan query_plan_parallel_replicas; + QueryPlanStepPtr reading_step = std::move(node->step); ClusterProxy::executeQueryWithParallelReplicas( query_plan_parallel_replicas, storage->getStorageID(), @@ -941,7 +942,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres table_expression_query_info.query_tree, table_expression_query_info.planner_context, query_context, - table_expression_query_info.storage_limits); + table_expression_query_info.storage_limits, + std::move(reading_step)); query_plan = std::move(query_plan_parallel_replicas); const Block & query_plan_header = query_plan.getCurrentDataStream().header; diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp index d4545482477..1f4f271fa6e 100644 --- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp +++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp @@ -1,8 +1,6 @@ #include -#include #include -#include #include #include #include diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.h b/src/Processors/QueryPlan/DistributedCreateLocalPlan.h index 50545d9ae81..f59123a7d88 100644 --- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.h +++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.h @@ -1,17 +1,12 @@ #pragma once #include -#include #include #include -#include namespace DB { -class PreparedSets; -using PreparedSetsPtr = std::shared_ptr; - std::unique_ptr createLocalPlan( const ASTPtr & query_ast, const Block & header, diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp new file mode 100644 index 00000000000..4d78e049b58 --- /dev/null +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -0,0 +1,78 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace +{ + +void addConvertingActions(QueryPlan & plan, const Block & header, bool has_missing_objects) +{ + if (blocksHaveEqualStructure(plan.getCurrentDataStream().header, header)) + return; + + auto mode = has_missing_objects ? ActionsDAG::MatchColumnsMode::Position : ActionsDAG::MatchColumnsMode::Name; + + auto get_converting_dag = [mode](const Block & block_, const Block & header_) + { + /// Convert header structure to expected. + /// Also we ignore constants from result and replace it with constants from header. + /// It is needed for functions like `now64()` or `randConstant()` because their values may be different. + return ActionsDAG::makeConvertingActions( + block_.getColumnsWithTypeAndName(), + header_.getColumnsWithTypeAndName(), + mode, + true); + }; + + auto convert_actions_dag = get_converting_dag(plan.getCurrentDataStream().header, header); + auto converting = std::make_unique(plan.getCurrentDataStream(), convert_actions_dag); + plan.addStep(std::move(converting)); +} + +} + +std::unique_ptr createLocalPlanForParallelReplicas( + const ASTPtr & query_ast, + const Block & header, + ContextPtr context, + QueryProcessingStage::Enum processed_stage, + ParallelReplicasReadingCoordinatorPtr /*coordinator*/, + QueryPlanStepPtr /*read_from_merge_tree*/, + bool has_missing_objects) +{ + checkStackSize(); + + auto query_plan = std::make_unique(); + auto new_context = Context::createCopy(context); + + /// Do not push down limit to local plan, as it will break `rows_before_limit_at_least` counter. + if (processed_stage == QueryProcessingStage::WithMergeableStateAfterAggregationAndLimit) + processed_stage = QueryProcessingStage::WithMergeableStateAfterAggregation; + + /// Do not apply AST optimizations, because query + /// is already optimized and some optimizations + /// can be applied only for non-distributed tables + /// and we can produce query, inconsistent with remote plans. + auto select_query_options = SelectQueryOptions(processed_stage).ignoreASTOptimizations(); + + /// For Analyzer, identifier in GROUP BY/ORDER BY/LIMIT BY lists has been resolved to + /// ConstantNode in QueryTree if it is an alias of a constant, so we should not replace + /// ConstantNode with ProjectionNode again(https://github.com/ClickHouse/ClickHouse/issues/62289). + new_context->setSetting("enable_positional_arguments", Field(false)); + auto interpreter = InterpreterSelectQueryAnalyzer(query_ast, new_context, select_query_options); + query_plan = std::make_unique(std::move(interpreter).extractQueryPlan()); + + addConvertingActions(*query_plan, header, has_missing_objects); + return query_plan; +} + +} diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.h b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.h new file mode 100644 index 00000000000..89d2019f807 --- /dev/null +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ + +std::unique_ptr createLocalPlanForParallelReplicas( + const ASTPtr & query_ast, + const Block & header, + ContextPtr context, + QueryProcessingStage::Enum processed_stage, + ParallelReplicasReadingCoordinatorPtr coordinator, + QueryPlanStepPtr read_from_merge_tree, + bool has_missing_objects); +} diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index b4e35af85d6..6e6edfa1208 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -369,7 +369,8 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep( Scalars scalars_, Tables external_tables_, LoggerPtr log_, - std::shared_ptr storage_limits_) + std::shared_ptr storage_limits_, + bool exclude_local_replica_) : ISourceStep(DataStream{.header = std::move(header_)}) , cluster(cluster_) , query_ast(query_ast_) @@ -382,6 +383,7 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep( , external_tables{external_tables_} , storage_limits(std::move(storage_limits_)) , log(log_) + , exclude_local_replica(exclude_local_replica_) { chassert(cluster->getShardCount() == 1); @@ -410,6 +412,9 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder const auto & shard = cluster->getShardsInfo().at(0); size_t all_replicas_count = current_settings.max_parallel_replicas; + if (exclude_local_replica) + --all_replicas_count; + if (all_replicas_count > shard.getAllNodeCount()) { LOG_INFO( diff --git a/src/Processors/QueryPlan/ReadFromRemote.h b/src/Processors/QueryPlan/ReadFromRemote.h index eb15269155a..442da098a17 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.h +++ b/src/Processors/QueryPlan/ReadFromRemote.h @@ -78,7 +78,8 @@ public: Scalars scalars_, Tables external_tables_, LoggerPtr log_, - std::shared_ptr storage_limits_); + std::shared_ptr storage_limits_, + bool exclude_local_replica = false); String getName() const override { return "ReadFromRemoteParallelReplicas"; } @@ -101,6 +102,7 @@ private: Tables external_tables; std::shared_ptr storage_limits; LoggerPtr log; + bool exclude_local_replica; }; } From be08ebd0f4755c475385fa3460c32eaea6f71312 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 24 May 2024 13:01:30 +0000 Subject: [PATCH 002/215] Fix --- src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp index 4d78e049b58..64646960824 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -68,6 +68,7 @@ std::unique_ptr createLocalPlanForParallelReplicas( /// ConstantNode in QueryTree if it is an alias of a constant, so we should not replace /// ConstantNode with ProjectionNode again(https://github.com/ClickHouse/ClickHouse/issues/62289). new_context->setSetting("enable_positional_arguments", Field(false)); + new_context->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0)); auto interpreter = InterpreterSelectQueryAnalyzer(query_ast, new_context, select_query_options); query_plan = std::make_unique(std::move(interpreter).extractQueryPlan()); From 5cdf8d336cf509f7af1031a1b9a856d06b5f2ac2 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 27 May 2024 12:22:18 +0000 Subject: [PATCH 003/215] Local reading step from merge tree --- .../ClusterProxy/executeQuery.cpp | 2 +- .../QueryPlan/ParallelReplicasLocalPlan.cpp | 217 +++++++++++++++++- .../QueryPlan/ReadFromMergeTree.cpp | 45 +++- src/Processors/QueryPlan/ReadFromMergeTree.h | 12 +- src/Storages/MergeTree/RequestResponse.h | 1 - 5 files changed, 264 insertions(+), 13 deletions(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 71912fa1081..e12e531ab51 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -495,7 +495,7 @@ void executeQueryWithParallelReplicas( query_ast, new_cluster, storage_id, - std::move(coordinator), + coordinator, header, processed_stage, new_context, diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp index 64646960824..2bfd8965269 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -1,12 +1,32 @@ #include #include +#include "Storages/MergeTree/RequestResponse.h" #include #include #include #include #include #include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +namespace ProfileEvents +{ + extern const Event SelectedParts; + extern const Event SelectedRanges; + extern const Event SelectedMarks; +} namespace DB { @@ -40,13 +60,179 @@ void addConvertingActions(QueryPlan & plan, const Block & header, bool has_missi } +class ReadFromMergeTreeCoordinated : public ISourceStep +{ +public: + ReadFromMergeTreeCoordinated(QueryPlanStepPtr read_from_merge_tree_, ParallelReplicasReadingCoordinatorPtr coordinator_) + : ISourceStep(read_from_merge_tree_->getOutputStream()) + , read_from_merge_tree(std::move(read_from_merge_tree_)) + , coordinator(std::move(coordinator_)) + { + } + + void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) override; + String getName() const override { return "ReadFromLocalParallelReplica"; } + +private: + QueryPlanStepPtr read_from_merge_tree; + ParallelReplicasReadingCoordinatorPtr coordinator; +}; + +void ReadFromMergeTreeCoordinated::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & /*settings*/) +{ + ReadFromMergeTree & reading = *typeid_cast(read_from_merge_tree.get()); + + auto result = reading.getAnalysisResult(); + const auto & query_info = reading.getQueryInfo(); + const auto & data = reading.data; + const auto & context = reading.getContext(); + const auto & storage_snapshot = reading.getStorageSnapshot(); + + if (reading.enable_remove_parts_from_snapshot_optimization) + { + /// Do not keep data parts in snapshot. + /// They are stored separately, and some could be released after PK analysis. + reading.storage_snapshot->data = std::make_unique(); + } + + LOG_DEBUG( + reading.log, + "Selected {}/{} parts by partition key, {} parts by primary key, {}/{} marks by primary key, {} marks to read from {} ranges", + result.parts_before_pk, + result.total_parts, + result.selected_parts, + result.selected_marks_pk, + result.total_marks_pk, + result.selected_marks, + result.selected_ranges); + + // Adding partition info to QueryAccessInfo. + if (context->hasQueryContext() && !query_info.is_internal) + { + Names partition_names; + for (const auto & part : result.parts_with_ranges) + { + partition_names.emplace_back( + fmt::format("{}.{}", data.getStorageID().getFullNameNotQuoted(), part.data_part->info.partition_id)); + } + context->getQueryContext()->addQueryAccessInfo(partition_names); + + if (storage_snapshot->projection) + context->getQueryContext()->addQueryAccessInfo( + Context::QualifiedProjectionName{.storage_id = data.getStorageID(), .projection_name = storage_snapshot->projection->name}); + } + + ProfileEvents::increment(ProfileEvents::SelectedParts, result.selected_parts); + ProfileEvents::increment(ProfileEvents::SelectedRanges, result.selected_ranges); + ProfileEvents::increment(ProfileEvents::SelectedMarks, result.selected_marks); + + auto query_id_holder = MergeTreeDataSelectExecutor::checkLimits(data, result, context); + + // TODO: check this on plan level, we should be here if there is nothing to read + if (result.parts_with_ranges.empty()) + { + pipeline.init(Pipe(std::make_shared(getOutputStream().header))); + return; + } + + /// Projection, that needed to drop columns, which have appeared by execution + /// of some extra expressions, and to allow execute the same expressions later. + /// NOTE: It may lead to double computation of expressions. + ActionsDAGPtr result_projection; + + Pipe pipe = reading.spreadMarkRanges(std::move(result.parts_with_ranges), reading.requested_num_streams, result, result_projection); + + for (const auto & processor : pipe.getProcessors()) + processor->setStorageLimits(query_info.storage_limits); + + if (pipe.empty()) + { + pipeline.init(Pipe(std::make_shared(getOutputStream().header))); + return; + } + + if (result.sampling.use_sampling) + { + auto sampling_actions = std::make_shared(result.sampling.filter_expression); + pipe.addSimpleTransform([&](const Block & header) + { + return std::make_shared( + header, + sampling_actions, + result.sampling.filter_function->getColumnName(), + false); + }); + } + + Block cur_header = pipe.getHeader(); + + auto append_actions = [&result_projection](ActionsDAGPtr actions) + { + if (!result_projection) + result_projection = std::move(actions); + else + result_projection = ActionsDAG::merge(std::move(*result_projection), std::move(*actions)); + }; + + if (result_projection) + cur_header = result_projection->updateHeader(cur_header); + + /// Extra columns may be returned (for example, if sampling is used). + /// Convert pipe to step header structure. + if (!isCompatibleHeader(cur_header, getOutputStream().header)) + { + auto converting = ActionsDAG::makeConvertingActions( + cur_header.getColumnsWithTypeAndName(), + getOutputStream().header.getColumnsWithTypeAndName(), + ActionsDAG::MatchColumnsMode::Name); + + append_actions(std::move(converting)); + } + + if (result_projection) + { + auto projection_actions = std::make_shared(result_projection); + pipe.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, projection_actions); + }); + } + + /// Some extra columns could be added by sample/final/in-order/etc + /// Remove them from header if not needed. + if (!blocksHaveEqualStructure(pipe.getHeader(), getOutputStream().header)) + { + auto convert_actions_dag = ActionsDAG::makeConvertingActions( + pipe.getHeader().getColumnsWithTypeAndName(), + getOutputStream().header.getColumnsWithTypeAndName(), + ActionsDAG::MatchColumnsMode::Name, + true); + + auto converting_dag_expr = std::make_shared(convert_actions_dag); + + pipe.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, converting_dag_expr); + }); + } + + for (const auto & processor : pipe.getProcessors()) + processors.emplace_back(processor); + + pipeline.init(std::move(pipe)); + pipeline.addContext(context); + // Attach QueryIdHolder if needed + if (query_id_holder) + pipeline.setQueryIdHolder(std::move(query_id_holder)); +} + std::unique_ptr createLocalPlanForParallelReplicas( const ASTPtr & query_ast, const Block & header, ContextPtr context, QueryProcessingStage::Enum processed_stage, - ParallelReplicasReadingCoordinatorPtr /*coordinator*/, - QueryPlanStepPtr /*read_from_merge_tree*/, + ParallelReplicasReadingCoordinatorPtr coordinator, + QueryPlanStepPtr read_from_merge_tree, bool has_missing_objects) { checkStackSize(); @@ -72,6 +258,33 @@ std::unique_ptr createLocalPlanForParallelReplicas( auto interpreter = InterpreterSelectQueryAnalyzer(query_ast, new_context, select_query_options); query_plan = std::make_unique(std::move(interpreter).extractQueryPlan()); + QueryPlan::Node * node = query_plan->getRootNode(); + ReadFromMergeTree * reading = nullptr; + while (node) + { + reading = typeid_cast(node->step.get()); + if (reading) + break; + + if (!node->children.empty()) + node = node->children.at(0); + } + + chassert(reading); + + MergeTreeAllRangesCallback all_ranges_cb = [coordinator](InitialAllRangesAnnouncement announcement) + { + chassert(coordinator); + coordinator->handleInitialAllRangesAnnouncement(std::move(announcement)); + }; + + MergeTreeReadTaskCallback read_task_cb = [coordinator](ParallelReadRequest req) -> std::optional + { return coordinator->handleRequest(std::move(req)); }; + + const auto * analyzed_merge_tree = typeid_cast(read_from_merge_tree.get()); + auto read_from_merge_tree_parallel_replicas = reading->createLocalParallelReplicasReadingStep(analyzed_merge_tree, true, all_ranges_cb, read_task_cb); + node->step = std::move(read_from_merge_tree_parallel_replicas); + addConvertingActions(*query_plan, header, has_missing_objects); return query_plan; } diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 6f0fa55c349..21303cf2af2 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -272,7 +272,9 @@ ReadFromMergeTree::ReadFromMergeTree( std::shared_ptr max_block_numbers_to_read_, LoggerPtr log_, AnalysisResultPtr analyzed_result_ptr_, - bool enable_parallel_reading) + bool enable_parallel_reading_, + std::optional all_ranges_callback_, + std::optional read_task_callback_) : SourceStepWithFilter(DataStream{.header = MergeTreeSelectProcessor::transformHeader( storage_snapshot_->getSampleBlockForColumns(all_column_names_), query_info_.prewhere_info)}, all_column_names_, query_info_, storage_snapshot_, context_) @@ -291,13 +293,20 @@ ReadFromMergeTree::ReadFromMergeTree( , max_block_numbers_to_read(std::move(max_block_numbers_to_read_)) , log(std::move(log_)) , analyzed_result_ptr(analyzed_result_ptr_) - , is_parallel_reading_from_replicas(enable_parallel_reading) + , is_parallel_reading_from_replicas(enable_parallel_reading_) , enable_remove_parts_from_snapshot_optimization(query_info_.merge_tree_enable_remove_parts_from_snapshot_optimization) { if (is_parallel_reading_from_replicas) { - all_ranges_callback = context->getMergeTreeAllRangesCallback(); - read_task_callback = context->getMergeTreeReadTaskCallback(); + if (all_ranges_callback_) + all_ranges_callback = all_ranges_callback_.value(); + else + all_ranges_callback = context->getMergeTreeAllRangesCallback(); + + if (read_task_callback_) + read_task_callback = read_task_callback_.value(); + else + read_task_callback = context->getMergeTreeReadTaskCallback(); } const auto & settings = context->getSettingsRef(); @@ -331,11 +340,31 @@ ReadFromMergeTree::ReadFromMergeTree( enable_vertical_final); } +std::unique_ptr ReadFromMergeTree::createLocalParallelReplicasReadingStep( + const ReadFromMergeTree * analyzed_merge_tree, + bool enable_parallel_reading_, + std::optional all_ranges_callback_, + std::optional read_task_callback_) +{ + return std::make_unique( + prepared_parts, + alter_conversions_for_parts, + all_column_names, + data, + getQueryInfo(), + getStorageSnapshot(), + getContext(), + block_size.max_block_size_rows, + requested_num_streams, + max_block_numbers_to_read, + log, + analyzed_merge_tree->analyzed_result_ptr, + enable_parallel_reading_, + all_ranges_callback_, + read_task_callback_); +} -Pipe ReadFromMergeTree::readFromPoolParallelReplicas( - RangesInDataParts parts_with_range, - Names required_columns, - PoolSettings pool_settings) +Pipe ReadFromMergeTree::readFromPoolParallelReplicas(RangesInDataParts parts_with_range, Names required_columns, PoolSettings pool_settings) { const auto & client_info = context->getClientInfo(); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 5d7879e8dee..bba5293e863 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -119,7 +119,15 @@ public: std::shared_ptr max_block_numbers_to_read_, LoggerPtr log_, AnalysisResultPtr analyzed_result_ptr_, - bool enable_parallel_reading); + bool enable_parallel_reading_, + std::optional all_ranges_callback_ = std::nullopt, + std::optional read_task_callback_ = std::nullopt); + + std::unique_ptr createLocalParallelReplicasReadingStep( + const ReadFromMergeTree * analyzed_merge_tree, + bool enable_parallel_reading_, + std::optional all_ranges_callback_, + std::optional read_task_callback_); static constexpr auto name = "ReadFromMergeTree"; String getName() const override { return name; } @@ -282,6 +290,8 @@ private: std::optional read_task_callback; bool enable_vertical_final = false; bool enable_remove_parts_from_snapshot_optimization = true; + + friend class ReadFromMergeTreeCoordinated; }; } diff --git a/src/Storages/MergeTree/RequestResponse.h b/src/Storages/MergeTree/RequestResponse.h index 3a5bfde6c20..5f5516a6804 100644 --- a/src/Storages/MergeTree/RequestResponse.h +++ b/src/Storages/MergeTree/RequestResponse.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include From 29346f607398dba5cb7796d53dc525fd937af104 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 28 May 2024 12:16:18 +0000 Subject: [PATCH 004/215] Init coordinator separately --- src/Planner/PlannerJoinTree.cpp | 9 +++++--- .../QueryPlan/ParallelReplicasLocalPlan.cpp | 23 ++++++++++++++++++- src/Processors/QueryPlan/ReadFromMergeTree.h | 6 +++++ .../ParallelReplicasReadingCoordinator.h | 2 +- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 275461fa109..01455d4b955 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -892,16 +892,19 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres if (!node->children.empty()) node = node->children.at(0); + else + node = nullptr; } chassert(reading); + auto result_ptr = reading->selectRangesToRead(); + UInt64 rows_to_read = result_ptr->selected_rows; + reading->setAnalyzedResult(std::move(result_ptr)); + // (2) if it's ReadFromMergeTree - run index analysis and check number of rows to read if (settings.parallel_replicas_min_number_of_rows_per_replica > 0) { - auto result_ptr = reading->selectRangesToRead(); - - UInt64 rows_to_read = result_ptr->selected_rows; if (table_expression_query_info.limit > 0 && table_expression_query_info.limit < rows_to_read) rows_to_read = table_expression_query_info.limit; diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp index 2bfd8965269..1c5e0861530 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -268,10 +268,32 @@ std::unique_ptr createLocalPlanForParallelReplicas( if (!node->children.empty()) node = node->children.at(0); + else + node = nullptr; } chassert(reading); + const auto * analyzed_merge_tree = typeid_cast(read_from_merge_tree.get()); + if (!analyzed_merge_tree->hasAnalyzedResult()) + analyzed_merge_tree->selectRangesToRead(); + + switch (analyzed_merge_tree->getReadType()) + { + case ReadFromMergeTree::ReadType::Default: + coordinator->initialize(CoordinationMode::Default); + break; + case ReadFromMergeTree::ReadType::InOrder: + coordinator->initialize(CoordinationMode::WithOrder); + break; + case ReadFromMergeTree::ReadType::InReverseOrder: + coordinator->initialize(CoordinationMode::ReverseOrder); + break; + case ReadFromMergeTree::ReadType::ParallelReplicas: + chassert(false); + UNREACHABLE(); + } + MergeTreeAllRangesCallback all_ranges_cb = [coordinator](InitialAllRangesAnnouncement announcement) { chassert(coordinator); @@ -281,7 +303,6 @@ std::unique_ptr createLocalPlanForParallelReplicas( MergeTreeReadTaskCallback read_task_cb = [coordinator](ParallelReadRequest req) -> std::optional { return coordinator->handleRequest(std::move(req)); }; - const auto * analyzed_merge_tree = typeid_cast(read_from_merge_tree.get()); auto read_from_merge_tree_parallel_replicas = reading->createLocalParallelReplicasReadingStep(analyzed_merge_tree, true, all_ranges_cb, read_task_cb); node->step = std::move(read_from_merge_tree_parallel_replicas); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index bba5293e863..adc3818c3ab 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -203,6 +203,12 @@ public: void applyFilters(ActionDAGNodes added_filter_nodes) override; + ReadType getReadType() const + { + chassert(analyzed_result_ptr); + return analyzed_result_ptr->read_type; + } + private: static AnalysisResultPtr selectRangesToReadImpl( MergeTreeData::DataPartsVector parts, diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h index 60343988f03..ff72decbf8d 100644 --- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h +++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h @@ -30,8 +30,8 @@ public: /// needed to report total rows to read void setProgressCallback(ProgressCallback callback); -private: void initialize(CoordinationMode mode); +private: std::mutex mutex; size_t replicas_count{0}; From 1aa5d70d2a65b860f0d469a6c7c00b9f4129b07d Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 28 May 2024 13:03:24 +0000 Subject: [PATCH 005/215] Fix style --- src/Interpreters/ClusterProxy/executeQuery.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index e12e531ab51..3d08219155f 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -530,7 +530,8 @@ void executeQueryWithParallelReplicas( auto union_step = std::make_unique(std::move(input_streams)); query_plan.unitePlans(std::move(union_step), std::move(plans)); } - else { + else + { auto read_from_remote = std::make_unique( query_ast, new_cluster, From a9b485a2c1b8eb85fa04a5fef50b1258b0b63102 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 28 May 2024 19:05:09 +0000 Subject: [PATCH 006/215] Disable temporary PR inorder test --- .../02898_parallel_replicas_progress_bar.reference | 6 ------ .../02898_parallel_replicas_progress_bar.sql | 10 +++++----- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.reference b/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.reference index c66597436f3..380aac4dbe8 100644 --- a/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.reference +++ b/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.reference @@ -1,8 +1,2 @@ 3000 1000 3999 2499.5 1 -1998 2944475297004403859 -1999 254596732598015005 -2000 6863370867519437063 -2001 17844331710293705251 -2002 1587587338113897332 -1 diff --git a/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.sql b/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.sql index d8bfec12b3a..42f8091db08 100644 --- a/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.sql +++ b/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.sql @@ -26,12 +26,12 @@ WHERE query_id in (select query_id from system.query_log where current_database AND message LIKE '%Total rows to read: 3000%' SETTINGS allow_experimental_parallel_reading_from_replicas=0; -- reading in order coordinator -SELECT k, sipHash64(v) FROM t1 order by k limit 5 offset 998 SETTINGS optimize_read_in_order=1, log_comment='02898_inorder_190aed82-2423-413b-ad4c-24dcca50f65b'; +-- SELECT k, sipHash64(v) FROM t1 order by k limit 5 offset 998 SETTINGS optimize_read_in_order=1, log_comment='02898_inorder_190aed82-2423-413b-ad4c-24dcca50f65b'; -SYSTEM FLUSH LOGS; -SELECT count() > 0 FROM system.text_log -WHERE query_id in (select query_id from system.query_log where current_database = currentDatabase() AND log_comment='02898_inorder_190aed82-2423-413b-ad4c-24dcca50f65b') - AND message LIKE '%Updated total rows to read: added % rows, total 3000 rows%' SETTINGS allow_experimental_parallel_reading_from_replicas=0; +-- SYSTEM FLUSH LOGS; +-- SELECT count() > 0 FROM system.text_log +-- WHERE query_id in (select query_id from system.query_log where current_database = currentDatabase() AND log_comment='02898_inorder_190aed82-2423-413b-ad4c-24dcca50f65b') +-- AND message LIKE '%Updated total rows to read: added % rows, total 3000 rows%' SETTINGS allow_experimental_parallel_reading_from_replicas=0; DROP TABLE t1 SYNC; DROP TABLE t2 SYNC; From d756729f384ff9a6fbf93f12934ed0f98b54b7a6 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 28 May 2024 21:04:33 +0000 Subject: [PATCH 007/215] Use local working set for parallel replicas --- .../QueryPlan/ParallelReplicasLocalPlan.cpp | 42 +++++++++---------- .../QueryPlan/ReadFromMergeTree.cpp | 29 +++++-------- src/Processors/QueryPlan/ReadFromMergeTree.h | 10 +++-- src/Processors/QueryPlan/ReadFromRemote.cpp | 19 ++++++--- .../MergeTree/MergeTreeSelectProcessor.h | 5 --- 5 files changed, 52 insertions(+), 53 deletions(-) diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp index 1c5e0861530..48184cb2b5f 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -273,37 +273,37 @@ std::unique_ptr createLocalPlanForParallelReplicas( } chassert(reading); - const auto * analyzed_merge_tree = typeid_cast(read_from_merge_tree.get()); - if (!analyzed_merge_tree->hasAnalyzedResult()) - analyzed_merge_tree->selectRangesToRead(); + chassert(analyzed_merge_tree->hasAnalyzedResult()); + CoordinationMode mode = CoordinationMode::Default; switch (analyzed_merge_tree->getReadType()) { - case ReadFromMergeTree::ReadType::Default: - coordinator->initialize(CoordinationMode::Default); - break; - case ReadFromMergeTree::ReadType::InOrder: - coordinator->initialize(CoordinationMode::WithOrder); - break; - case ReadFromMergeTree::ReadType::InReverseOrder: - coordinator->initialize(CoordinationMode::ReverseOrder); - break; - case ReadFromMergeTree::ReadType::ParallelReplicas: - chassert(false); - UNREACHABLE(); + case ReadFromMergeTree::ReadType::Default: + mode = CoordinationMode::Default; + break; + case ReadFromMergeTree::ReadType::InOrder: + mode = CoordinationMode::WithOrder; + break; + case ReadFromMergeTree::ReadType::InReverseOrder: + mode = CoordinationMode::ReverseOrder; + break; + case ReadFromMergeTree::ReadType::ParallelReplicas: + chassert(false); + UNREACHABLE(); } - MergeTreeAllRangesCallback all_ranges_cb = [coordinator](InitialAllRangesAnnouncement announcement) - { - chassert(coordinator); - coordinator->handleInitialAllRangesAnnouncement(std::move(announcement)); - }; + const auto number_of_local_replica = new_context->getSettingsRef().max_parallel_replicas - 1; + coordinator->handleInitialAllRangesAnnouncement(InitialAllRangesAnnouncement( + mode, analyzed_merge_tree->getAnalysisResult().parts_with_ranges.getDescriptions(), number_of_local_replica)); + + MergeTreeAllRangesCallback all_ranges_cb = [coordinator](InitialAllRangesAnnouncement) {}; MergeTreeReadTaskCallback read_task_cb = [coordinator](ParallelReadRequest req) -> std::optional { return coordinator->handleRequest(std::move(req)); }; - auto read_from_merge_tree_parallel_replicas = reading->createLocalParallelReplicasReadingStep(analyzed_merge_tree, true, all_ranges_cb, read_task_cb); + auto read_from_merge_tree_parallel_replicas + = reading->createLocalParallelReplicasReadingStep(analyzed_merge_tree, true, all_ranges_cb, read_task_cb, number_of_local_replica); node->step = std::move(read_from_merge_tree_parallel_replicas); addConvertingActions(*query_plan, header, has_missing_objects); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 198236c4f49..e3e09673431 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -273,8 +273,9 @@ ReadFromMergeTree::ReadFromMergeTree( LoggerPtr log_, AnalysisResultPtr analyzed_result_ptr_, bool enable_parallel_reading_, - std::optional all_ranges_callback_, - std::optional read_task_callback_) + std::optional all_ranges_callback_, + std::optional read_task_callback_, + std::optional number_of_current_replica_) : SourceStepWithFilter(DataStream{.header = MergeTreeSelectProcessor::transformHeader( storage_snapshot_->getSampleBlockForColumns(all_column_names_), query_info_.prewhere_info)}, all_column_names_, query_info_, storage_snapshot_, context_) @@ -295,18 +296,12 @@ ReadFromMergeTree::ReadFromMergeTree( , analyzed_result_ptr(analyzed_result_ptr_) , is_parallel_reading_from_replicas(enable_parallel_reading_) , enable_remove_parts_from_snapshot_optimization(query_info_.merge_tree_enable_remove_parts_from_snapshot_optimization) + , number_of_current_replica(number_of_current_replica_) { if (is_parallel_reading_from_replicas) { - if (all_ranges_callback_) - all_ranges_callback = all_ranges_callback_.value(); - else - all_ranges_callback = context->getMergeTreeAllRangesCallback(); - - if (read_task_callback_) - read_task_callback = read_task_callback_.value(); - else - read_task_callback = context->getMergeTreeReadTaskCallback(); + all_ranges_callback = all_ranges_callback_.value_or(context->getMergeTreeAllRangesCallback()); + read_task_callback = read_task_callback_.value_or(context->getMergeTreeReadTaskCallback()); } const auto & settings = context->getSettingsRef(); @@ -344,7 +339,8 @@ std::unique_ptr ReadFromMergeTree::createLocalParallelReplica const ReadFromMergeTree * analyzed_merge_tree, bool enable_parallel_reading_, std::optional all_ranges_callback_, - std::optional read_task_callback_) + std::optional read_task_callback_, + std::optional number_of_current_replica_) { return std::make_unique( prepared_parts, @@ -361,7 +357,8 @@ std::unique_ptr ReadFromMergeTree::createLocalParallelReplica analyzed_merge_tree->analyzed_result_ptr, enable_parallel_reading_, all_ranges_callback_, - read_task_callback_); + read_task_callback_, + number_of_current_replica_); } Pipe ReadFromMergeTree::readFromPoolParallelReplicas(RangesInDataParts parts_with_range, Names required_columns, PoolSettings pool_settings) @@ -372,9 +369,7 @@ Pipe ReadFromMergeTree::readFromPoolParallelReplicas(RangesInDataParts parts_wit { .all_callback = all_ranges_callback.value(), .callback = read_task_callback.value(), - .count_participating_replicas = client_info.count_participating_replicas, - .number_of_current_replica = client_info.number_of_current_replica, - .columns_to_read = required_columns, + .number_of_current_replica = number_of_current_replica.value_or(client_info.number_of_current_replica), }; /// We have a special logic for local replica. It has to read less data, because in some cases it should @@ -545,9 +540,7 @@ Pipe ReadFromMergeTree::readInOrder( { .all_callback = all_ranges_callback.value(), .callback = read_task_callback.value(), - .count_participating_replicas = client_info.count_participating_replicas, .number_of_current_replica = client_info.number_of_current_replica, - .columns_to_read = required_columns, }; const auto multiplier = context->getSettingsRef().parallel_replicas_single_task_marks_count_multiplier; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index adc3818c3ab..8cabc6822c1 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -121,13 +121,15 @@ public: AnalysisResultPtr analyzed_result_ptr_, bool enable_parallel_reading_, std::optional all_ranges_callback_ = std::nullopt, - std::optional read_task_callback_ = std::nullopt); + std::optional read_task_callback_ = std::nullopt, + std::optional number_of_current_replica_ = std::nullopt); std::unique_ptr createLocalParallelReplicasReadingStep( const ReadFromMergeTree * analyzed_merge_tree, bool enable_parallel_reading_, std::optional all_ranges_callback_, - std::optional read_task_callback_); + std::optional read_task_callback_, + std::optional number_of_current_replica_); static constexpr auto name = "ReadFromMergeTree"; String getName() const override { return name; } @@ -192,6 +194,7 @@ public: bool hasAnalyzedResult() const { return analyzed_result_ptr != nullptr; } void setAnalyzedResult(AnalysisResultPtr analyzed_result_ptr_) { analyzed_result_ptr = std::move(analyzed_result_ptr_); } + ReadFromMergeTree::AnalysisResult getAnalysisResult() const; const MergeTreeData::DataPartsVector & getParts() const { return prepared_parts; } const std::vector & getAlterConvertionsForParts() const { return alter_conversions_for_parts; } @@ -286,8 +289,6 @@ private: Pipe spreadMarkRangesAmongStreamsFinal( RangesInDataParts && parts, size_t num_streams, const Names & origin_column_names, const Names & column_names, ActionsDAGPtr & out_projection); - ReadFromMergeTree::AnalysisResult getAnalysisResult() const; - AnalysisResultPtr analyzed_result_ptr; VirtualFields shared_virtual_fields; @@ -296,6 +297,7 @@ private: std::optional read_task_callback; bool enable_vertical_final = false; bool enable_remove_parts_from_snapshot_optimization = true; + std::optional number_of_current_replica; friend class ReadFromMergeTreeCoordinated; }; diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 485d5f675ab..23d8f2b496f 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -390,8 +390,18 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep( std::vector description; description.push_back(fmt::format("query: {}", formattedAST(query_ast))); - for (const auto & pool : cluster->getShardsInfo().front().per_replica_pools) - description.push_back(fmt::format("Replica: {}", pool->getHost())); + bool first_local = false; + for (const auto & addr : cluster->getShardsAddresses().front()) + { + /// skip first local + if (exclude_local_replica && addr.is_local && !first_local) + { + first_local = true; + continue; + } + + description.push_back(fmt::format("Replica: {}", addr.host_name)); + } setStepDescription(boost::algorithm::join(description, ", ")); } @@ -414,9 +424,6 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder const auto & shard = cluster->getShardsInfo().at(0); size_t all_replicas_count = current_settings.max_parallel_replicas; - if (exclude_local_replica) - --all_replicas_count; - if (all_replicas_count > shard.getAllNodeCount()) { LOG_INFO( @@ -427,6 +434,8 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder shard.getAllNodeCount()); all_replicas_count = shard.getAllNodeCount(); } + if (exclude_local_replica) + --all_replicas_count; std::vector shuffled_pool; if (all_replicas_count < shard.getAllNodeCount()) diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 8f41f5deacb..03ca30dd5b3 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -26,12 +26,7 @@ struct ParallelReadingExtension { MergeTreeAllRangesCallback all_callback; MergeTreeReadTaskCallback callback; - size_t count_participating_replicas{0}; size_t number_of_current_replica{0}; - /// This is needed to estimate the number of bytes - /// between a pair of marks to perform one request - /// over the network for a 1Gb of data. - Names columns_to_read; }; /// Base class for MergeTreeThreadSelectAlgorithm and MergeTreeSelectAlgorithm From 40aab93db123840042a0f5f4703ae26f4e2ab507 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 29 May 2024 20:55:02 +0000 Subject: [PATCH 008/215] Correct replica id for inorder case --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index e3e09673431..9516072f269 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -540,7 +540,7 @@ Pipe ReadFromMergeTree::readInOrder( { .all_callback = all_ranges_callback.value(), .callback = read_task_callback.value(), - .number_of_current_replica = client_info.number_of_current_replica, + .number_of_current_replica = number_of_current_replica.value_or(client_info.number_of_current_replica), }; const auto multiplier = context->getSettingsRef().parallel_replicas_single_task_marks_count_multiplier; From e8a1a800dcdc44001c6c9f08f78390df9b3133e3 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 29 May 2024 20:57:16 +0000 Subject: [PATCH 009/215] Fix crash with JOINs --- src/Interpreters/ClusterProxy/executeQuery.cpp | 8 ++++---- src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp | 9 ++++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 464ce2ec586..0eeae112062 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -405,7 +405,7 @@ void executeQueryWithParallelReplicas( const ASTPtr & query_ast, ContextPtr context, std::shared_ptr storage_limits, - QueryPlanStepPtr read_from_merge_tree) + QueryPlanStepPtr analyzed_read_from_merge_tree) { auto logger = getLogger("executeQueryWithParallelReplicas"); LOG_DEBUG(logger, "Executing read from {}, header {}, query ({}), stage {} with parallel replicas", @@ -519,7 +519,7 @@ void executeQueryWithParallelReplicas( new_context, processed_stage, coordinator, - std::move(read_from_merge_tree), + std::move(analyzed_read_from_merge_tree), /*has_missing_objects=*/false); DataStreams input_streams; @@ -563,7 +563,7 @@ void executeQueryWithParallelReplicas( const PlannerContextPtr & planner_context, ContextPtr context, std::shared_ptr storage_limits, - QueryPlanStepPtr read_from_merge_tree) + QueryPlanStepPtr analyzed_read_from_merge_tree) { QueryTreeNodePtr modified_query_tree = query_tree->clone(); rewriteJoinToGlobalJoin(modified_query_tree, context); @@ -574,7 +574,7 @@ void executeQueryWithParallelReplicas( auto modified_query_ast = queryNodeToDistributedSelectQuery(modified_query_tree); executeQueryWithParallelReplicas( - query_plan, storage_id, header, processed_stage, modified_query_ast, context, storage_limits, std::move(read_from_merge_tree)); + query_plan, storage_id, header, processed_stage, modified_query_ast, context, storage_limits, std::move(analyzed_read_from_merge_tree)); } void executeQueryWithParallelReplicas( diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp index 48184cb2b5f..e847a0ece26 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -273,7 +273,14 @@ std::unique_ptr createLocalPlanForParallelReplicas( } chassert(reading); - const auto * analyzed_merge_tree = typeid_cast(read_from_merge_tree.get()); + if (!read_from_merge_tree) + read_from_merge_tree = std::move(node->step); + + auto * analyzed_merge_tree = typeid_cast(read_from_merge_tree.get()); + /// if no analysis is done yet, let's do it (happens with JOINs) + if (!analyzed_merge_tree->hasAnalyzedResult()) + analyzed_merge_tree->setAnalyzedResult(analyzed_merge_tree->selectRangesToRead()); + chassert(analyzed_merge_tree->hasAnalyzedResult()); CoordinationMode mode = CoordinationMode::Default; From eece76bc412b4973d7f13a19c46e0f24c5580d02 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Sat, 1 Jun 2024 20:53:33 +0000 Subject: [PATCH 010/215] Fix build --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 1b57515d850..a969b84dbe8 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -540,11 +540,7 @@ Pipe ReadFromMergeTree::readInOrder( { .all_callback = all_ranges_callback.value(), .callback = read_task_callback.value(), -<<<<<<< HEAD .number_of_current_replica = number_of_current_replica.value_or(client_info.number_of_current_replica), -======= - .number_of_current_replica = client_info.number_of_current_replica, ->>>>>>> origin/pr-plan-rewrite }; const auto multiplier = context->getSettingsRef().parallel_replicas_single_task_marks_count_multiplier; From 6476e9ad9bb2557f5010e4358e4881c80281a18c Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 5 Jun 2024 06:45:56 +0000 Subject: [PATCH 011/215] Temporary test fix --- .../queries/0_stateless/02841_parallel_replicas_summary.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02841_parallel_replicas_summary.sh b/tests/queries/0_stateless/02841_parallel_replicas_summary.sh index c82d2c8b0c0..bff56feacbf 100755 --- a/tests/queries/0_stateless/02841_parallel_replicas_summary.sh +++ b/tests/queries/0_stateless/02841_parallel_replicas_summary.sh @@ -27,6 +27,7 @@ $CLICKHOUSE_CLIENT --query "CREATE TABLE replicas_summary (n Int64) ENGINE = Mer query_id_base="02841_summary_$CLICKHOUSE_DATABASE" +# TODO: rethink the test, for now temporary disable allow_experimental_analyzer echo " SELECT * FROM replicas_summary @@ -36,7 +37,8 @@ echo " cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_parallel_reading_from_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, - interactive_delay=0 + interactive_delay=0, + allow_experimental_analyzer=0 "\ | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&wait_end_of_query=1&query_id=${query_id_base}_interactive_0" --data-binary @- -vvv 2>&1 \ | grep "Summary" | grep -cv '"read_rows":"0"' @@ -50,7 +52,8 @@ echo " cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_parallel_reading_from_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, - interactive_delay=99999999999 + interactive_delay=99999999999, + allow_experimental_analyzer=0 "\ | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&wait_end_of_query=1&query_id=${query_id_base}_interactive_high" --data-binary @- -vvv 2>&1 \ | grep "Summary" | grep -cv '"read_rows":"0"' From 3210d0f4716b5ac820d6f267437d13bf9fb02011 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 5 Jun 2024 11:56:12 +0000 Subject: [PATCH 012/215] Simple renaming --- src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp index e847a0ece26..8ea826b861c 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -232,7 +232,7 @@ std::unique_ptr createLocalPlanForParallelReplicas( ContextPtr context, QueryProcessingStage::Enum processed_stage, ParallelReplicasReadingCoordinatorPtr coordinator, - QueryPlanStepPtr read_from_merge_tree, + QueryPlanStepPtr analyzed_read_from_merge_tree, bool has_missing_objects) { checkStackSize(); @@ -273,10 +273,10 @@ std::unique_ptr createLocalPlanForParallelReplicas( } chassert(reading); - if (!read_from_merge_tree) - read_from_merge_tree = std::move(node->step); + if (!analyzed_read_from_merge_tree) + analyzed_read_from_merge_tree = std::move(node->step); - auto * analyzed_merge_tree = typeid_cast(read_from_merge_tree.get()); + auto * analyzed_merge_tree = typeid_cast(analyzed_read_from_merge_tree.get()); /// if no analysis is done yet, let's do it (happens with JOINs) if (!analyzed_merge_tree->hasAnalyzedResult()) analyzed_merge_tree->setAnalyzedResult(analyzed_merge_tree->selectRangesToRead()); From f8d4aabfe0df44638184017ee90bfbe63ee79e90 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 6 Jun 2024 14:28:17 +0000 Subject: [PATCH 013/215] Initiliaze working set on pipelie initialization, right after analysis --- src/Planner/PlannerJoinTree.cpp | 8 ++-- .../QueryPlan/ParallelReplicasLocalPlan.cpp | 39 ++++--------------- .../QueryPlan/ReadFromMergeTree.cpp | 39 +++++++++++++++---- src/Processors/QueryPlan/ReadFromMergeTree.h | 6 ++- 4 files changed, 48 insertions(+), 44 deletions(-) diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 25206763a40..26e78ea69ac 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -908,13 +908,13 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres chassert(reading); - auto result_ptr = reading->selectRangesToRead(); - UInt64 rows_to_read = result_ptr->selected_rows; - reading->setAnalyzedResult(std::move(result_ptr)); - // (2) if it's ReadFromMergeTree - run index analysis and check number of rows to read if (settings.parallel_replicas_min_number_of_rows_per_replica > 0) { + auto result_ptr = reading->selectRangesToRead(); + UInt64 rows_to_read = result_ptr->selected_rows; + reading->setAnalyzedResult(std::move(result_ptr)); + if (table_expression_query_info.limit > 0 && table_expression_query_info.limit < rows_to_read) rows_to_read = table_expression_query_info.limit; diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp index 8ea826b861c..27d86f656c5 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -273,44 +273,21 @@ std::unique_ptr createLocalPlanForParallelReplicas( } chassert(reading); - if (!analyzed_read_from_merge_tree) - analyzed_read_from_merge_tree = std::move(node->step); - auto * analyzed_merge_tree = typeid_cast(analyzed_read_from_merge_tree.get()); - /// if no analysis is done yet, let's do it (happens with JOINs) - if (!analyzed_merge_tree->hasAnalyzedResult()) - analyzed_merge_tree->setAnalyzedResult(analyzed_merge_tree->selectRangesToRead()); + ReadFromMergeTree * analyzed_merge_tree = nullptr; + if (analyzed_read_from_merge_tree.get()) + analyzed_merge_tree = typeid_cast(analyzed_read_from_merge_tree.get()); - chassert(analyzed_merge_tree->hasAnalyzedResult()); - - CoordinationMode mode = CoordinationMode::Default; - switch (analyzed_merge_tree->getReadType()) - { - case ReadFromMergeTree::ReadType::Default: - mode = CoordinationMode::Default; - break; - case ReadFromMergeTree::ReadType::InOrder: - mode = CoordinationMode::WithOrder; - break; - case ReadFromMergeTree::ReadType::InReverseOrder: - mode = CoordinationMode::ReverseOrder; - break; - case ReadFromMergeTree::ReadType::ParallelReplicas: - chassert(false); - UNREACHABLE(); - } - - const auto number_of_local_replica = new_context->getSettingsRef().max_parallel_replicas - 1; - coordinator->handleInitialAllRangesAnnouncement(InitialAllRangesAnnouncement( - mode, analyzed_merge_tree->getAnalysisResult().parts_with_ranges.getDescriptions(), number_of_local_replica)); - - MergeTreeAllRangesCallback all_ranges_cb = [coordinator](InitialAllRangesAnnouncement) {}; + MergeTreeAllRangesCallback all_ranges_cb + = [coordinator](InitialAllRangesAnnouncement announcement) { coordinator->handleInitialAllRangesAnnouncement(announcement); }; MergeTreeReadTaskCallback read_task_cb = [coordinator](ParallelReadRequest req) -> std::optional { return coordinator->handleRequest(std::move(req)); }; + const auto number_of_local_replica = new_context->getSettingsRef().max_parallel_replicas - 1; + auto read_from_merge_tree_parallel_replicas - = reading->createLocalParallelReplicasReadingStep(analyzed_merge_tree, true, all_ranges_cb, read_task_cb, number_of_local_replica); + = reading->createLocalParallelReplicasReadingStep(analyzed_merge_tree, all_ranges_cb, read_task_cb, number_of_local_replica); node->step = std::move(read_from_merge_tree_parallel_replicas); addConvertingActions(*query_plan, header, has_missing_objects); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 7a13f072e4a..1aa48dbcff8 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -337,7 +337,6 @@ ReadFromMergeTree::ReadFromMergeTree( std::unique_ptr ReadFromMergeTree::createLocalParallelReplicasReadingStep( const ReadFromMergeTree * analyzed_merge_tree, - bool enable_parallel_reading_, std::optional all_ranges_callback_, std::optional read_task_callback_, std::optional number_of_current_replica_) @@ -354,8 +353,8 @@ std::unique_ptr ReadFromMergeTree::createLocalParallelReplica requested_num_streams, max_block_numbers_to_read, log, - analyzed_merge_tree->analyzed_result_ptr, - enable_parallel_reading_, + (analyzed_merge_tree ? analyzed_merge_tree->analyzed_result_ptr : nullptr), + true, all_ranges_callback_, read_task_callback_, number_of_current_replica_); @@ -1424,11 +1423,8 @@ static void buildIndexes( const auto & settings = context->getSettingsRef(); - indexes.emplace(ReadFromMergeTree::Indexes{{ - filter_actions_dag, - context, - primary_key_column_names, - primary_key.expression}, {}, {}, {}, {}, false, {}}); + indexes.emplace( + ReadFromMergeTree::Indexes{KeyCondition{filter_actions_dag, context, primary_key_column_names, primary_key.expression}}); if (metadata_snapshot->hasPartitionKey()) { @@ -1951,6 +1947,33 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons { auto result = getAnalysisResult(); + if (is_parallel_reading_from_replicas && context->canUseParallelReplicasOnInitiator()) + { + CoordinationMode mode = CoordinationMode::Default; + switch (result.read_type) + { + case ReadFromMergeTree::ReadType::Default: + mode = CoordinationMode::Default; + break; + case ReadFromMergeTree::ReadType::InOrder: + mode = CoordinationMode::WithOrder; + break; + case ReadFromMergeTree::ReadType::InReverseOrder: + mode = CoordinationMode::ReverseOrder; + break; + case ReadFromMergeTree::ReadType::ParallelReplicas: + chassert(false); + UNREACHABLE(); + } + + chassert(number_of_current_replica.has_value()); + chassert(all_ranges_callback.has_value()); + + /// initialize working set from local replica + all_ranges_callback.value()( + InitialAllRangesAnnouncement(mode, result.parts_with_ranges.getDescriptions(), number_of_current_replica.value())); + } + if (enable_remove_parts_from_snapshot_optimization) { /// Do not keep data parts in snapshot. diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index b541481fa62..dc1d20cc807 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -126,7 +126,6 @@ public: std::unique_ptr createLocalParallelReplicasReadingStep( const ReadFromMergeTree * analyzed_merge_tree, - bool enable_parallel_reading_, std::optional all_ranges_callback_, std::optional read_task_callback_, std::optional number_of_current_replica_); @@ -151,6 +150,11 @@ public: struct Indexes { + explicit Indexes(KeyCondition key_condition_) + : key_condition(std::move(key_condition_)) + , use_skip_indexes(false) + {} + KeyCondition key_condition; std::optional partition_pruner; std::optional minmax_idx_condition; From 4f37fafe426bbefe55aabc36faf386dc047e8517 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 6 Jun 2024 15:57:12 +0000 Subject: [PATCH 014/215] Fix 02771_parallel_replicas_analyzer --- .../0_stateless/02771_parallel_replicas_analyzer.reference | 2 +- .../queries/0_stateless/02771_parallel_replicas_analyzer.sql | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference b/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference index 5bf3520ccdb..f87b3244f09 100644 --- a/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference +++ b/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference @@ -9,4 +9,4 @@ 7885388429666205427 8124171311239967992 1 1 -- Simple query with analyzer and pure parallel replicas\nSELECT number\nFROM join_inner_table__fuzz_146_replicated\n SETTINGS\n allow_experimental_analyzer = 1,\n max_parallel_replicas = 2,\n cluster_for_parallel_replicas = \'test_cluster_one_shard_three_replicas_localhost\',\n allow_experimental_parallel_reading_from_replicas = 1; -0 2 SELECT `__table1`.`number` AS `number` FROM `default`.`join_inner_table__fuzz_146_replicated` AS `__table1` +0 1 SELECT `__table1`.`number` AS `number` FROM `default`.`join_inner_table__fuzz_146_replicated` AS `__table1` diff --git a/tests/queries/0_stateless/02771_parallel_replicas_analyzer.sql b/tests/queries/0_stateless/02771_parallel_replicas_analyzer.sql index 88a0d2163d6..7e27507ada9 100644 --- a/tests/queries/0_stateless/02771_parallel_replicas_analyzer.sql +++ b/tests/queries/0_stateless/02771_parallel_replicas_analyzer.sql @@ -1,5 +1,5 @@ -- Tags: zookeeper - +DROP TABLE IF EXISTS join_inner_table__fuzz_146_replicated; CREATE TABLE join_inner_table__fuzz_146_replicated ( `id` UUID, @@ -49,3 +49,5 @@ WHERE ) GROUP BY is_initial_query, query ORDER BY is_initial_query DESC, c, query; + +DROP TABLE join_inner_table__fuzz_146_replicated; From 7be90470d5c33877426cfc22cb5b3ad949a523ad Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 6 Jun 2024 16:49:05 +0000 Subject: [PATCH 015/215] Fix 02731_parallel_replicas_join_subquery --- .../02731_parallel_replicas_join_subquery.reference | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference b/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference index 451f0d6d485..f0d096d2072 100644 --- a/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference +++ b/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference @@ -31,7 +31,7 @@ 29 2j&S)ba?XG QuQj 17163829389637435056 3 UlI+1 14144472852965836438 =============== QUERIES EXECUTED BY PARALLEL INNER QUERY ALONE =============== -0 3 SELECT `__table1`.`key` AS `key`, `__table1`.`value1` AS `value1`, `__table1`.`value2` AS `value2`, toUInt64(min(`__table1`.`time`)) AS `start_ts` FROM `default`.`join_inner_table` AS `__table1` PREWHERE (`__table1`.`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`__table1`.`number` > _CAST(1610517366120, \'UInt64\')) GROUP BY `__table1`.`key`, `__table1`.`value1`, `__table1`.`value2` ORDER BY `__table1`.`key` ASC, `__table1`.`value1` ASC, `__table1`.`value2` ASC LIMIT _CAST(10, \'UInt64\') +0 2 SELECT `__table1`.`key` AS `key`, `__table1`.`value1` AS `value1`, `__table1`.`value2` AS `value2`, toUInt64(min(`__table1`.`time`)) AS `start_ts` FROM `default`.`join_inner_table` AS `__table1` PREWHERE (`__table1`.`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`__table1`.`number` > _CAST(1610517366120, \'UInt64\')) GROUP BY `__table1`.`key`, `__table1`.`value1`, `__table1`.`value2` ORDER BY `__table1`.`key` ASC, `__table1`.`value1` ASC, `__table1`.`value2` ASC LIMIT _CAST(10, \'UInt64\') 0 3 SELECT `key`, `value1`, `value2`, toUInt64(min(`time`)) AS `start_ts` FROM `default`.`join_inner_table` PREWHERE (`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`number` > toUInt64(\'1610517366120\')) GROUP BY `key`, `value1`, `value2` ORDER BY `key` ASC, `value1` ASC, `value2` ASC LIMIT 10 1 1 -- Parallel inner query alone\nSELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\nFROM join_inner_table\nPREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\nGROUP BY key, value1, value2\nORDER BY key, value1, value2\nLIMIT 10\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, allow_experimental_analyzer=0; 1 1 -- Parallel inner query alone\nSELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\nFROM join_inner_table\nPREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\nGROUP BY key, value1, value2\nORDER BY key, value1, value2\nLIMIT 10\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, allow_experimental_analyzer=1; @@ -58,7 +58,7 @@ U c 10 UlI+1 10 bX?}ix [ Ny]2 G 10 t _CAST(1610517366120, \'UInt64\')) GROUP BY `__table3`.`key`, `__table3`.`value1`, `__table3`.`value2`) AS `__table2` USING (`key`) GROUP BY `__table1`.`key`, `__table2`.`value1`, `__table2`.`value2` +0 2 SELECT `__table2`.`value1` AS `value1`, `__table2`.`value2` AS `value2`, count() AS `count` FROM `default`.`join_outer_table` AS `__table1` ALL INNER JOIN (SELECT `__table3`.`key` AS `key`, `__table3`.`value1` AS `value1`, `__table3`.`value2` AS `value2` FROM `default`.`join_inner_table` AS `__table3` PREWHERE (`__table3`.`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`__table3`.`number` > _CAST(1610517366120, \'UInt64\')) GROUP BY `__table3`.`key`, `__table3`.`value1`, `__table3`.`value2`) AS `__table2` USING (`key`) GROUP BY `__table1`.`key`, `__table2`.`value1`, `__table2`.`value2` 0 3 SELECT `key`, `value1`, `value2` FROM `default`.`join_inner_table` PREWHERE (`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`number` > toUInt64(\'1610517366120\')) GROUP BY `key`, `value1`, `value2` 0 3 SELECT `value1`, `value2`, count() AS `count` FROM `default`.`join_outer_table` ALL INNER JOIN `_data_` USING (`key`) GROUP BY `key`, `value1`, `value2` 1 1 -- Parallel full query\nSELECT\n value1,\n value2,\n avg(count) AS avg\nFROM\n (\n SELECT\n key,\n value1,\n value2,\n count() AS count\n FROM join_outer_table\n INNER JOIN\n (\n SELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\n FROM join_inner_table\n PREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\n GROUP BY key, value1, value2\n ) USING (key)\n GROUP BY key, value1, value2\n )\nGROUP BY value1, value2\nORDER BY value1, value2\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, allow_experimental_analyzer=0; From 44fecf66ca3cd1969b2fd03d30861dac92df5ed0 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 6 Jun 2024 17:11:42 +0000 Subject: [PATCH 016/215] Fix 02967_parallel_replicas_joins_and_analyzer --- ...llel_replicas_joins_and_analyzer.reference | 365 +++++++++++++++--- 1 file changed, 303 insertions(+), 62 deletions(-) diff --git a/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.reference b/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.reference index 100e4e500cd..baec06244e2 100644 --- a/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.reference +++ b/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.reference @@ -19,8 +19,20 @@ select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x 14 14 14 14 0 0 15 15 0 0 0 0 explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1; -Expression - ReadFromRemoteParallelReplicas +Union + Expression + Join + Expression + Join + Expression + Expression + ReadFromMergeTree + Expression + ReadFromMemoryStorage + Expression + ReadFromMemoryStorage + Expression + ReadFromRemoteParallelReplicas -- -- The same query with cte; with sub1 as (select x, y from tab1 where x != 2), @@ -55,8 +67,22 @@ select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1; Expression Sorting - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Sorting + Expression + Join + Expression + Join + Expression + Expression + ReadFromMergeTree + Expression + ReadFromMemoryStorage + Expression + ReadFromMemoryStorage + Expression + ReadFromRemoteParallelReplicas -- -- GROUP BY should work up to WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -80,8 +106,22 @@ Expression Sorting Expression MergingAggregated - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Aggregating + Expression + Join + Expression + Join + Expression + Expression + ReadFromMergeTree + Expression + ReadFromMemoryStorage + Expression + ReadFromMemoryStorage + Expression + ReadFromRemoteParallelReplicas -- -- ORDER BY in sub3 : sub1 is fully pushed, sub3 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -118,10 +158,22 @@ Expression Sorting Expression Join - Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Join + Expression + Expression + ReadFromMergeTree + Expression + ReadFromMemoryStorage + Expression + ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- ORDER BY in sub1 : sub1 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2 order by y), @@ -160,12 +212,24 @@ Expression Join Expression Join + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- RIGHT JOIN in sub3: sub3 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -202,12 +266,24 @@ Expression Join Expression Join + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- RIGHT JOIN in sub5: sub5 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -242,14 +318,26 @@ Expression Sorting Expression Join - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas Expression Join - Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- Subqueries for IN allowed with sub1 as (select x, y from tab1 where x in (select number from numbers(16) where number != 2)), @@ -284,8 +372,28 @@ select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1; Expression Sorting - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Sorting + Expression + Join + Expression + Join + Expression + CreatingSets + Expression + Expression + ReadFromMergeTree + CreatingSet + Expression + Filter + ReadFromSystemNumbers + Expression + ReadFromMemoryStorage + Expression + ReadFromMemoryStorage + Expression + ReadFromRemoteParallelReplicas -- -- Subqueries for IN are not allowed with sub1 as (select x, y from tab1 where x in (select number from numbers(16) where number != 2)), @@ -333,10 +441,18 @@ Expression Expression Filter ReadFromSystemNumbers + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas set parallel_replicas_prefer_local_join = 1; -- A query with only INNER/LEFT joins is fully send to replicas. JOIN is executed in GLOBAL mode. select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1; @@ -356,8 +472,22 @@ select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x 14 14 14 14 0 0 15 15 0 0 0 0 explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1; -Expression - ReadFromRemoteParallelReplicas +Union + Expression + Join + Expression + Join + Expression + Expression + ReadFromMergeTree + Expression + Expression + ReadFromMergeTree + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- The same query with cte; with sub1 as (select x, y from tab1 where x != 2), @@ -392,8 +522,24 @@ select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1; Expression Sorting - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Sorting + Expression + Join + Expression + Join + Expression + Expression + ReadFromMergeTree + Expression + Expression + ReadFromMergeTree + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- GROUP BY should work up to WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -417,8 +563,24 @@ Expression Sorting Expression MergingAggregated - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Aggregating + Expression + Join + Expression + Join + Expression + Expression + ReadFromMergeTree + Expression + Expression + ReadFromMergeTree + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- ORDER BY in sub3 : sub1 is fully pushed, sub3 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -455,10 +617,23 @@ Expression Sorting Expression Join - Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Join + Expression + Expression + ReadFromMergeTree + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- ORDER BY in sub1 : sub1 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2 order by y), @@ -497,12 +672,24 @@ Expression Join Expression Join + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- RIGHT JOIN in sub3: sub3 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -539,12 +726,24 @@ Expression Join Expression Join + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- RIGHT JOIN in sub5: sub5 -> WithMergableStage with sub1 as (select x, y from tab1 where x != 2), @@ -579,14 +778,26 @@ Expression Sorting Expression Join - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas Expression Join - Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- Subqueries for IN allowed with sub1 as (select x, y from tab1 where x in (select number from numbers(16) where number != 2)), @@ -621,8 +832,30 @@ select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1; Expression Sorting - Expression - ReadFromRemoteParallelReplicas + Union + Expression + Sorting + Expression + Join + Expression + Join + Expression + CreatingSets + Expression + Expression + ReadFromMergeTree + CreatingSet + Expression + Filter + ReadFromSystemNumbers + Expression + Expression + ReadFromMergeTree + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas -- -- Subqueries for IN are not allowed with sub1 as (select x, y from tab1 where x in (select number from numbers(16) where number != 2)), @@ -670,7 +903,15 @@ Expression Expression Filter ReadFromSystemNumbers + Union + Expression + Expression + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas + Union + Expression Expression - ReadFromRemoteParallelReplicas - Expression - ReadFromRemoteParallelReplicas + ReadFromMergeTree + Expression + ReadFromRemoteParallelReplicas From 09995013085511b206388816f896cdf251535ee4 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 7 Jun 2024 14:35:06 +0000 Subject: [PATCH 017/215] Fix crash in in-order coordinator --- src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp index f3318a48883..9eea40c4c5f 100644 --- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp +++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp @@ -887,7 +887,9 @@ void InOrderCoordinator::doHandleInitialAllRangesAnnouncement(InitialAllRa ++stats[announcement.replica_num].number_of_requests; - if (new_rows_to_read > 0) + /// FIXME: this code updating total_rows_to_read but it needs to be done only once since we're taking working set from initiator + /// util I missing something, it seems this code is not necessary if working set is taken from initiator (todo: check it) + if (new_rows_to_read > 0 && progress_callback) { Progress progress; progress.total_rows_to_read = new_rows_to_read; From 29d56b61b3425effb0f1fedc0c51319d8fa4062b Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 7 Jun 2024 14:54:00 +0000 Subject: [PATCH 018/215] fix 02404_memory_bound_merging --- .../02404_memory_bound_merging.reference | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/02404_memory_bound_merging.reference b/tests/queries/0_stateless/02404_memory_bound_merging.reference index 0409c48f846..3daa99f91c6 100644 --- a/tests/queries/0_stateless/02404_memory_bound_merging.reference +++ b/tests/queries/0_stateless/02404_memory_bound_merging.reference @@ -113,12 +113,18 @@ ExpressionTransform (Expression) ExpressionTransform × 4 (MergingAggregated) - Resize 1 → 4 - SortingAggregatedTransform 4 → 1 - MergingAggregatedBucketTransform × 4 - Resize 1 → 4 - GroupingAggregatedTransform 3 → 1 - (ReadFromRemoteParallelReplicas) + MergingAggregatedBucketTransform × 4 + Resize 1 → 4 + FinishAggregatingInOrderTransform 3 → 1 + (Union) + (Aggregating) + FinalizeAggregatedTransform + AggregatingInOrderTransform + (Expression) + ExpressionTransform + (ReadFromMergeTree) + MergeTreeSelect(pool: ReadPoolParallelReplicasInOrder, algorithm: InOrder) 0 → 1 + (ReadFromRemoteParallelReplicas) select a, count() from pr_t group by a order by a limit 5 offset 500; 500 1000 501 1000 From 23ca8608ba12dc9221effc760ecf11b3e2aefdf0 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 11 Jun 2024 13:25:19 +0000 Subject: [PATCH 019/215] Fix 03006_parallel_replicas_prewhere --- .../03006_parallel_replicas_prewhere.reference | 1 + .../0_stateless/03006_parallel_replicas_prewhere.sql | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03006_parallel_replicas_prewhere.reference b/tests/queries/0_stateless/03006_parallel_replicas_prewhere.reference index e69de29bb2d..8b8d0e5d565 100644 --- a/tests/queries/0_stateless/03006_parallel_replicas_prewhere.reference +++ b/tests/queries/0_stateless/03006_parallel_replicas_prewhere.reference @@ -0,0 +1 @@ +default url_na_log 1 130000 130 diff --git a/tests/queries/0_stateless/03006_parallel_replicas_prewhere.sql b/tests/queries/0_stateless/03006_parallel_replicas_prewhere.sql index 4b84646c034..afe6a00cc4d 100644 --- a/tests/queries/0_stateless/03006_parallel_replicas_prewhere.sql +++ b/tests/queries/0_stateless/03006_parallel_replicas_prewhere.sql @@ -21,11 +21,21 @@ SELECT FROM numbers(130000) SETTINGS max_insert_block_size = 200000; +SET max_block_size = 1048576, max_threads = 1, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3, parallel_replicas_min_number_of_rows_per_replica=10000; + EXPLAIN ESTIMATE SELECT count() FROM url_na_log PREWHERE (DateVisit >= toFixedString('2022-08-10', 10)) AND (DateVisit <= '2022-08-20') -SETTINGS max_block_size = 1048576, max_threads = 1, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3, parallel_replicas_min_number_of_rows_per_replica=10000; +SETTINGS allow_experimental_analyzer=0; + +-- here parallel replicas uses local snapshot as working set +-- so, the estimation can be done +EXPLAIN ESTIMATE +SELECT count() +FROM url_na_log +PREWHERE (DateVisit >= toFixedString('2022-08-10', 10)) AND (DateVisit <= '2022-08-20') +SETTINGS allow_experimental_analyzer=1; DROP POLICY url_na_log_policy0 ON url_na_log; DROP TABLE url_na_log; From d6dec38103cce55ba6c026ed8d56148eeed16e46 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 11 Jun 2024 14:37:42 +0000 Subject: [PATCH 020/215] Fix test_parallel_replicas_over_distributed --- src/Interpreters/ClusterProxy/executeQuery.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 0eeae112062..cde33697915 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -493,7 +493,8 @@ void executeQueryWithParallelReplicas( new_cluster->getShardsInfo().begin()->getAllNodeCount(), settings.parallel_replicas_mark_segment_size); auto external_tables = new_context->getExternalTables(); - if (settings.allow_experimental_analyzer) + /// do not build local plan for distributed queries for now (address it later) + if (settings.allow_experimental_analyzer && !shard_num) { auto read_from_remote = std::make_unique( query_ast, From f999eed3767ba5e98ee9de7d1f94d13fcab20ebc Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 11 Jun 2024 16:49:11 +0000 Subject: [PATCH 021/215] Fix 02967_parallel_replicas_join_algo_and_analyzer --- ...llel_replicas_join_algo_and_analyzer.reference | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference index d7fa419aeab..d911335c6c0 100644 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference @@ -23,10 +23,8 @@ simple (global) join with analyzer and parallel replicas 4200048 4200048 4200048 -1400016 4200054 4200054 4200054 -1400018 SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` (stage: WithMergeableState) DefaultCoordinator: Coordination done SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done simple (local) join with analyzer and parallel replicas @@ -41,7 +39,6 @@ simple (local) join with analyzer and parallel replicas 4200048 4200048 4200048 -1400016 4200054 4200054 4200054 -1400018 SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done simple (local) join with analyzer and parallel replicas and full sorting merge join @@ -56,7 +53,6 @@ simple (local) join with analyzer and parallel replicas and full sorting merge j 4200048 4200048 4200048 -1400016 4200054 4200054 4200054 -1400018 SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) WithOrderCoordinator: Coordination done nested join with analyzer @@ -83,7 +79,6 @@ nested join with analyzer and parallel replicas, both local 420336 420336 420336 -140112 420378 420378 420378 -140126 SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` SETTINGS parallel_replicas_prefer_local_join = 1) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` SETTINGS parallel_replicas_prefer_local_join = 1) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) WithOrderCoordinator: Coordination done nested join with analyzer and parallel replicas, both global @@ -98,10 +93,8 @@ nested join with analyzer and parallel replicas, both global 420336 420336 420336 -140112 420378 420378 420378 -140126 SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) DefaultCoordinator: Coordination done SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done nested join with analyzer and parallel replicas, global + local @@ -116,10 +109,8 @@ nested join with analyzer and parallel replicas, global + local 420336 420336 420336 -140112 420378 420378 420378 -140126 SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` ALL INNER JOIN (SELECT `__table3`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table3`) AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` ALL INNER JOIN (SELECT `__table3`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table3`) AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) DefaultCoordinator: Coordination done SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done nested join with analyzer and parallel replicas, both local, both full sorting merge join @@ -134,10 +125,8 @@ nested join with analyzer and parallel replicas, both local, both full sorting m 420336 420336 420336 -140112 420378 420378 420378 -140126 SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) WithOrderCoordinator: Coordination done SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) WithOrderCoordinator: Coordination done nested join with analyzer and parallel replicas, both local, both full sorting and hash join @@ -152,10 +141,8 @@ nested join with analyzer and parallel replicas, both local, both full sorting a 420336 420336 420336 -140112 420378 420378 420378 -140126 SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) DefaultCoordinator: Coordination done SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) WithOrderCoordinator: Coordination done nested join with analyzer and parallel replicas, both local, both full sorting and hash join @@ -170,8 +157,6 @@ nested join with analyzer and parallel replicas, both local, both full sorting a 420336 420336 420336 -140112 420378 420378 420378 -140126 SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table2` ON `__table1`.`key` = `__table2`.`key` (stage: WithMergeableState) WithOrderCoordinator: Coordination done SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) -SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState) DefaultCoordinator: Coordination done From f46deb4e793647b505514eb1a826dc94d7a69207 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 11 Jun 2024 16:49:51 +0000 Subject: [PATCH 022/215] Fix clang-tidy --- .../QueryPlan/ParallelReplicasLocalPlan.cpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp index 27d86f656c5..16cca6a7f5d 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -1,26 +1,22 @@ #include #include -#include "Storages/MergeTree/RequestResponse.h" -#include #include +#include #include +#include +#include #include #include -#include #include -#include #include -#include #include -#include +#include #include -#include - +#include +#include #include -#include - namespace ProfileEvents { extern const Event SelectedParts; From da4ed273e304cf01010289b374074818e04771d7 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 11 Jun 2024 19:24:50 +0000 Subject: [PATCH 023/215] Fix 02811_parallel_replicas_prewhere_count --- src/Planner/PlannerJoinTree.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 9d1301305b8..15d0fa45891 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -959,6 +959,19 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres std::move(reading_step)); query_plan = std::move(query_plan_parallel_replicas); } + else { + QueryPlan query_plan_no_parallel_replicas; + storage->read( + query_plan_no_parallel_replicas, + columns_names, + storage_snapshot, + table_expression_query_info, + query_context, + from_stage, + max_block_size, + max_streams); + query_plan = std::move(query_plan_no_parallel_replicas); + } } const auto & alias_column_expressions = table_expression_data.getAliasColumnExpressions(); From eccd56c9b93f6ac3a8ef5c23c4bbc5ddc8a77fb6 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 11 Jun 2024 20:16:00 +0000 Subject: [PATCH 024/215] Fix style check --- src/Planner/PlannerJoinTree.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 15d0fa45891..7a8ad50b9ab 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -959,7 +959,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres std::move(reading_step)); query_plan = std::move(query_plan_parallel_replicas); } - else { + else + { QueryPlan query_plan_no_parallel_replicas; storage->read( query_plan_no_parallel_replicas, From 164b5e47a81ae3ad0719d00319f5193fd59ee680 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 12 Jun 2024 10:30:15 +0000 Subject: [PATCH 025/215] Fix 02769_parallel_replicas_unavailable_shards --- .../0_stateless/02769_parallel_replicas_unavailable_shards.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql b/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql index 1a75e000349..34678fdd146 100644 --- a/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql +++ b/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql @@ -2,7 +2,7 @@ DROP TABLE IF EXISTS test_parallel_replicas_unavailable_shards; CREATE TABLE test_parallel_replicas_unavailable_shards (n UInt64) ENGINE=MergeTree() ORDER BY tuple(); INSERT INTO test_parallel_replicas_unavailable_shards SELECT * FROM numbers(10); -SET allow_experimental_parallel_reading_from_replicas=2, max_parallel_replicas=11, cluster_for_parallel_replicas='parallel_replicas', parallel_replicas_for_non_replicated_merge_tree=1; +SET allow_experimental_parallel_reading_from_replicas=2, max_parallel_replicas=11, cluster_for_parallel_replicas='parallel_replicas', parallel_replicas_for_non_replicated_merge_tree=1, allow_experimental_analyzer=0; SET send_logs_level='error'; SELECT count() FROM test_parallel_replicas_unavailable_shards WHERE NOT ignore(*) SETTINGS log_comment = '02769_7b513191-5082-4073-8568-53b86a49da79'; From 048cbb17a6b016c20128bfa156d52a7554974fdc Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 13 Jun 2024 19:55:30 +0000 Subject: [PATCH 026/215] Comment to 02769_parallel_replicas_unavailable_shards.sql --- .../02769_parallel_replicas_unavailable_shards.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql b/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql index 34678fdd146..337b05263d0 100644 --- a/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql +++ b/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql @@ -2,9 +2,11 @@ DROP TABLE IF EXISTS test_parallel_replicas_unavailable_shards; CREATE TABLE test_parallel_replicas_unavailable_shards (n UInt64) ENGINE=MergeTree() ORDER BY tuple(); INSERT INTO test_parallel_replicas_unavailable_shards SELECT * FROM numbers(10); -SET allow_experimental_parallel_reading_from_replicas=2, max_parallel_replicas=11, cluster_for_parallel_replicas='parallel_replicas', parallel_replicas_for_non_replicated_merge_tree=1, allow_experimental_analyzer=0; +SET allow_experimental_parallel_reading_from_replicas=2, max_parallel_replicas=11, cluster_for_parallel_replicas='parallel_replicas', parallel_replicas_for_non_replicated_merge_tree=1; SET send_logs_level='error'; -SELECT count() FROM test_parallel_replicas_unavailable_shards WHERE NOT ignore(*) SETTINGS log_comment = '02769_7b513191-5082-4073-8568-53b86a49da79'; +-- with local plan for initiator, the query can be executed fast on initator, we can simply not come to the point where unavailable replica can be detected +-- therefore disable local plan for now +SELECT count() FROM test_parallel_replicas_unavailable_shards WHERE NOT ignore(*) SETTINGS log_comment = '02769_7b513191-5082-4073-8568-53b86a49da79', allow_experimental_analyzer=0; SYSTEM FLUSH LOGS; From 245476b34bbbf7b0fd24fffa54539fa9c2b3b490 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 14 Jun 2024 12:02:47 +0000 Subject: [PATCH 027/215] Consistent replica id assignment --- .../QueryPlan/ParallelReplicasLocalPlan.cpp | 3 +- src/Processors/QueryPlan/ReadFromRemote.cpp | 34 ++++++++++++++++--- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp index 16cca6a7f5d..bd79fc38ae9 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -280,8 +280,7 @@ std::unique_ptr createLocalPlanForParallelReplicas( MergeTreeReadTaskCallback read_task_cb = [coordinator](ParallelReadRequest req) -> std::optional { return coordinator->handleRequest(std::move(req)); }; - const auto number_of_local_replica = new_context->getSettingsRef().max_parallel_replicas - 1; - + const auto number_of_local_replica = 0; auto read_from_merge_tree_parallel_replicas = reading->createLocalParallelReplicasReadingStep(analyzed_merge_tree, all_ranges_cb, read_task_cb, number_of_local_replica); node->step = std::move(read_from_merge_tree_parallel_replicas); diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 23d8f2b496f..6949c35e0ca 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -434,14 +434,11 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder shard.getAllNodeCount()); all_replicas_count = shard.getAllNodeCount(); } - if (exclude_local_replica) - --all_replicas_count; std::vector shuffled_pool; if (all_replicas_count < shard.getAllNodeCount()) { shuffled_pool = shard.pool->getShuffledPools(current_settings); - shuffled_pool.resize(all_replicas_count); } else { @@ -451,10 +448,37 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder shuffled_pool = shard.pool->getShuffledPools(current_settings, priority_func); } - for (size_t i=0; i < all_replicas_count; ++i) + std::vector pools_to_use; + if (exclude_local_replica) { - IConnections::ReplicaInfo replica_info + std::vector local_addr_possitions; + for (auto & pool : shuffled_pool) { + const auto & hostname = pool.pool->getHost(); + auto it = std::find_if( + begin(shard.local_addresses), + end(shard.local_addresses), + [&hostname](const Cluster::Address & local_addr) { return hostname == local_addr.host_name; }); + if (it != shard.local_addresses.end()) + pool.pool.reset(); + } + } + for (const auto & pool : shuffled_pool) + { + if (pool.pool) + pools_to_use.push_back(pool.pool); + } + + if (pools_to_use.size() > all_replicas_count) + pools_to_use.resize(all_replicas_count); + else + all_replicas_count = pools_to_use.size(); + + /// local replicas has number 0 + size_t offset = (exclude_local_replica ? 1 : 0); + for (size_t i = 0 + offset; i < all_replicas_count + offset; ++i) + { + IConnections::ReplicaInfo replica_info{ .all_replicas_count = all_replicas_count, /// we should use this number specifically because efficiency of data distribution by consistent hash depends on it. .number_of_current_replica = i, From c2d38b3c93c2182f096b21558db2b7f574fa435c Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 14 Jun 2024 14:06:12 +0000 Subject: [PATCH 028/215] Fix crash --- src/Processors/QueryPlan/ReadFromRemote.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 6949c35e0ca..1a96668e30e 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -474,9 +474,17 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder else all_replicas_count = pools_to_use.size(); + chassert(all_replicas_count == pools_to_use.size()); + + if (exclude_local_replica && !pools_to_use.empty()) + pools_to_use.resize(all_replicas_count - 1); + + if (pools_to_use.empty()) + return; + /// local replicas has number 0 size_t offset = (exclude_local_replica ? 1 : 0); - for (size_t i = 0 + offset; i < all_replicas_count + offset; ++i) + for (size_t i = 0 + offset; i < all_replicas_count; ++i) { IConnections::ReplicaInfo replica_info{ .all_replicas_count = all_replicas_count, @@ -484,7 +492,7 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder .number_of_current_replica = i, }; - addPipeForSingeReplica(pipes, shuffled_pool[i].pool, replica_info); + addPipeForSingeReplica(pipes, pools_to_use[i - offset], replica_info); } auto pipe = Pipe::unitePipes(std::move(pipes)); From 70a33e633cd8acd0ec7220b5ce6e45d9c7bd4e9e Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 14 Jun 2024 19:53:04 +0000 Subject: [PATCH 029/215] Fix 02784_parallel_replicas_automatic_decision --- .../02784_parallel_replicas_automatic_decision.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision.sh b/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision.sh index 8a3b34e5cfa..dd9b5c3f6d9 100755 --- a/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision.sh +++ b/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision.sh @@ -10,12 +10,14 @@ function were_parallel_replicas_used () { $CLICKHOUSE_CLIENT --query " SELECT initial_query_id, - concat('Used parallel replicas: ', (countIf(initial_query_id != query_id) != 0)::bool::String) as used + concat('Used parallel replicas: ', (ProfileEvents['ParallelReplicasUsedCount'] > 0)::bool::String) as used FROM system.query_log WHERE event_date >= yesterday() AND initial_query_id LIKE '$1%' - GROUP BY initial_query_id - ORDER BY min(event_time_microseconds) ASC + AND query_id = initial_query_id + AND type = 'QueryFinish' + AND current_database = '$CLICKHOUSE_DATABASE' + ORDER BY event_time_microseconds ASC FORMAT TSV" } @@ -48,7 +50,6 @@ function run_query_with_pure_parallel_replicas () { --query "$3" \ --query_id "${1}_pure" \ --max_parallel_replicas 3 \ - --prefer_localhost_replica 1 \ --cluster_for_parallel_replicas "parallel_replicas" \ --allow_experimental_parallel_reading_from_replicas 1 \ --parallel_replicas_for_non_replicated_merge_tree 1 \ From c87bfe102e36612ebb4b474eb822543de0cf011e Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 17 Jun 2024 12:24:14 +0000 Subject: [PATCH 030/215] Fix: correct local replica exclusion - fixes 02731_parallel_replicas_join_subquery --- .../ClusterProxy/executeQuery.cpp | 8 +++++-- src/Processors/QueryPlan/ReadFromRemote.cpp | 21 +++++++++++++++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index cde33697915..0937e121426 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -489,8 +489,12 @@ void executeQueryWithParallelReplicas( "`cluster_for_parallel_replicas` setting refers to cluster with several shards. Expected a cluster with one shard"); } - auto coordinator = std::make_shared( - new_cluster->getShardsInfo().begin()->getAllNodeCount(), settings.parallel_replicas_mark_segment_size); + auto replica_count = new_cluster->getShardsInfo().begin()->getAllNodeCount(); + if (settings.max_parallel_replicas < replica_count) + replica_count = settings.max_parallel_replicas; + + auto coordinator = std::make_shared(replica_count, settings.parallel_replicas_mark_segment_size); + auto external_tables = new_context->getExternalTables(); /// do not build local plan for distributed queries for now (address it later) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 1a96668e30e..f7cb88154af 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -449,6 +449,7 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder } std::vector pools_to_use; + pools_to_use.reserve(shuffled_pool.size()); if (exclude_local_replica) { std::vector local_addr_possitions; @@ -460,7 +461,9 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder end(shard.local_addresses), [&hostname](const Cluster::Address & local_addr) { return hostname == local_addr.host_name; }); if (it != shard.local_addresses.end()) + { pool.pool.reset(); + } } } for (const auto & pool : shuffled_pool) @@ -469,12 +472,14 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder pools_to_use.push_back(pool.pool); } + LOG_DEBUG( + getLogger("ReadFromParallelRemoteReplicasStep"), + "Number of pools to use is {}. Originally {}", + pools_to_use.size(), + shuffled_pool.size()); + if (pools_to_use.size() > all_replicas_count) pools_to_use.resize(all_replicas_count); - else - all_replicas_count = pools_to_use.size(); - - chassert(all_replicas_count == pools_to_use.size()); if (exclude_local_replica && !pools_to_use.empty()) pools_to_use.resize(all_replicas_count - 1); @@ -482,6 +487,14 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder if (pools_to_use.empty()) return; + { + String pool_addresses; + for (const auto & pool : pools_to_use) + pool_addresses += pool->getAddress() + ";"; + + LOG_DEBUG(getLogger("ReadFromParallelRemoteReplicasStep"), "Addresses to use: {}", pool_addresses); + } + /// local replicas has number 0 size_t offset = (exclude_local_replica ? 1 : 0); for (size_t i = 0 + offset; i < all_replicas_count; ++i) From c286188419df31cdc17ac29f341a99b723da6360 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 18 Jun 2024 10:39:13 +0000 Subject: [PATCH 031/215] Update 00177_memory_bound_merging.reference --- .../queries/1_stateful/00177_memory_bound_merging.reference | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/queries/1_stateful/00177_memory_bound_merging.reference b/tests/queries/1_stateful/00177_memory_bound_merging.reference index 5689152d60a..4c7505d1123 100644 --- a/tests/queries/1_stateful/00177_memory_bound_merging.reference +++ b/tests/queries/1_stateful/00177_memory_bound_merging.reference @@ -10,6 +10,8 @@ http://auto.ru/chatay-baranta_bound-in-thankYou=ru/tver/zhanny 2014-03-18 http:/ http://auto.ru/chatay-baranta_bound-in-thankYou=ru/tver/zhanny 2014-03-19 http://auto.ru/chatay-baranta_bound-in-thankYou=ru/tver/zhanny http://auto.ru/chatay-baranta_bound-in-thankYou=ru/tver/zhanny 2014-03-20 http://auto.ru/chatay-baranta_bound-in-thankYou=ru/tver/zhanny 1 -SortingAggregatedTransform MergingAggregatedBucketTransform -GroupingAggregatedTransform +FinishAggregatingInOrderTransform +FinalizeAggregatedTransform +AggregatingInOrderTransform +MergeTreeSelect From 366de07856cd25d7212ae8106d68994cd84a2de3 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 18 Jun 2024 10:39:50 +0000 Subject: [PATCH 032/215] Remove prefer_localhost_replica --- .../0_stateless/02731_parallel_replicas_join_subquery.sql | 1 - tests/queries/1_stateful/00177_memory_bound_merging.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.sql b/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.sql index 7693d0da295..e91e2a19526 100644 --- a/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.sql +++ b/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.sql @@ -22,7 +22,6 @@ SELECT LIMIT 100; SET max_parallel_replicas = 3; -SET prefer_localhost_replica = 1; SET cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; SET joined_subquery_requires_alias = 0; diff --git a/tests/queries/1_stateful/00177_memory_bound_merging.sh b/tests/queries/1_stateful/00177_memory_bound_merging.sh index d5cd1a05cd8..74b5e80e059 100755 --- a/tests/queries/1_stateful/00177_memory_bound_merging.sh +++ b/tests/queries/1_stateful/00177_memory_bound_merging.sh @@ -55,7 +55,7 @@ test2() { test3() { $CLICKHOUSE_CLIENT -nq " SET cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; - SET max_threads = 16, prefer_localhost_replica = 1, read_in_order_two_level_merge_threshold = 1000, query_plan_aggregation_in_order = 1, distributed_aggregation_memory_efficient = 1; + SET max_threads = 16, read_in_order_two_level_merge_threshold = 1000, query_plan_aggregation_in_order = 1, distributed_aggregation_memory_efficient = 1; SELECT replaceRegexpOne(explain, '^ *(\w+).*', '\\1') FROM ( From 0ecefa6973ecb1356e117d944d26f26fb204d8c9 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 18 Jun 2024 11:07:15 +0000 Subject: [PATCH 033/215] Better replicas notation in plan description --- src/Processors/QueryPlan/ReadFromRemote.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index f7cb88154af..26c57c4a760 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -387,8 +387,8 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep( { chassert(cluster->getShardCount() == 1); - std::vector description; - description.push_back(fmt::format("query: {}", formattedAST(query_ast))); + std::vector replicas; + replicas.reserve(cluster->getShardsAddresses().front().size()); bool first_local = false; for (const auto & addr : cluster->getShardsAddresses().front()) @@ -400,10 +400,16 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep( continue; } - description.push_back(fmt::format("Replica: {}", addr.host_name)); + /// replace hostname with replica name if the hostname started with replica namespace, + /// it makes description shorter and more readable + if (!addr.database_replica_name.empty() && addr.host_name.starts_with(addr.database_replica_name)) + replicas.push_back(fmt::format("{}", addr.database_replica_name)); + else + replicas.push_back(fmt::format("{}", addr.host_name)); } - setStepDescription(boost::algorithm::join(description, ", ")); + auto description = fmt::format("Query: {} Replicas: ", formattedAST(query_ast)) + boost::algorithm::join(replicas, ", "); + setStepDescription(std::move(description)); } void ReadFromParallelRemoteReplicasStep::enforceSorting(SortDescription output_sort_description) From dba6ea078f259433dd91a0b2ddc88b6a4f3c143d Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 18 Jun 2024 19:47:07 +0000 Subject: [PATCH 034/215] Setting parallel_replicas_local_plan --- src/Core/Settings.h | 1 + src/Interpreters/ClusterProxy/executeQuery.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 6197a7cf6e1..c16efc5717b 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -216,6 +216,7 @@ class IColumn; M(UInt64, parallel_replicas_min_number_of_rows_per_replica, 0, "Limit the number of replicas used in a query to (estimated rows to read / min_number_of_rows_per_replica). The max is still limited by 'max_parallel_replicas'", 0) \ M(Bool, parallel_replicas_prefer_local_join, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN.", 0) \ M(UInt64, parallel_replicas_mark_segment_size, 128, "Parts virtually divided into segments to be distributed between replicas for parallel reading. This setting controls the size of these segments. Not recommended to change until you're absolutely sure in what you're doing", 0) \ + M(Bool, parallel_replicas_local_plan, true, "Build local plan for local replica", 0) \ \ M(Bool, skip_unavailable_shards, false, "If true, ClickHouse silently skips unavailable shards. Shard is marked as unavailable when: 1) The shard cannot be reached due to a connection failure. 2) Shard is unresolvable through DNS. 3) Table does not exist on the shard.", 0) \ \ diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 0937e121426..ab25da090d6 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -498,7 +498,7 @@ void executeQueryWithParallelReplicas( auto external_tables = new_context->getExternalTables(); /// do not build local plan for distributed queries for now (address it later) - if (settings.allow_experimental_analyzer && !shard_num) + if (settings.allow_experimental_analyzer && settings.parallel_replicas_local_plan && !shard_num) { auto read_from_remote = std::make_unique( query_ast, From 9658d37d49e3c47cd306657abb8a314f47c4011d Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 19 Jun 2024 11:27:50 +0000 Subject: [PATCH 035/215] Add parallel_replicas_local_plan to settings history --- src/Core/SettingsChangesHistory.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index eddf83f7912..1d197db9e31 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -113,6 +113,7 @@ static const std::map Date: Mon, 24 Jun 2024 14:31:05 +0000 Subject: [PATCH 036/215] Cleanup --- src/Processors/Executors/PipelineExecutor.h | 1 - .../QueryPlan/ParallelReplicasLocalPlan.cpp | 173 ------------------ 2 files changed, 174 deletions(-) diff --git a/src/Processors/Executors/PipelineExecutor.h b/src/Processors/Executors/PipelineExecutor.h index 03f0f7f1a0a..ae119355cb5 100644 --- a/src/Processors/Executors/PipelineExecutor.h +++ b/src/Processors/Executors/PipelineExecutor.h @@ -9,7 +9,6 @@ #include #include -#include #include diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp index bd79fc38ae9..bd5a4793872 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -17,13 +17,6 @@ #include #include -namespace ProfileEvents -{ - extern const Event SelectedParts; - extern const Event SelectedRanges; - extern const Event SelectedMarks; -} - namespace DB { @@ -56,172 +49,6 @@ void addConvertingActions(QueryPlan & plan, const Block & header, bool has_missi } -class ReadFromMergeTreeCoordinated : public ISourceStep -{ -public: - ReadFromMergeTreeCoordinated(QueryPlanStepPtr read_from_merge_tree_, ParallelReplicasReadingCoordinatorPtr coordinator_) - : ISourceStep(read_from_merge_tree_->getOutputStream()) - , read_from_merge_tree(std::move(read_from_merge_tree_)) - , coordinator(std::move(coordinator_)) - { - } - - void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) override; - String getName() const override { return "ReadFromLocalParallelReplica"; } - -private: - QueryPlanStepPtr read_from_merge_tree; - ParallelReplicasReadingCoordinatorPtr coordinator; -}; - -void ReadFromMergeTreeCoordinated::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & /*settings*/) -{ - ReadFromMergeTree & reading = *typeid_cast(read_from_merge_tree.get()); - - auto result = reading.getAnalysisResult(); - const auto & query_info = reading.getQueryInfo(); - const auto & data = reading.data; - const auto & context = reading.getContext(); - const auto & storage_snapshot = reading.getStorageSnapshot(); - - if (reading.enable_remove_parts_from_snapshot_optimization) - { - /// Do not keep data parts in snapshot. - /// They are stored separately, and some could be released after PK analysis. - reading.storage_snapshot->data = std::make_unique(); - } - - LOG_DEBUG( - reading.log, - "Selected {}/{} parts by partition key, {} parts by primary key, {}/{} marks by primary key, {} marks to read from {} ranges", - result.parts_before_pk, - result.total_parts, - result.selected_parts, - result.selected_marks_pk, - result.total_marks_pk, - result.selected_marks, - result.selected_ranges); - - // Adding partition info to QueryAccessInfo. - if (context->hasQueryContext() && !query_info.is_internal) - { - Names partition_names; - for (const auto & part : result.parts_with_ranges) - { - partition_names.emplace_back( - fmt::format("{}.{}", data.getStorageID().getFullNameNotQuoted(), part.data_part->info.partition_id)); - } - context->getQueryContext()->addQueryAccessInfo(partition_names); - - if (storage_snapshot->projection) - context->getQueryContext()->addQueryAccessInfo( - Context::QualifiedProjectionName{.storage_id = data.getStorageID(), .projection_name = storage_snapshot->projection->name}); - } - - ProfileEvents::increment(ProfileEvents::SelectedParts, result.selected_parts); - ProfileEvents::increment(ProfileEvents::SelectedRanges, result.selected_ranges); - ProfileEvents::increment(ProfileEvents::SelectedMarks, result.selected_marks); - - auto query_id_holder = MergeTreeDataSelectExecutor::checkLimits(data, result, context); - - // TODO: check this on plan level, we should be here if there is nothing to read - if (result.parts_with_ranges.empty()) - { - pipeline.init(Pipe(std::make_shared(getOutputStream().header))); - return; - } - - /// Projection, that needed to drop columns, which have appeared by execution - /// of some extra expressions, and to allow execute the same expressions later. - /// NOTE: It may lead to double computation of expressions. - ActionsDAGPtr result_projection; - - Pipe pipe = reading.spreadMarkRanges(std::move(result.parts_with_ranges), reading.requested_num_streams, result, result_projection); - - for (const auto & processor : pipe.getProcessors()) - processor->setStorageLimits(query_info.storage_limits); - - if (pipe.empty()) - { - pipeline.init(Pipe(std::make_shared(getOutputStream().header))); - return; - } - - if (result.sampling.use_sampling) - { - auto sampling_actions = std::make_shared(result.sampling.filter_expression); - pipe.addSimpleTransform([&](const Block & header) - { - return std::make_shared( - header, - sampling_actions, - result.sampling.filter_function->getColumnName(), - false); - }); - } - - Block cur_header = pipe.getHeader(); - - auto append_actions = [&result_projection](ActionsDAGPtr actions) - { - if (!result_projection) - result_projection = std::move(actions); - else - result_projection = ActionsDAG::merge(std::move(*result_projection), std::move(*actions)); - }; - - if (result_projection) - cur_header = result_projection->updateHeader(cur_header); - - /// Extra columns may be returned (for example, if sampling is used). - /// Convert pipe to step header structure. - if (!isCompatibleHeader(cur_header, getOutputStream().header)) - { - auto converting = ActionsDAG::makeConvertingActions( - cur_header.getColumnsWithTypeAndName(), - getOutputStream().header.getColumnsWithTypeAndName(), - ActionsDAG::MatchColumnsMode::Name); - - append_actions(std::move(converting)); - } - - if (result_projection) - { - auto projection_actions = std::make_shared(result_projection); - pipe.addSimpleTransform([&](const Block & header) - { - return std::make_shared(header, projection_actions); - }); - } - - /// Some extra columns could be added by sample/final/in-order/etc - /// Remove them from header if not needed. - if (!blocksHaveEqualStructure(pipe.getHeader(), getOutputStream().header)) - { - auto convert_actions_dag = ActionsDAG::makeConvertingActions( - pipe.getHeader().getColumnsWithTypeAndName(), - getOutputStream().header.getColumnsWithTypeAndName(), - ActionsDAG::MatchColumnsMode::Name, - true); - - auto converting_dag_expr = std::make_shared(convert_actions_dag); - - pipe.addSimpleTransform([&](const Block & header) - { - return std::make_shared(header, converting_dag_expr); - }); - } - - for (const auto & processor : pipe.getProcessors()) - processors.emplace_back(processor); - - pipeline.init(std::move(pipe)); - pipeline.addContext(context); - // Attach QueryIdHolder if needed - if (query_id_holder) - pipeline.setQueryIdHolder(std::move(query_id_holder)); -} - std::unique_ptr createLocalPlanForParallelReplicas( const ASTPtr & query_ast, const Block & header, From f62873b1735bc22e8a24a1ed91a43edbf405fea4 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 25 Jun 2024 12:29:21 +0000 Subject: [PATCH 037/215] Fix 02404_memory_bound_merging --- .../02404_memory_bound_merging.reference | 21 ++++++++++++++++++- .../02404_memory_bound_merging.sql | 3 ++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02404_memory_bound_merging.reference b/tests/queries/0_stateless/02404_memory_bound_merging.reference index 3daa99f91c6..b2195051c0b 100644 --- a/tests/queries/0_stateless/02404_memory_bound_merging.reference +++ b/tests/queries/0_stateless/02404_memory_bound_merging.reference @@ -100,7 +100,26 @@ select a, count() from dist_t_different_dbs group by a, b order by a limit 5 off 504 2000 1000000 -- { echoOn } -- -explain pipeline select a from pr_t group by a order by a limit 5 offset 500; +explain pipeline select a from pr_t group by a order by a limit 5 offset 500 settings allow_experimental_analyzer=0; +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + MergingSortedTransform 4 → 1 + MergeSortingTransform × 4 + LimitsCheckingTransform × 4 + PartialSortingTransform × 4 + (Expression) + ExpressionTransform × 4 + (MergingAggregated) + Resize 1 → 4 + SortingAggregatedTransform 4 → 1 + MergingAggregatedBucketTransform × 4 + Resize 1 → 4 + GroupingAggregatedTransform 3 → 1 + (ReadFromRemoteParallelReplicas) +explain pipeline select a from pr_t group by a order by a limit 5 offset 500 settings allow_experimental_analyzer=1, parallel_replicas_local_plan=1; (Expression) ExpressionTransform (Limit) diff --git a/tests/queries/0_stateless/02404_memory_bound_merging.sql b/tests/queries/0_stateless/02404_memory_bound_merging.sql index 5e017e79309..a12ccb2ffb3 100644 --- a/tests/queries/0_stateless/02404_memory_bound_merging.sql +++ b/tests/queries/0_stateless/02404_memory_bound_merging.sql @@ -72,7 +72,8 @@ set distributed_aggregation_memory_efficient=1; select count() from pr_t; -- { echoOn } -- -explain pipeline select a from pr_t group by a order by a limit 5 offset 500; +explain pipeline select a from pr_t group by a order by a limit 5 offset 500 settings allow_experimental_analyzer=0; +explain pipeline select a from pr_t group by a order by a limit 5 offset 500 settings allow_experimental_analyzer=1, parallel_replicas_local_plan=1; select a, count() from pr_t group by a order by a limit 5 offset 500; select a, count() from pr_t group by a, b order by a limit 5 offset 500; From 96b68cb920600cb1a080cefd5f7e82de332ac8dd Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 25 Jun 2024 13:33:36 +0000 Subject: [PATCH 038/215] Update setting in 02404_memory_bound_merging --- tests/queries/0_stateless/02404_memory_bound_merging.reference | 2 +- tests/queries/0_stateless/02404_memory_bound_merging.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02404_memory_bound_merging.reference b/tests/queries/0_stateless/02404_memory_bound_merging.reference index b2195051c0b..e02684c3b95 100644 --- a/tests/queries/0_stateless/02404_memory_bound_merging.reference +++ b/tests/queries/0_stateless/02404_memory_bound_merging.reference @@ -100,7 +100,7 @@ select a, count() from dist_t_different_dbs group by a, b order by a limit 5 off 504 2000 1000000 -- { echoOn } -- -explain pipeline select a from pr_t group by a order by a limit 5 offset 500 settings allow_experimental_analyzer=0; +explain pipeline select a from pr_t group by a order by a limit 5 offset 500 settings parallel_replicas_local_plan=0; (Expression) ExpressionTransform (Limit) diff --git a/tests/queries/0_stateless/02404_memory_bound_merging.sql b/tests/queries/0_stateless/02404_memory_bound_merging.sql index a12ccb2ffb3..2958cc22ce0 100644 --- a/tests/queries/0_stateless/02404_memory_bound_merging.sql +++ b/tests/queries/0_stateless/02404_memory_bound_merging.sql @@ -72,7 +72,7 @@ set distributed_aggregation_memory_efficient=1; select count() from pr_t; -- { echoOn } -- -explain pipeline select a from pr_t group by a order by a limit 5 offset 500 settings allow_experimental_analyzer=0; +explain pipeline select a from pr_t group by a order by a limit 5 offset 500 settings parallel_replicas_local_plan=0; explain pipeline select a from pr_t group by a order by a limit 5 offset 500 settings allow_experimental_analyzer=1, parallel_replicas_local_plan=1; select a, count() from pr_t group by a order by a limit 5 offset 500; From 318af3af95845166ff7cd9c38c6a2cfd1e97ccfd Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 25 Jun 2024 14:31:11 +0000 Subject: [PATCH 039/215] Cleanup --- .../QueryPlan/ParallelReplicasLocalPlan.cpp | 34 ++----------------- .../QueryPlan/ParallelReplicasLocalPlan.h | 3 +- .../QueryPlan/ReadFromMergeTree.cpp | 10 +++--- src/Processors/QueryPlan/ReadFromMergeTree.h | 9 ++--- .../ParallelReplicasReadingCoordinator.h | 2 +- ...9_parallel_replicas_unavailable_shards.sql | 2 +- 6 files changed, 14 insertions(+), 46 deletions(-) diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp index bd5a4793872..6822d8b0a71 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -20,34 +20,7 @@ namespace DB { -namespace -{ - -void addConvertingActions(QueryPlan & plan, const Block & header, bool has_missing_objects) -{ - if (blocksHaveEqualStructure(plan.getCurrentDataStream().header, header)) - return; - - auto mode = has_missing_objects ? ActionsDAG::MatchColumnsMode::Position : ActionsDAG::MatchColumnsMode::Name; - - auto get_converting_dag = [mode](const Block & block_, const Block & header_) - { - /// Convert header structure to expected. - /// Also we ignore constants from result and replace it with constants from header. - /// It is needed for functions like `now64()` or `randConstant()` because their values may be different. - return ActionsDAG::makeConvertingActions( - block_.getColumnsWithTypeAndName(), - header_.getColumnsWithTypeAndName(), - mode, - true); - }; - - auto convert_actions_dag = get_converting_dag(plan.getCurrentDataStream().header, header); - auto converting = std::make_unique(plan.getCurrentDataStream(), convert_actions_dag); - plan.addStep(std::move(converting)); -} - -} +void addConvertingActions(QueryPlan & plan, const Block & header, bool has_missing_objects); std::unique_ptr createLocalPlanForParallelReplicas( const ASTPtr & query_ast, @@ -55,8 +28,7 @@ std::unique_ptr createLocalPlanForParallelReplicas( ContextPtr context, QueryProcessingStage::Enum processed_stage, ParallelReplicasReadingCoordinatorPtr coordinator, - QueryPlanStepPtr analyzed_read_from_merge_tree, - bool has_missing_objects) + QueryPlanStepPtr analyzed_read_from_merge_tree) { checkStackSize(); @@ -112,7 +84,7 @@ std::unique_ptr createLocalPlanForParallelReplicas( = reading->createLocalParallelReplicasReadingStep(analyzed_merge_tree, all_ranges_cb, read_task_cb, number_of_local_replica); node->step = std::move(read_from_merge_tree_parallel_replicas); - addConvertingActions(*query_plan, header, has_missing_objects); + addConvertingActions(*query_plan, header, /*has_missing_objects=*/false); return query_plan; } diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.h b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.h index 89d2019f807..123754458a1 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.h +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.h @@ -14,6 +14,5 @@ std::unique_ptr createLocalPlanForParallelReplicas( ContextPtr context, QueryProcessingStage::Enum processed_stage, ParallelReplicasReadingCoordinatorPtr coordinator, - QueryPlanStepPtr read_from_merge_tree, - bool has_missing_objects); + QueryPlanStepPtr read_from_merge_tree); } diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 71caece30ed..1de701cff0b 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -339,9 +339,9 @@ ReadFromMergeTree::ReadFromMergeTree( std::unique_ptr ReadFromMergeTree::createLocalParallelReplicasReadingStep( const ReadFromMergeTree * analyzed_merge_tree, std::optional all_ranges_callback_, - std::optional read_task_callback_, - std::optional number_of_current_replica_) + std::optional read_task_callback_) { + const auto number_of_local_replica = 0; return std::make_unique( prepared_parts, alter_conversions_for_parts, @@ -356,9 +356,9 @@ std::unique_ptr ReadFromMergeTree::createLocalParallelReplica log, (analyzed_merge_tree ? analyzed_merge_tree->analyzed_result_ptr : nullptr), true, - all_ranges_callback_, - read_task_callback_, - number_of_current_replica_); + all_ranges_callback, + read_task_callback, + number_of_local_replica); } Pipe ReadFromMergeTree::readFromPoolParallelReplicas(RangesInDataParts parts_with_range, Names required_columns, PoolSettings pool_settings) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 079f95be908..59a85413dd9 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -123,14 +123,13 @@ public: AnalysisResultPtr analyzed_result_ptr_, bool enable_parallel_reading_, std::optional all_ranges_callback_ = std::nullopt, - std::optional read_task_callback_ = std::nullopt, + std::optional read_task_callback_ = std::nullopt); std::optional number_of_current_replica_ = std::nullopt); std::unique_ptr createLocalParallelReplicasReadingStep( const ReadFromMergeTree * analyzed_merge_tree, - std::optional all_ranges_callback_, - std::optional read_task_callback_, - std::optional number_of_current_replica_); + std::optional all_ranges_callback, + std::optional read_task_callback); static constexpr auto name = "ReadFromMergeTree"; String getName() const override { return name; } @@ -291,8 +290,6 @@ private: bool enable_vertical_final = false; bool enable_remove_parts_from_snapshot_optimization = true; std::optional number_of_current_replica; - - friend class ReadFromMergeTreeCoordinated; }; } diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h index afde5f336af..8b463fda395 100644 --- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h +++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h @@ -30,8 +30,8 @@ public: /// needed to report total rows to read void setProgressCallback(ProgressCallback callback); - void initialize(CoordinationMode mode); private: + void initialize(CoordinationMode mode); std::mutex mutex; const size_t replicas_count{0}; diff --git a/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql b/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql index 337b05263d0..7dbc389b55c 100644 --- a/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql +++ b/tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql @@ -6,7 +6,7 @@ SET allow_experimental_parallel_reading_from_replicas=2, max_parallel_replicas=1 SET send_logs_level='error'; -- with local plan for initiator, the query can be executed fast on initator, we can simply not come to the point where unavailable replica can be detected -- therefore disable local plan for now -SELECT count() FROM test_parallel_replicas_unavailable_shards WHERE NOT ignore(*) SETTINGS log_comment = '02769_7b513191-5082-4073-8568-53b86a49da79', allow_experimental_analyzer=0; +SELECT count() FROM test_parallel_replicas_unavailable_shards WHERE NOT ignore(*) SETTINGS log_comment = '02769_7b513191-5082-4073-8568-53b86a49da79', parallel_replicas_local_plan=0; SYSTEM FLUSH LOGS; From 510cb961a1d41b592f986b98f2c40a360cd83314 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 25 Jun 2024 14:47:38 +0000 Subject: [PATCH 040/215] Fix --- src/Interpreters/ClusterProxy/executeQuery.cpp | 3 +-- src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp | 5 ----- src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp | 3 +-- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 4 ++-- src/Processors/QueryPlan/ReadFromMergeTree.h | 4 ++-- 5 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index ab25da090d6..da88f16b504 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -524,8 +524,7 @@ void executeQueryWithParallelReplicas( new_context, processed_stage, coordinator, - std::move(analyzed_read_from_merge_tree), - /*has_missing_objects=*/false); + std::move(analyzed_read_from_merge_tree)); DataStreams input_streams; input_streams.reserve(2); diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp index 1f4f271fa6e..4d4ac69ec25 100644 --- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp +++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp @@ -9,9 +9,6 @@ namespace DB { -namespace -{ - void addConvertingActions(QueryPlan & plan, const Block & header, bool has_missing_objects) { if (blocksHaveEqualStructure(plan.getCurrentDataStream().header, header)) @@ -36,8 +33,6 @@ void addConvertingActions(QueryPlan & plan, const Block & header, bool has_missi plan.addStep(std::move(converting)); } -} - std::unique_ptr createLocalPlan( const ASTPtr & query_ast, const Block & header, diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp index 6822d8b0a71..67c10985408 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -79,9 +79,8 @@ std::unique_ptr createLocalPlanForParallelReplicas( MergeTreeReadTaskCallback read_task_cb = [coordinator](ParallelReadRequest req) -> std::optional { return coordinator->handleRequest(std::move(req)); }; - const auto number_of_local_replica = 0; auto read_from_merge_tree_parallel_replicas - = reading->createLocalParallelReplicasReadingStep(analyzed_merge_tree, all_ranges_cb, read_task_cb, number_of_local_replica); + = reading->createLocalParallelReplicasReadingStep(analyzed_merge_tree, all_ranges_cb, read_task_cb); node->step = std::move(read_from_merge_tree_parallel_replicas); addConvertingActions(*query_plan, header, /*has_missing_objects=*/false); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 1de701cff0b..0ed06639f8c 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -356,8 +356,8 @@ std::unique_ptr ReadFromMergeTree::createLocalParallelReplica log, (analyzed_merge_tree ? analyzed_merge_tree->analyzed_result_ptr : nullptr), true, - all_ranges_callback, - read_task_callback, + all_ranges_callback_, + read_task_callback_, number_of_local_replica); } diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 59a85413dd9..fd66a8af126 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -123,12 +123,12 @@ public: AnalysisResultPtr analyzed_result_ptr_, bool enable_parallel_reading_, std::optional all_ranges_callback_ = std::nullopt, - std::optional read_task_callback_ = std::nullopt); + std::optional read_task_callback_ = std::nullopt, std::optional number_of_current_replica_ = std::nullopt); std::unique_ptr createLocalParallelReplicasReadingStep( const ReadFromMergeTree * analyzed_merge_tree, - std::optional all_ranges_callback, + std::optional all_ranges_callback_, std::optional read_task_callback); static constexpr auto name = "ReadFromMergeTree"; From 0eff924475af085f055fb04f514ea656446c9996 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 25 Jun 2024 15:11:02 +0000 Subject: [PATCH 041/215] Use parallel_replicas_local_plan in test --- .../queries/0_stateless/03006_parallel_replicas_prewhere.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/03006_parallel_replicas_prewhere.sql b/tests/queries/0_stateless/03006_parallel_replicas_prewhere.sql index afe6a00cc4d..de43302690a 100644 --- a/tests/queries/0_stateless/03006_parallel_replicas_prewhere.sql +++ b/tests/queries/0_stateless/03006_parallel_replicas_prewhere.sql @@ -27,7 +27,7 @@ EXPLAIN ESTIMATE SELECT count() FROM url_na_log PREWHERE (DateVisit >= toFixedString('2022-08-10', 10)) AND (DateVisit <= '2022-08-20') -SETTINGS allow_experimental_analyzer=0; +SETTINGS parallel_replicas_local_plan=0; -- here parallel replicas uses local snapshot as working set -- so, the estimation can be done @@ -35,7 +35,7 @@ EXPLAIN ESTIMATE SELECT count() FROM url_na_log PREWHERE (DateVisit >= toFixedString('2022-08-10', 10)) AND (DateVisit <= '2022-08-20') -SETTINGS allow_experimental_analyzer=1; +SETTINGS allow_experimental_analyzer=1, parallel_replicas_local_plan=1; DROP POLICY url_na_log_policy0 ON url_na_log; DROP TABLE url_na_log; From 64cfe1628f74c3797455c26b47770012fe5119b1 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 25 Jun 2024 15:21:44 +0000 Subject: [PATCH 042/215] Cleanup ReadFromMergeTree.h --- src/Processors/QueryPlan/ReadFromMergeTree.h | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index fd66a8af126..99205c1de1d 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -199,7 +199,6 @@ public: AnalysisResultPtr getAnalyzedResult() const { return analyzed_result_ptr; } void setAnalyzedResult(AnalysisResultPtr analyzed_result_ptr_) { analyzed_result_ptr = std::move(analyzed_result_ptr_); } - ReadFromMergeTree::AnalysisResult getAnalysisResult() const; const MergeTreeData::DataPartsVector & getParts() const { return prepared_parts; } const std::vector & getAlterConvertionsForParts() const { return alter_conversions_for_parts; } @@ -211,12 +210,6 @@ public: void applyFilters(ActionDAGNodes added_filter_nodes) override; - ReadType getReadType() const - { - chassert(analyzed_result_ptr); - return analyzed_result_ptr->read_type; - } - private: int getSortDirection() const { @@ -281,6 +274,8 @@ private: Pipe spreadMarkRangesAmongStreamsFinal( RangesInDataParts && parts, size_t num_streams, const Names & origin_column_names, const Names & column_names, ActionsDAGPtr & out_projection); + ReadFromMergeTree::AnalysisResult getAnalysisResult() const; + mutable AnalysisResultPtr analyzed_result_ptr; VirtualFields shared_virtual_fields; From 3d35d31655edd271dcdc1b6e3a562edbcb8c17fd Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 25 Jun 2024 16:11:04 +0000 Subject: [PATCH 043/215] More cleanup --- src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp | 10 +++++++--- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 7 ++++--- src/Processors/QueryPlan/ReadFromMergeTree.h | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp index 67c10985408..5f48a12072b 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -69,9 +69,13 @@ std::unique_ptr createLocalPlanForParallelReplicas( chassert(reading); - ReadFromMergeTree * analyzed_merge_tree = nullptr; + ReadFromMergeTree::AnalysisResultPtr analyzed_result_ptr; if (analyzed_read_from_merge_tree.get()) - analyzed_merge_tree = typeid_cast(analyzed_read_from_merge_tree.get()); + { + auto * analyzed_merge_tree = typeid_cast(analyzed_read_from_merge_tree.get()); + if (analyzed_merge_tree) + analyzed_result_ptr = analyzed_merge_tree->getAnalyzedResult(); + } MergeTreeAllRangesCallback all_ranges_cb = [coordinator](InitialAllRangesAnnouncement announcement) { coordinator->handleInitialAllRangesAnnouncement(announcement); }; @@ -80,7 +84,7 @@ std::unique_ptr createLocalPlanForParallelReplicas( { return coordinator->handleRequest(std::move(req)); }; auto read_from_merge_tree_parallel_replicas - = reading->createLocalParallelReplicasReadingStep(analyzed_merge_tree, all_ranges_cb, read_task_cb); + = reading->createLocalParallelReplicasReadingStep(analyzed_result_ptr, all_ranges_cb, read_task_cb); node->step = std::move(read_from_merge_tree_parallel_replicas); addConvertingActions(*query_plan, header, /*has_missing_objects=*/false); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 0ed06639f8c..95c337e2150 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -337,11 +337,12 @@ ReadFromMergeTree::ReadFromMergeTree( } std::unique_ptr ReadFromMergeTree::createLocalParallelReplicasReadingStep( - const ReadFromMergeTree * analyzed_merge_tree, + AnalysisResultPtr analyzed_result_ptr_, std::optional all_ranges_callback_, std::optional read_task_callback_) { const auto number_of_local_replica = 0; + const bool enable_parallel_reading = true; return std::make_unique( prepared_parts, alter_conversions_for_parts, @@ -354,8 +355,8 @@ std::unique_ptr ReadFromMergeTree::createLocalParallelReplica requested_num_streams, max_block_numbers_to_read, log, - (analyzed_merge_tree ? analyzed_merge_tree->analyzed_result_ptr : nullptr), - true, + analyzed_result_ptr_, + enable_parallel_reading, all_ranges_callback_, read_task_callback_, number_of_local_replica); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 99205c1de1d..c9c6030d207 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -127,9 +127,9 @@ public: std::optional number_of_current_replica_ = std::nullopt); std::unique_ptr createLocalParallelReplicasReadingStep( - const ReadFromMergeTree * analyzed_merge_tree, + AnalysisResultPtr analyzed_result_ptr_, std::optional all_ranges_callback_, - std::optional read_task_callback); + std::optional read_task_callback_); static constexpr auto name = "ReadFromMergeTree"; String getName() const override { return name; } From 7c6293a747a813742c663d59342c0c0cc9db4956 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 26 Jun 2024 10:12:09 +0000 Subject: [PATCH 044/215] Fix test_parallel_replicas_no_replicas --- tests/integration/test_parallel_replicas_no_replicas/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_parallel_replicas_no_replicas/test.py b/tests/integration/test_parallel_replicas_no_replicas/test.py index 9f716459643..73ab6fde5a3 100644 --- a/tests/integration/test_parallel_replicas_no_replicas/test.py +++ b/tests/integration/test_parallel_replicas_no_replicas/test.py @@ -46,5 +46,6 @@ def test_skip_all_replicas(start_cluster, skip_unavailable_shards): "max_parallel_replicas": 3, "cluster_for_parallel_replicas": cluster_name, "skip_unavailable_shards": skip_unavailable_shards, + "parallel_replicas_local_plan": 0, }, ) From 372b948d34fe24257e096a6b7d6c9ea03ba0023a Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 27 Jun 2024 20:30:05 +0000 Subject: [PATCH 045/215] Fix: progress bar, reading in order --- src/Interpreters/ClusterProxy/executeQuery.cpp | 16 ++++++++-------- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 10 ++++++---- ...2898_parallel_replicas_progress_bar.reference | 6 ++++++ .../02898_parallel_replicas_progress_bar.sql | 10 +++++----- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index da88f16b504..9302baf57ca 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -500,6 +500,14 @@ void executeQueryWithParallelReplicas( /// do not build local plan for distributed queries for now (address it later) if (settings.allow_experimental_analyzer && settings.parallel_replicas_local_plan && !shard_num) { + auto local_plan = createLocalPlanForParallelReplicas( + query_ast, + header, + new_context, + processed_stage, + coordinator, + std::move(analyzed_read_from_merge_tree)); + auto read_from_remote = std::make_unique( query_ast, new_cluster, @@ -518,14 +526,6 @@ void executeQueryWithParallelReplicas( auto remote_plan = std::make_unique(); remote_plan->addStep(std::move(read_from_remote)); - auto local_plan = createLocalPlanForParallelReplicas( - query_ast, - header, - new_context, - processed_stage, - coordinator, - std::move(analyzed_read_from_merge_tree)); - DataStreams input_streams; input_streams.reserve(2); input_streams.emplace_back(local_plan->getCurrentDataStream()); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 7cf574ddf5c..e4e251b694e 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -586,12 +586,14 @@ Pipe ReadFromMergeTree::readInOrder( context); } - /// Actually it means that parallel reading from replicas enabled - /// and we have to collaborate with initiator. + /// Actually it means that parallel reading from replicas enabled and read snapshot is not local - + /// we can't rely on local snapshot /// In this case we won't set approximate rows, because it will be accounted multiple times. /// Also do not count amount of read rows if we read in order of sorting key, /// because we don't know actual amount of read rows in case when limit is set. - bool set_rows_approx = !is_parallel_reading_from_replicas && !reader_settings.read_in_order; + const UInt64 in_order_limit = query_info.input_order_info ? query_info.input_order_info->limit : 0; + const bool set_total_rows_approx + = !(is_parallel_reading_from_replicas && context->canUseParallelReplicasOnFollower()) && !in_order_limit; Pipes pipes; for (size_t i = 0; i < parts_with_ranges.size(); ++i) @@ -621,7 +623,7 @@ Pipe ReadFromMergeTree::readInOrder( processor->addPartLevelToChunk(isQueryWithFinal()); auto source = std::make_shared(std::move(processor)); - if (set_rows_approx) + if (set_total_rows_approx) source->addTotalRowsApprox(total_rows); pipes.emplace_back(std::move(source)); diff --git a/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.reference b/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.reference index 380aac4dbe8..c66597436f3 100644 --- a/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.reference +++ b/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.reference @@ -1,2 +1,8 @@ 3000 1000 3999 2499.5 1 +1998 2944475297004403859 +1999 254596732598015005 +2000 6863370867519437063 +2001 17844331710293705251 +2002 1587587338113897332 +1 diff --git a/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.sql b/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.sql index 42f8091db08..d3bf228e0fb 100644 --- a/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.sql +++ b/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.sql @@ -26,12 +26,12 @@ WHERE query_id in (select query_id from system.query_log where current_database AND message LIKE '%Total rows to read: 3000%' SETTINGS allow_experimental_parallel_reading_from_replicas=0; -- reading in order coordinator --- SELECT k, sipHash64(v) FROM t1 order by k limit 5 offset 998 SETTINGS optimize_read_in_order=1, log_comment='02898_inorder_190aed82-2423-413b-ad4c-24dcca50f65b'; +SELECT k, sipHash64(v) FROM t1 order by k limit 5 offset 998 SETTINGS optimize_read_in_order=1, parallel_replicas_local_plan=0, log_comment='02898_inorder_190aed82-2423-413b-ad4c-24dcca50f65b'; --- SYSTEM FLUSH LOGS; --- SELECT count() > 0 FROM system.text_log --- WHERE query_id in (select query_id from system.query_log where current_database = currentDatabase() AND log_comment='02898_inorder_190aed82-2423-413b-ad4c-24dcca50f65b') --- AND message LIKE '%Updated total rows to read: added % rows, total 3000 rows%' SETTINGS allow_experimental_parallel_reading_from_replicas=0; +SYSTEM FLUSH LOGS; +SELECT count() > 0 FROM system.text_log +WHERE query_id in (select query_id from system.query_log where current_database = currentDatabase() AND log_comment='02898_inorder_190aed82-2423-413b-ad4c-24dcca50f65b') + AND message LIKE '%Updated total rows to read: added % rows, total 3000 rows%' SETTINGS allow_experimental_parallel_reading_from_replicas=0; DROP TABLE t1 SYNC; DROP TABLE t2 SYNC; From ebcf455f4a88043234fb19770bfc29958789c090 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 28 Jun 2024 22:17:34 +0000 Subject: [PATCH 046/215] Fix: progress bar for reading in order --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 9 +++++---- .../MergeTree/ParallelReplicasReadingCoordinator.cpp | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index e4e251b694e..08c6989242e 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -592,8 +592,10 @@ Pipe ReadFromMergeTree::readInOrder( /// Also do not count amount of read rows if we read in order of sorting key, /// because we don't know actual amount of read rows in case when limit is set. const UInt64 in_order_limit = query_info.input_order_info ? query_info.input_order_info->limit : 0; - const bool set_total_rows_approx - = !(is_parallel_reading_from_replicas && context->canUseParallelReplicasOnFollower()) && !in_order_limit; + const bool parallel_replicas_remote_plan_for_initiator = is_parallel_reading_from_replicas + && !context->getSettingsRef().parallel_replicas_local_plan && context->canUseParallelReplicasOnInitiator(); + const bool parallel_replicas_follower = is_parallel_reading_from_replicas && context->canUseParallelReplicasOnFollower(); + const bool set_total_rows_approx = !parallel_replicas_follower && !parallel_replicas_remote_plan_for_initiator && !in_order_limit; Pipes pipes; for (size_t i = 0; i < parts_with_ranges.size(); ++i) @@ -1968,8 +1970,7 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons mode = CoordinationMode::ReverseOrder; break; case ReadFromMergeTree::ReadType::ParallelReplicas: - chassert(false); - UNREACHABLE(); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Read type can't be ParallelReplicas on initiator"); } chassert(number_of_current_replica.has_value()); diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp index 601ab3aeb70..f66cfdafa1a 100644 --- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp +++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp @@ -888,9 +888,8 @@ void InOrderCoordinator::doHandleInitialAllRangesAnnouncement(InitialAllRa ++stats[announcement.replica_num].number_of_requests; - /// FIXME: this code updating total_rows_to_read but it needs to be done only once since we're taking working set from initiator - /// util I missing something, it seems this code is not necessary if working set is taken from initiator (todo: check it) - if (new_rows_to_read > 0 && progress_callback) + // progress_callback is not set when local plan is used for initiator + if (progress_callback && new_rows_to_read > 0) { Progress progress; progress.total_rows_to_read = new_rows_to_read; @@ -1052,6 +1051,7 @@ void ParallelReplicasReadingCoordinator::initialize(CoordinationMode mode) break; } + // progress_callback is not set when local plan is used for initiator if (progress_callback) pimpl->setProgressCallback(std::move(progress_callback)); From 37fbf905dda2cdeb683488b46ff364531d1ac2d5 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Sun, 30 Jun 2024 20:33:43 +0000 Subject: [PATCH 047/215] Use only local snapshot for in order coordinator --- .../ParallelReplicasReadingCoordinator.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp index f66cfdafa1a..b510da13d90 100644 --- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp +++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp @@ -836,6 +836,7 @@ public: Parts all_parts_to_read; size_t total_rows_to_read = 0; + bool state_initialized{false}; LoggerPtr log = getLogger(fmt::format("{}{}", magic_enum::enum_name(mode), "Coordinator")); }; @@ -857,6 +858,11 @@ void InOrderCoordinator::doHandleInitialAllRangesAnnouncement(InitialAllRa { LOG_TRACE(log, "Received an announcement {}", announcement.describe()); + ++stats[announcement.replica_num].number_of_requests; + + if (state_initialized) + return; + size_t new_rows_to_read = 0; /// To get rid of duplicates @@ -886,7 +892,7 @@ void InOrderCoordinator::doHandleInitialAllRangesAnnouncement(InitialAllRa std::sort(ranges.begin(), ranges.end()); } - ++stats[announcement.replica_num].number_of_requests; + state_initialized = true; // progress_callback is not set when local plan is used for initiator if (progress_callback && new_rows_to_read > 0) @@ -923,8 +929,15 @@ ParallelReadResponse InOrderCoordinator::handleRequest(ParallelReadRequest if (global_part_it == all_parts_to_read.end()) continue; + if (global_part_it->replicas.empty()) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Part {} requested by replica {} is not registered in working set", + part.info.getPartNameV1(), + request.replica_num); + if (!global_part_it->replicas.contains(request.replica_num)) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Part {} doesn't exist on replica {} according to the global state", part.info.getPartNameV1(), request.replica_num); + continue; size_t current_mark_size = 0; From 6b3750ff83f0801322ac0870a30339e34e9182aa Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 1 Jul 2024 08:28:54 +0000 Subject: [PATCH 048/215] Clean up --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 3 ++- .../queries/0_stateless/02841_parallel_replicas_summary.sh | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 7d798e2a399..8ecb36aab7f 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -1954,7 +1954,8 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons { auto result = getAnalysisResult(); - if (is_parallel_reading_from_replicas && context->canUseParallelReplicasOnInitiator()) + if (is_parallel_reading_from_replicas && context->canUseParallelReplicasOnInitiator() + && context->getSettingsRef().parallel_replicas_local_plan) { CoordinationMode mode = CoordinationMode::Default; switch (result.read_type) diff --git a/tests/queries/0_stateless/02841_parallel_replicas_summary.sh b/tests/queries/0_stateless/02841_parallel_replicas_summary.sh index bff56feacbf..01e2883c547 100755 --- a/tests/queries/0_stateless/02841_parallel_replicas_summary.sh +++ b/tests/queries/0_stateless/02841_parallel_replicas_summary.sh @@ -27,7 +27,7 @@ $CLICKHOUSE_CLIENT --query "CREATE TABLE replicas_summary (n Int64) ENGINE = Mer query_id_base="02841_summary_$CLICKHOUSE_DATABASE" -# TODO: rethink the test, for now temporary disable allow_experimental_analyzer +# TODO: rethink the test, for now temporary disable parallel_replicas_local_plan echo " SELECT * FROM replicas_summary @@ -38,7 +38,7 @@ echo " allow_experimental_parallel_reading_from_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, interactive_delay=0, - allow_experimental_analyzer=0 + parallel_replicas_local_plan=0 "\ | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&wait_end_of_query=1&query_id=${query_id_base}_interactive_0" --data-binary @- -vvv 2>&1 \ | grep "Summary" | grep -cv '"read_rows":"0"' @@ -53,7 +53,7 @@ echo " allow_experimental_parallel_reading_from_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, interactive_delay=99999999999, - allow_experimental_analyzer=0 + parallel_replicas_local_plan=0 "\ | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&wait_end_of_query=1&query_id=${query_id_base}_interactive_high" --data-binary @- -vvv 2>&1 \ | grep "Summary" | grep -cv '"read_rows":"0"' From 105d39b09f241f4b48b70425ba07288d32ae107f Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 2 Jul 2024 15:06:19 +0000 Subject: [PATCH 049/215] Twick number of threads --- src/Planner/PlannerJoinTree.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 094562a2837..a4a8dd08561 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -958,6 +958,18 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres table_expression_query_info.storage_limits, std::move(reading_step)); query_plan = std::move(query_plan_parallel_replicas); + + if (settings.parallel_replicas_local_plan) + { + const auto old_max_threads = query_plan.getMaxThreads(); + query_plan.setMaxThreads(old_max_threads * 2); + + LOG_TRACE( + getLogger("Planner"), + "Increase max threads from {} to {} to have similar number of threads to remote plan", + old_max_threads, + query_plan.getMaxThreads()); + } } else { From 04c3661b0b18eb4be82ecf4261ded70fa52d7bcc Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 12 Jul 2024 13:42:25 +0000 Subject: [PATCH 050/215] Randomize parallel_replicas_local_plan --- src/Interpreters/ClusterProxy/executeQuery.cpp | 2 -- src/Planner/PlannerJoinTree.cpp | 12 ------------ tests/clickhouse-test | 1 + .../02731_parallel_replicas_join_subquery.sql | 1 + .../0_stateless/02771_parallel_replicas_analyzer.sql | 2 ++ 5 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 832f04913c7..6039c545085 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -29,8 +29,6 @@ #include #include #include -#include -#include #include #include diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 1d9972ae03f..6d8d58fc8ef 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -998,18 +998,6 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres table_expression_query_info.storage_limits, std::move(reading_step)); query_plan = std::move(query_plan_parallel_replicas); - - if (settings.parallel_replicas_local_plan) - { - const auto old_max_threads = query_plan.getMaxThreads(); - query_plan.setMaxThreads(old_max_threads * 2); - - LOG_TRACE( - getLogger("Planner"), - "Increase max threads from {} to {} to have similar number of threads to remote plan", - old_max_threads, - query_plan.getMaxThreads()); - } } else { diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 958dde0606f..77e984aa960 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -832,6 +832,7 @@ class SettingsRandomizer: "cross_join_min_bytes_to_compress": lambda: random.choice([0, 1, 100000000]), "min_external_table_block_size_bytes": lambda: random.choice([0, 1, 100000000]), "max_parsing_threads": lambda: random.choice([0, 1, 10]), + "parallel_replicas_local_plan": lambda: random.randint(0, 1), } @staticmethod diff --git a/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.sql b/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.sql index e91e2a19526..f15e7631753 100644 --- a/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.sql +++ b/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.sql @@ -23,6 +23,7 @@ LIMIT 100; SET max_parallel_replicas = 3; SET cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; +SET parallel_replicas_local_plan = 1; SET joined_subquery_requires_alias = 0; SELECT '=============== INNER QUERY (NO PARALLEL) ==============='; diff --git a/tests/queries/0_stateless/02771_parallel_replicas_analyzer.sql b/tests/queries/0_stateless/02771_parallel_replicas_analyzer.sql index 7e27507ada9..3a19768c0cc 100644 --- a/tests/queries/0_stateless/02771_parallel_replicas_analyzer.sql +++ b/tests/queries/0_stateless/02771_parallel_replicas_analyzer.sql @@ -17,6 +17,8 @@ INSERT INTO join_inner_table__fuzz_146_replicated SELECT CAST('833c9e22-c245-4eb5-8745-117a9a1f26b1', 'UUID') AS id, CAST(rowNumberInAllBlocks(), 'String') AS key, * FROM generateRandom('number Int64, value1 String, value2 String, time Int64', 1, 10, 2) LIMIT 10; +SET parallel_replicas_local_plan = 1; + -- Simple query with analyzer and pure parallel replicas SELECT number FROM join_inner_table__fuzz_146_replicated From 4e44ecf286b85d56ba4c17bc6877d62fff98828b Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 12 Jul 2024 13:52:33 +0000 Subject: [PATCH 051/215] Fixate setting in 02967_parallel_replicas_joins_and_analyzer --- .../02967_parallel_replicas_joins_and_analyzer.sql.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.sql.j2 b/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.sql.j2 index 54505b147a3..fd03e4c0857 100644 --- a/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.sql.j2 +++ b/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.sql.j2 @@ -10,6 +10,8 @@ insert into tab1 select number, number, number from numbers(16); insert into tab2 select number * 2, number * 2 from numbers(8); insert into tab3 select number * 4, number * 4 from numbers(4); +set parallel_replicas_local_plan=1; + {% for use_global_in in [0, 1] -%} -- { echoOn } From e22d6035fa3496108b2c9619dcb3fa54c131bf6e Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 12 Jul 2024 14:11:04 +0000 Subject: [PATCH 052/215] Cleanup --- tests/queries/0_stateless/02404_memory_bound_merging.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/queries/0_stateless/02404_memory_bound_merging.sql b/tests/queries/0_stateless/02404_memory_bound_merging.sql index 0167c27ae44..112640a2e2c 100644 --- a/tests/queries/0_stateless/02404_memory_bound_merging.sql +++ b/tests/queries/0_stateless/02404_memory_bound_merging.sql @@ -66,7 +66,6 @@ insert into pr_t select number % 1000, number % 1000 from numbers_mt(1e6); set allow_experimental_parallel_reading_from_replicas = 1; set parallel_replicas_for_non_replicated_merge_tree = 1; set max_parallel_replicas = 3; -set use_hedged_requests = 0; set cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; set distributed_aggregation_memory_efficient=1; From b952a11f35c93b4df1e97325119c6295791ec748 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 12 Jul 2024 21:20:23 +0000 Subject: [PATCH 053/215] Fix 00177_memory_bound_merging --- tests/queries/1_stateful/00177_memory_bound_merging.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/1_stateful/00177_memory_bound_merging.sh b/tests/queries/1_stateful/00177_memory_bound_merging.sh index 74b5e80e059..1110ab9a61d 100755 --- a/tests/queries/1_stateful/00177_memory_bound_merging.sh +++ b/tests/queries/1_stateful/00177_memory_bound_merging.sh @@ -64,7 +64,7 @@ test3() { FROM test.hits WHERE CounterID = 1704509 AND UserID = 4322253409885123546 GROUP BY URL, EventDate - SETTINGS optimize_aggregation_in_order = 1, enable_memory_bound_merging_of_aggregation_results = 1, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, max_parallel_replicas = 3 + SETTINGS optimize_aggregation_in_order = 1, enable_memory_bound_merging_of_aggregation_results = 1, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, max_parallel_replicas = 3, parallel_replicas_local_plan=1 ) WHERE explain LIKE '%Aggr%Transform%' OR explain LIKE '%InOrder%'" } From 32dc3fe8d1c0384221cd1f27adc53bfe45a01e28 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Sat, 13 Jul 2024 21:19:14 +0000 Subject: [PATCH 054/215] Simplify code --- src/Interpreters/ClusterProxy/executeQuery.cpp | 4 +--- src/Processors/QueryPlan/ReadFromRemote.cpp | 10 ++-------- .../02898_parallel_replicas_progress_bar.sql | 1 + 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 6039c545085..79f4344e6cc 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -519,9 +519,7 @@ void executeQueryWithParallelReplicas( "`cluster_for_parallel_replicas` setting refers to cluster with several shards. Expected a cluster with one shard"); } - auto replica_count = new_cluster->getShardsInfo().begin()->getAllNodeCount(); - if (settings.max_parallel_replicas < replica_count) - replica_count = settings.max_parallel_replicas; + const auto replica_count = std::min(settings.max_parallel_replicas.value, new_cluster->getShardsInfo().begin()->getAllNodeCount()); auto coordinator = std::make_shared(replica_count, settings.parallel_replicas_mark_segment_size); diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 26c57c4a760..3da22265c5c 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -458,7 +458,6 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder pools_to_use.reserve(shuffled_pool.size()); if (exclude_local_replica) { - std::vector local_addr_possitions; for (auto & pool : shuffled_pool) { const auto & hostname = pool.pool->getHost(); @@ -466,17 +465,12 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder begin(shard.local_addresses), end(shard.local_addresses), [&hostname](const Cluster::Address & local_addr) { return hostname == local_addr.host_name; }); - if (it != shard.local_addresses.end()) + if (it == shard.local_addresses.end()) { - pool.pool.reset(); + pools_to_use.push_back(pool.pool); } } } - for (const auto & pool : shuffled_pool) - { - if (pool.pool) - pools_to_use.push_back(pool.pool); - } LOG_DEBUG( getLogger("ReadFromParallelRemoteReplicasStep"), diff --git a/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.sql b/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.sql index d3bf228e0fb..9348ea1dc32 100644 --- a/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.sql +++ b/tests/queries/0_stateless/02898_parallel_replicas_progress_bar.sql @@ -26,6 +26,7 @@ WHERE query_id in (select query_id from system.query_log where current_database AND message LIKE '%Total rows to read: 3000%' SETTINGS allow_experimental_parallel_reading_from_replicas=0; -- reading in order coordinator +-- disable parallel_replicas_local_plan since the test relay on traces which only present in case of no local plan SELECT k, sipHash64(v) FROM t1 order by k limit 5 offset 998 SETTINGS optimize_read_in_order=1, parallel_replicas_local_plan=0, log_comment='02898_inorder_190aed82-2423-413b-ad4c-24dcca50f65b'; SYSTEM FLUSH LOGS; From 0c986cca7c91a462ec77fc6a198e65fdb0ef4b10 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Sun, 14 Jul 2024 16:37:00 +0000 Subject: [PATCH 055/215] Fix in-order coordinator + register all replicas for parts in working set --- .../MergeTree/ParallelReplicasReadingCoordinator.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp index 14c706c77ef..ff309f17893 100644 --- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp +++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp @@ -863,9 +863,6 @@ void InOrderCoordinator::doHandleInitialAllRangesAnnouncement(InitialAllRa ++stats[announcement.replica_num].number_of_requests; - if (state_initialized) - return; - size_t new_rows_to_read = 0; /// To get rid of duplicates @@ -874,13 +871,16 @@ void InOrderCoordinator::doHandleInitialAllRangesAnnouncement(InitialAllRa auto the_same_it = std::find_if(all_parts_to_read.begin(), all_parts_to_read.end(), [&part] (const Part & other) { return other.description.info == part.info; }); - /// We have the same part - add the info about presence on current replica to it + /// We have the same part - add the info about presence on the corresponding replica to it if (the_same_it != all_parts_to_read.end()) { the_same_it->replicas.insert(announcement.replica_num); continue; } + if (state_initialized) + continue; + auto covering_or_the_same_it = std::find_if(all_parts_to_read.begin(), all_parts_to_read.end(), [&part] (const Part & other) { return other.description.info.contains(part.info) || part.info.contains(other.description.info); }); From 9f20e33d50810f20ea17355512abed0a04a495a2 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Sun, 14 Jul 2024 17:52:51 +0000 Subject: [PATCH 056/215] Fix --- src/Processors/QueryPlan/ReadFromRemote.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 3da22265c5c..712f714376b 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -456,9 +456,9 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder std::vector pools_to_use; pools_to_use.reserve(shuffled_pool.size()); - if (exclude_local_replica) + for (const auto & pool : shuffled_pool) { - for (auto & pool : shuffled_pool) + if (exclude_local_replica) { const auto & hostname = pool.pool->getHost(); auto it = std::find_if( @@ -466,9 +466,11 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder end(shard.local_addresses), [&hostname](const Cluster::Address & local_addr) { return hostname == local_addr.host_name; }); if (it == shard.local_addresses.end()) - { pools_to_use.push_back(pool.pool); - } + } + else + { + pools_to_use.push_back(pool.pool); } } From f7befaf68e19d60fe3678c15b7367b815ce6da17 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Sun, 14 Jul 2024 18:46:14 +0000 Subject: [PATCH 057/215] Polish + comments --- src/Processors/QueryPlan/ReadFromRemote.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 712f714376b..c05bda62648 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -474,18 +474,17 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder } } + pools_to_use.resize(std::min(pools_to_use.size(), all_replicas_count)); + // if local plan is used for local replica, we should exclude one remote replica + if (exclude_local_replica && !pools_to_use.empty()) + pools_to_use.resize(all_replicas_count - 1); + LOG_DEBUG( getLogger("ReadFromParallelRemoteReplicasStep"), "Number of pools to use is {}. Originally {}", pools_to_use.size(), shuffled_pool.size()); - if (pools_to_use.size() > all_replicas_count) - pools_to_use.resize(all_replicas_count); - - if (exclude_local_replica && !pools_to_use.empty()) - pools_to_use.resize(all_replicas_count - 1); - if (pools_to_use.empty()) return; @@ -497,7 +496,7 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder LOG_DEBUG(getLogger("ReadFromParallelRemoteReplicasStep"), "Addresses to use: {}", pool_addresses); } - /// local replicas has number 0 + /// when using local plan for local replica, local replica has 0 number size_t offset = (exclude_local_replica ? 1 : 0); for (size_t i = 0 + offset; i < all_replicas_count; ++i) { From 261ff133b82a61885a54e10ac890be2b8eb44831 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 15 Jul 2024 12:03:12 +0000 Subject: [PATCH 058/215] Fix 02784_parallel_replicas_automatic_decision_join --- .../02784_parallel_replicas_automatic_decision.sh | 1 - ...parallel_replicas_automatic_decision_join.reference | 10 +++++----- .../02784_parallel_replicas_automatic_decision_join.sh | 10 +++++----- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision.sh b/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision.sh index dd9b5c3f6d9..e5658f31a34 100755 --- a/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision.sh +++ b/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision.sh @@ -6,7 +6,6 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) CLICKHOUSE_CLIENT_TRACE=${CLICKHOUSE_CLIENT/"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"/"--send_logs_level=trace"} function were_parallel_replicas_used () { - # Not using current_database = '$CLICKHOUSE_DATABASE' as nested parallel queries aren't run with it $CLICKHOUSE_CLIENT --query " SELECT initial_query_id, diff --git a/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision_join.reference b/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision_join.reference index 521e3e2edbc..595b35db610 100644 --- a/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision_join.reference +++ b/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision_join.reference @@ -4,8 +4,8 @@ 02784_automatic_parallel_replicas_join-default_simple_join_1M_pure 10 estimated parallel replicas 02784_automatic_parallel_replicas_join-default_simple_join_300k_pure 3 estimated parallel replicas 02784_automatic_parallel_replicas_join-default_simple_join_300k_pure 33 estimated parallel replicas -02784_automatic_parallel_replicas_join-default_simple_join_0_pure Distinct parallel subqueries: 2 Used parallel replicas: true -02784_automatic_parallel_replicas_join-default_simple_join_10M_pure Distinct parallel subqueries: 0 Used parallel replicas: false -02784_automatic_parallel_replicas_join-default_simple_join_5M_pure Distinct parallel subqueries: 1 Used parallel replicas: true -02784_automatic_parallel_replicas_join-default_simple_join_1M_pure Distinct parallel subqueries: 1 Used parallel replicas: true -02784_automatic_parallel_replicas_join-default_simple_join_300k_pure Distinct parallel subqueries: 2 Used parallel replicas: true +02784_automatic_parallel_replicas_join-default_simple_join_0_pure Used parallel replicas: true +02784_automatic_parallel_replicas_join-default_simple_join_10M_pure Used parallel replicas: false +02784_automatic_parallel_replicas_join-default_simple_join_5M_pure Used parallel replicas: true +02784_automatic_parallel_replicas_join-default_simple_join_1M_pure Used parallel replicas: true +02784_automatic_parallel_replicas_join-default_simple_join_300k_pure Used parallel replicas: true diff --git a/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision_join.sh b/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision_join.sh index 801cd22b79f..238d63e6980 100755 --- a/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision_join.sh +++ b/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision_join.sh @@ -9,17 +9,17 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) CLICKHOUSE_CLIENT_TRACE=${CLICKHOUSE_CLIENT/"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"/"--send_logs_level=trace"} function were_parallel_replicas_used () { - # Not using current_database = '$CLICKHOUSE_DATABASE' as nested parallel queries aren't run with it $CLICKHOUSE_CLIENT --query " SELECT initial_query_id, - concat('Distinct parallel subqueries: ' , countDistinctIf(query, initial_query_id != query_id)::String) as subqueries_parallelized, - concat('Used parallel replicas: ', (countIf(initial_query_id != query_id) != 0)::bool::String) as used + concat('Used parallel replicas: ', (ProfileEvents['ParallelReplicasUsedCount'] > 0)::bool::String) as used FROM system.query_log WHERE event_date >= yesterday() AND initial_query_id LIKE '$1%' - GROUP BY initial_query_id - ORDER BY min(event_time_microseconds) ASC + AND query_id = initial_query_id + AND type = 'QueryFinish' + AND current_database = '$CLICKHOUSE_DATABASE' + ORDER BY event_time_microseconds ASC FORMAT TSV" } From 02a2f509163e5df370440b4eed792dcbd995942c Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 15 Jul 2024 18:37:19 +0000 Subject: [PATCH 059/215] Fix clang warning --- src/Interpreters/ClusterProxy/executeQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 79f4344e6cc..4a637db3887 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -519,7 +519,7 @@ void executeQueryWithParallelReplicas( "`cluster_for_parallel_replicas` setting refers to cluster with several shards. Expected a cluster with one shard"); } - const auto replica_count = std::min(settings.max_parallel_replicas.value, new_cluster->getShardsInfo().begin()->getAllNodeCount()); + const auto replica_count = std::min(settings.max_parallel_replicas.value, new_cluster->getShardsInfo().begin()->getAllNodeCount()); auto coordinator = std::make_shared(replica_count, settings.parallel_replicas_mark_segment_size); From 860b18c251a3492682d410fbcc54576e87f4ca25 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 15 Jul 2024 20:40:55 +0000 Subject: [PATCH 060/215] Update test for in-(reverse)-order coordinator --- .../ParallelReplicasReadingCoordinator.cpp | 2 +- src/Storages/MergeTree/RequestResponse.cpp | 4 +--- ...02950_parallel_replicas_used_count.reference | 10 ++++++++-- .../02950_parallel_replicas_used_count.sql | 17 ++++++++++++----- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp index ff309f17893..e1c0d87837a 100644 --- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp +++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp @@ -918,7 +918,7 @@ ParallelReadResponse InOrderCoordinator::handleRequest(ParallelReadRequest "Replica {} decided to read in {} mode, not in {}. This is a bug", request.replica_num, magic_enum::enum_name(request.mode), magic_enum::enum_name(mode)); - LOG_TRACE(log, "Got request from replica {}, data {}", request.replica_num, request.describe()); + LOG_TRACE(log, "Got read request: {}", request.describe()); ParallelReadResponse response; response.description = request.description; diff --git a/src/Storages/MergeTree/RequestResponse.cpp b/src/Storages/MergeTree/RequestResponse.cpp index 2ce0e20dcd2..33cd935c88b 100644 --- a/src/Storages/MergeTree/RequestResponse.cpp +++ b/src/Storages/MergeTree/RequestResponse.cpp @@ -44,9 +44,7 @@ void ParallelReadRequest::serialize(WriteBuffer & out) const String ParallelReadRequest::describe() const { - String result; - result += fmt::format("replica_num: {} \n", replica_num); - result += fmt::format("min_num_of_marks: {} \n", min_number_of_marks); + String result = fmt::format("replica_num {}, min_num_of_marks {}, ", replica_num, min_number_of_marks); result += description.describe(); return result; } diff --git a/tests/queries/0_stateless/02950_parallel_replicas_used_count.reference b/tests/queries/0_stateless/02950_parallel_replicas_used_count.reference index 21b7b527b7a..cc8284bfeac 100644 --- a/tests/queries/0_stateless/02950_parallel_replicas_used_count.reference +++ b/tests/queries/0_stateless/02950_parallel_replicas_used_count.reference @@ -1,8 +1,14 @@ -100 4950 +10000 49995000 1 89 90 91 92 93 -1 +3 +93 +92 +91 +90 +89 +3 diff --git a/tests/queries/0_stateless/02950_parallel_replicas_used_count.sql b/tests/queries/0_stateless/02950_parallel_replicas_used_count.sql index 22f55acd365..69f1dc47ded 100644 --- a/tests/queries/0_stateless/02950_parallel_replicas_used_count.sql +++ b/tests/queries/0_stateless/02950_parallel_replicas_used_count.sql @@ -2,11 +2,12 @@ DROP TABLE IF EXISTS test; CREATE TABLE test (k UInt64, v String) ENGINE = MergeTree -ORDER BY k; +ORDER BY k +SETTINGS index_granularity=1; -INSERT INTO test SELECT number, toString(number) FROM numbers(100); +INSERT INTO test SELECT number, toString(number) FROM numbers(10_000); -SET allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; +SET allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 3, parallel_replicas_for_non_replicated_merge_tree=1, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; -- default coordinator SELECT count(), sum(k) @@ -14,12 +15,18 @@ FROM test SETTINGS log_comment = '02950_parallel_replicas_used_replicas_count'; SYSTEM FLUSH LOGS; -SELECT ProfileEvents['ParallelReplicasUsedCount'] FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02950_parallel_replicas_used_replicas_count' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS allow_experimental_parallel_reading_from_replicas=0; +SELECT ProfileEvents['ParallelReplicasUsedCount'] > 0 FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02950_parallel_replicas_used_replicas_count' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS allow_experimental_parallel_reading_from_replicas=0; -- In order coordinator -SELECT k FROM test order by k limit 5 offset 89 SETTINGS optimize_read_in_order=1, log_comment='02950_parallel_replicas_used_replicas_count_2'; +SELECT k FROM test order by k limit 5 offset 89 SETTINGS optimize_read_in_order=1, log_comment='02950_parallel_replicas_used_replicas_count_2', merge_tree_min_rows_for_concurrent_read=1, max_threads=1; SYSTEM FLUSH LOGS; SELECT ProfileEvents['ParallelReplicasUsedCount'] FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02950_parallel_replicas_used_replicas_count_2' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS allow_experimental_parallel_reading_from_replicas=0; +-- In reverse order coordinator +SELECT k FROM test order by k desc limit 5 offset 9906 SETTINGS optimize_read_in_order=1, log_comment='02950_parallel_replicas_used_replicas_count_3', merge_tree_min_rows_for_concurrent_read=1, max_threads=1; + +SYSTEM FLUSH LOGS; +SELECT ProfileEvents['ParallelReplicasUsedCount'] FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02950_parallel_replicas_used_replicas_count_3' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS allow_experimental_parallel_reading_from_replicas=0; + DROP TABLE test; From 69c1e68359f08ee648eba819f445cbde9b8c82e0 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 16 Jul 2024 07:48:40 +0000 Subject: [PATCH 061/215] Fix clang-tidy warning: remove include duplicate --- src/Interpreters/Context.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index fc1e87e7b7e..2602afd8b78 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -51,7 +51,6 @@ #include #include #include -#include #include #include #include From 27db36cd4fbc2b0d74e982a3992778c62ccc2482 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 16 Jul 2024 08:46:24 +0000 Subject: [PATCH 062/215] Separate converting actions in separte source file --- .../QueryPlan/ConvertingActions.cpp | 32 +++++++++++++++++++ src/Processors/QueryPlan/ConvertingActions.h | 9 ++++++ .../QueryPlan/DistributedCreateLocalPlan.cpp | 27 +--------------- .../QueryPlan/ParallelReplicasLocalPlan.cpp | 5 +-- 4 files changed, 45 insertions(+), 28 deletions(-) create mode 100644 src/Processors/QueryPlan/ConvertingActions.cpp create mode 100644 src/Processors/QueryPlan/ConvertingActions.h diff --git a/src/Processors/QueryPlan/ConvertingActions.cpp b/src/Processors/QueryPlan/ConvertingActions.cpp new file mode 100644 index 00000000000..ff106ff08c1 --- /dev/null +++ b/src/Processors/QueryPlan/ConvertingActions.cpp @@ -0,0 +1,32 @@ +#include +#include +#include + +namespace DB +{ + +void addConvertingActions(QueryPlan & plan, const Block & header, bool has_missing_objects) +{ + if (blocksHaveEqualStructure(plan.getCurrentDataStream().header, header)) + return; + + auto mode = has_missing_objects ? ActionsDAG::MatchColumnsMode::Position : ActionsDAG::MatchColumnsMode::Name; + + auto get_converting_dag = [mode](const Block & block_, const Block & header_) + { + /// Convert header structure to expected. + /// Also we ignore constants from result and replace it with constants from header. + /// It is needed for functions like `now64()` or `randConstant()` because their values may be different. + return ActionsDAG::makeConvertingActions( + block_.getColumnsWithTypeAndName(), + header_.getColumnsWithTypeAndName(), + mode, + true); + }; + + auto convert_actions_dag = get_converting_dag(plan.getCurrentDataStream().header, header); + auto converting = std::make_unique(plan.getCurrentDataStream(), convert_actions_dag); + plan.addStep(std::move(converting)); +} + +} diff --git a/src/Processors/QueryPlan/ConvertingActions.h b/src/Processors/QueryPlan/ConvertingActions.h new file mode 100644 index 00000000000..6bdf9b8af9a --- /dev/null +++ b/src/Processors/QueryPlan/ConvertingActions.h @@ -0,0 +1,9 @@ +#pragma once + +namespace DB +{ +class QueryPlan; +class Block; + +void addConvertingActions(QueryPlan & plan, const Block & header, bool has_missing_objects); +} diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp index bad0380cb46..eb699858bdf 100644 --- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp +++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp @@ -2,38 +2,13 @@ #include #include -#include #include #include -#include +#include namespace DB { -void addConvertingActions(QueryPlan & plan, const Block & header, bool has_missing_objects) -{ - if (blocksHaveEqualStructure(plan.getCurrentDataStream().header, header)) - return; - - auto mode = has_missing_objects ? ActionsDAG::MatchColumnsMode::Position : ActionsDAG::MatchColumnsMode::Name; - - auto get_converting_dag = [mode](const Block & block_, const Block & header_) - { - /// Convert header structure to expected. - /// Also we ignore constants from result and replace it with constants from header. - /// It is needed for functions like `now64()` or `randConstant()` because their values may be different. - return ActionsDAG::makeConvertingActions( - block_.getColumnsWithTypeAndName(), - header_.getColumnsWithTypeAndName(), - mode, - true); - }; - - auto convert_actions_dag = get_converting_dag(plan.getCurrentDataStream().header, header); - auto converting = std::make_unique(plan.getCurrentDataStream(), convert_actions_dag); - plan.addStep(std::move(converting)); -} - std::unique_ptr createLocalPlan( const ASTPtr & query_ast, const Block & header, diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp index 5f48a12072b..d2e862a3416 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -77,8 +78,8 @@ std::unique_ptr createLocalPlanForParallelReplicas( analyzed_result_ptr = analyzed_merge_tree->getAnalyzedResult(); } - MergeTreeAllRangesCallback all_ranges_cb - = [coordinator](InitialAllRangesAnnouncement announcement) { coordinator->handleInitialAllRangesAnnouncement(announcement); }; + MergeTreeAllRangesCallback all_ranges_cb = [coordinator](InitialAllRangesAnnouncement announcement) + { coordinator->handleInitialAllRangesAnnouncement(std::move(announcement)); }; MergeTreeReadTaskCallback read_task_cb = [coordinator](ParallelReadRequest req) -> std::optional { return coordinator->handleRequest(std::move(req)); }; From 4caf9b6e6fa53adb1e7208c992469175a9344c2f Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 16 Jul 2024 08:57:15 +0000 Subject: [PATCH 063/215] Better diagnostic message formatting --- src/Processors/QueryPlan/ReadFromRemote.cpp | 22 ++++++++------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 3c6c651ae02..caa332c2ebc 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -22,7 +22,7 @@ #include #include -#include +#include namespace DB { @@ -391,15 +391,11 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep( std::vector replicas; replicas.reserve(cluster->getShardsAddresses().front().size()); - bool first_local = false; for (const auto & addr : cluster->getShardsAddresses().front()) { /// skip first local - if (exclude_local_replica && addr.is_local && !first_local) - { - first_local = true; + if (exclude_local_replica && addr.is_local) continue; - } /// replace hostname with replica name if the hostname started with replica namespace, /// it makes description shorter and more readable @@ -409,7 +405,7 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep( replicas.push_back(fmt::format("{}", addr.host_name)); } - auto description = fmt::format("Query: {} Replicas: ", formattedAST(query_ast)) + boost::algorithm::join(replicas, ", "); + auto description = fmt::format("Query: {} Replicas: {}", formattedAST(query_ast), fmt::join(replicas, ", ")); setStepDescription(std::move(description)); } @@ -489,13 +485,11 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder if (pools_to_use.empty()) return; - { - String pool_addresses; - for (const auto & pool : pools_to_use) - pool_addresses += pool->getAddress() + ";"; - - LOG_DEBUG(getLogger("ReadFromParallelRemoteReplicasStep"), "Addresses to use: {}", pool_addresses); - } + std::vector addresses; + addresses.reserve(pools_to_use.size()); + for (const auto & pool : pools_to_use) + addresses.emplace_back(pool->getAddress()); + LOG_DEBUG(getLogger("ReadFromParallelRemoteReplicasStep"), "Addresses to use: {}", fmt::join(addresses, ", ")); /// when using local plan for local replica, local replica has 0 number size_t offset = (exclude_local_replica ? 1 : 0); From 08dc1c8c37d287ddf3cf3a785b6c8334e773840a Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 16 Jul 2024 19:59:16 +0000 Subject: [PATCH 064/215] Fallback to local execution in case of cluster(shard) has only one node --- .../ClusterProxy/executeQuery.cpp | 39 ++++++++++++++++++- src/Interpreters/ClusterProxy/executeQuery.h | 2 + src/Planner/PlannerJoinTree.cpp | 2 +- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 4a637db3887..7207efd9ef6 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -488,12 +488,12 @@ void executeQueryWithParallelReplicas( shard_num = column->getUInt(0); } - const auto shard_count = not_optimized_cluster->getShardCount(); ClusterPtr new_cluster = not_optimized_cluster; /// if got valid shard_num from query initiator, then parallel replicas scope is the specified shard /// shards are numbered in order of appearance in the cluster config if (shard_num > 0) { + const auto shard_count = not_optimized_cluster->getShardCount(); if (shard_num > shard_count) throw Exception( ErrorCodes::LOGICAL_ERROR, @@ -702,6 +702,43 @@ void executeQueryWithParallelReplicasCustomKey( context, query_info.query, storage_id.getDatabaseName(), storage_id.getTableName(), /*table_function_ptr=*/nullptr); executeQueryWithParallelReplicasCustomKey(query_plan, storage_id, query_info, columns, snapshot, processed_stage, header, context); } + +bool canUseParallelReplicasOnInitiator(const ContextPtr & context) +{ + if (!context->canUseParallelReplicasOnInitiator()) + return false; + + auto cluster = context->getClusterForParallelReplicas(); + if (cluster->getShardCount() == 1) + return cluster->getShardsInfo()[0].getAllNodeCount() > 1; + + /// parallel replicas with distributed table + auto scalars = context->hasQueryContext() ? context->getQueryContext()->getScalars() : Scalars{}; + UInt64 shard_num = 0; /// shard_num is 1-based, so 0 - no shard specified + const auto it = scalars.find("_shard_num"); + if (it != scalars.end()) + { + const Block & block = it->second; + const auto & column = block.safeGetByPosition(0).column; + shard_num = column->getUInt(0); + } + if (shard_num > 0) + { + const auto shard_count = cluster->getShardCount(); + if (shard_num > shard_count) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Shard number is greater than shard count: shard_num={} shard_count={} cluster={}", + shard_num, + shard_count, + cluster->getName()); + + return cluster->getShardsInfo().at(shard_num - 1).getAllNodeCount() > 1; + } + + return false; +} + } } diff --git a/src/Interpreters/ClusterProxy/executeQuery.h b/src/Interpreters/ClusterProxy/executeQuery.h index 2c63b7b89f0..2a21f3e8255 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.h +++ b/src/Interpreters/ClusterProxy/executeQuery.h @@ -58,6 +58,8 @@ using AdditionalShardFilterGenerator = std::function; AdditionalShardFilterGenerator getShardFilterGeneratorForCustomKey(const Cluster & cluster, ContextPtr context, const ColumnsDescription & columns); +bool canUseParallelReplicasOnInitiator(const ContextPtr & context); + /// Execute a distributed query, creating a query plan, from which the query pipeline can be built. /// `stream_factory` object encapsulates the logic of creating plans for a different type of query /// (currently SELECT, DESCRIBE). diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 4c21f0d00cc..c1eb2b48045 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -920,7 +920,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres query_plan = std::move(query_plan_parallel_replicas); } } - else if (query_context->canUseParallelReplicasOnInitiator()) + else if (ClusterProxy::canUseParallelReplicasOnInitiator(query_context)) { // (1) find read step QueryPlan::Node * node = query_plan.getRootNode(); From fc693cc982cce7b4aaf71933bd18aa751ed3b7a0 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 16 Jul 2024 21:05:27 +0000 Subject: [PATCH 065/215] Fix 02982_parallel_replicas_unexpected_cluster --- src/Interpreters/ClusterProxy/executeQuery.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 7207efd9ef6..c0d156a8894 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -736,6 +736,12 @@ bool canUseParallelReplicasOnInitiator(const ContextPtr & context) return cluster->getShardsInfo().at(shard_num - 1).getAllNodeCount() > 1; } + if (cluster->getShardCount() > 1) + throw DB::Exception( + ErrorCodes::UNEXPECTED_CLUSTER, + "`cluster_for_parallel_replicas` setting refers to cluster with {} shards. Expected a cluster with one shard", + cluster->getShardCount()); + return false; } From a3d4fd9246b2e1953ed04d75840a9f3399d37ba4 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 17 Jul 2024 10:01:26 +0000 Subject: [PATCH 066/215] Fix 02950_parallel_replicas_used_count --- .../0_stateless/02950_parallel_replicas_used_count.reference | 4 ++-- .../0_stateless/02950_parallel_replicas_used_count.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/02950_parallel_replicas_used_count.reference b/tests/queries/0_stateless/02950_parallel_replicas_used_count.reference index cc8284bfeac..c1265b7ca14 100644 --- a/tests/queries/0_stateless/02950_parallel_replicas_used_count.reference +++ b/tests/queries/0_stateless/02950_parallel_replicas_used_count.reference @@ -5,10 +5,10 @@ 91 92 93 -3 +1 93 92 91 90 89 -3 +1 diff --git a/tests/queries/0_stateless/02950_parallel_replicas_used_count.sql b/tests/queries/0_stateless/02950_parallel_replicas_used_count.sql index 69f1dc47ded..4396ca60e0e 100644 --- a/tests/queries/0_stateless/02950_parallel_replicas_used_count.sql +++ b/tests/queries/0_stateless/02950_parallel_replicas_used_count.sql @@ -21,12 +21,12 @@ SELECT ProfileEvents['ParallelReplicasUsedCount'] > 0 FROM system.query_log WHER SELECT k FROM test order by k limit 5 offset 89 SETTINGS optimize_read_in_order=1, log_comment='02950_parallel_replicas_used_replicas_count_2', merge_tree_min_rows_for_concurrent_read=1, max_threads=1; SYSTEM FLUSH LOGS; -SELECT ProfileEvents['ParallelReplicasUsedCount'] FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02950_parallel_replicas_used_replicas_count_2' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS allow_experimental_parallel_reading_from_replicas=0; +SELECT ProfileEvents['ParallelReplicasUsedCount'] > 0 FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02950_parallel_replicas_used_replicas_count_2' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS allow_experimental_parallel_reading_from_replicas=0; -- In reverse order coordinator SELECT k FROM test order by k desc limit 5 offset 9906 SETTINGS optimize_read_in_order=1, log_comment='02950_parallel_replicas_used_replicas_count_3', merge_tree_min_rows_for_concurrent_read=1, max_threads=1; SYSTEM FLUSH LOGS; -SELECT ProfileEvents['ParallelReplicasUsedCount'] FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02950_parallel_replicas_used_replicas_count_3' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS allow_experimental_parallel_reading_from_replicas=0; +SELECT ProfileEvents['ParallelReplicasUsedCount'] > 0 FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02950_parallel_replicas_used_replicas_count_3' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS allow_experimental_parallel_reading_from_replicas=0; DROP TABLE test; From baf3408b16b3634c59d6635afbb53b849cf55d0e Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 17 Jul 2024 11:45:00 +0000 Subject: [PATCH 067/215] Fix: fallback to local plan w/o PR in case of empty table --- src/Interpreters/ClusterProxy/executeQuery.cpp | 8 +++++++- .../QueryPlan/ParallelReplicasLocalPlan.cpp | 11 ++++++----- src/Processors/QueryPlan/ParallelReplicasLocalPlan.h | 2 +- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index c0d156a8894..a8cf3022a61 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -528,7 +528,7 @@ void executeQueryWithParallelReplicas( /// do not build local plan for distributed queries for now (address it later) if (settings.allow_experimental_analyzer && settings.parallel_replicas_local_plan && !shard_num) { - auto local_plan = createLocalPlanForParallelReplicas( + auto [local_plan, with_parallel_replicas] = createLocalPlanForParallelReplicas( query_ast, header, new_context, @@ -536,6 +536,12 @@ void executeQueryWithParallelReplicas( coordinator, std::move(analyzed_read_from_merge_tree)); + if (!with_parallel_replicas) + { + query_plan = std::move(*local_plan); + return; + } + auto read_from_remote = std::make_unique( query_ast, new_cluster, diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp index d2e862a3416..e9d5ef90e30 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -21,9 +21,7 @@ namespace DB { -void addConvertingActions(QueryPlan & plan, const Block & header, bool has_missing_objects); - -std::unique_ptr createLocalPlanForParallelReplicas( +std::pair, bool> createLocalPlanForParallelReplicas( const ASTPtr & query_ast, const Block & header, ContextPtr context, @@ -68,7 +66,9 @@ std::unique_ptr createLocalPlanForParallelReplicas( node = nullptr; } - chassert(reading); + if (!reading) + /// it can happened if merge tree table is empty, - it'll be replaced with ReadFromPreparedSource + return {std::move(query_plan), false}; ReadFromMergeTree::AnalysisResultPtr analyzed_result_ptr; if (analyzed_read_from_merge_tree.get()) @@ -89,7 +89,8 @@ std::unique_ptr createLocalPlanForParallelReplicas( node->step = std::move(read_from_merge_tree_parallel_replicas); addConvertingActions(*query_plan, header, /*has_missing_objects=*/false); - return query_plan; + + return {std::move(query_plan), true}; } } diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.h b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.h index 123754458a1..2a49be6347a 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.h +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.h @@ -8,7 +8,7 @@ namespace DB { -std::unique_ptr createLocalPlanForParallelReplicas( +std::pair, bool> createLocalPlanForParallelReplicas( const ASTPtr & query_ast, const Block & header, ContextPtr context, From 97f4ec2adbf614842e9b16badb69a8d5c642abe0 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 17 Jul 2024 16:59:35 +0200 Subject: [PATCH 068/215] Read cgroup memory usage in async metrics thread --- programs/keeper/Keeper.cpp | 26 ++-- programs/server/Server.cpp | 34 ++--- src/Common/AsynchronousMetrics.cpp | 13 +- src/Common/AsynchronousMetrics.h | 5 +- src/Common/CgroupsMemoryUsageObserver.cpp | 127 ++++-------------- src/Common/CgroupsMemoryUsageObserver.h | 19 +-- src/Common/MemoryTracker.cpp | 38 ++---- src/Common/MemoryTracker.h | 4 +- .../KeeperAsynchronousMetrics.cpp | 2 +- src/Interpreters/Context.cpp | 13 ++ src/Interpreters/Context.h | 4 + .../ServerAsynchronousMetrics.cpp | 2 +- 12 files changed, 112 insertions(+), 175 deletions(-) diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index 44c2daa33ad..3f6020ad48c 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -399,6 +399,18 @@ try registerDisks(/*global_skip_access_check=*/false); + auto cgroups_memory_observer_wait_time = config().getUInt64("keeper_server.cgroups_memory_observer_wait_time", 15); + try + { + auto cgroups_reader = createCgroupsReader(); + global_context->setCgroupsReader(createCgroupsReader()); + } + catch (...) + { + if (cgroups_memory_observer_wait_time != 0) + tryLogCurrentException(log, "Failed to create cgroups reader"); + } + /// This object will periodically calculate some metrics. KeeperAsynchronousMetrics async_metrics( global_context, @@ -622,21 +634,19 @@ try main_config_reloader->start(); std::optional cgroups_memory_usage_observer; - try + if (cgroups_memory_observer_wait_time != 0) { - auto wait_time = config().getUInt64("keeper_server.cgroups_memory_observer_wait_time", 15); - if (wait_time != 0) + auto cgroups_reader = global_context->getCgroupsReader(); + if (cgroups_reader) { - cgroups_memory_usage_observer.emplace(std::chrono::seconds(wait_time)); + cgroups_memory_usage_observer.emplace(std::chrono::seconds(cgroups_memory_observer_wait_time), global_context->getCgroupsReader()); /// Not calling cgroups_memory_usage_observer->setLimits() here (as for the normal ClickHouse server) because Keeper controls /// its memory usage by other means (via setting 'max_memory_usage_soft_limit'). cgroups_memory_usage_observer->setOnMemoryAmountAvailableChangedFn([&]() { main_config_reloader->reload(); }); cgroups_memory_usage_observer->startThread(); } - } - catch (Exception &) - { - tryLogCurrentException(log, "Disabling cgroup memory observer because of an error during initialization"); + else + LOG_ERROR(log, "Disabling cgroup memory observer because of an error during initialization of cgroups reader"); } diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 053ddaf8d8b..c52b1e037ec 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -897,6 +897,17 @@ try LOG_INFO(log, "Background threads finished in {} ms", watch.elapsedMilliseconds()); }); + try + { + auto cgroups_reader = createCgroupsReader(); + global_context->setCgroupsReader(createCgroupsReader()); + } + catch (...) + { + if (server_settings.cgroups_memory_usage_observer_wait_time != 0) + tryLogCurrentException(log, "Failed to create cgroups reader"); + } + /// This object will periodically calculate some metrics. ServerAsynchronousMetrics async_metrics( global_context, @@ -1456,15 +1467,13 @@ try } std::optional cgroups_memory_usage_observer; - try + if (auto wait_time = server_settings.cgroups_memory_usage_observer_wait_time; wait_time != 0) { - auto wait_time = server_settings.cgroups_memory_usage_observer_wait_time; - if (wait_time != 0) - cgroups_memory_usage_observer.emplace(std::chrono::seconds(wait_time)); - } - catch (Exception &) - { - tryLogCurrentException(log, "Disabling cgroup memory observer because of an error during initialization"); + auto cgroups_reader = global_context->getCgroupsReader(); + if (cgroups_reader) + cgroups_memory_usage_observer.emplace(std::chrono::seconds(wait_time), std::move(cgroups_reader)); + else + LOG_ERROR(log, "Disabling cgroup memory observer because of an error during initialization of cgroups reader"); } std::string cert_path = config().getString("openSSL.server.certificateFile", ""); @@ -1532,15 +1541,6 @@ try total_memory_tracker.setDescription("(total)"); total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking); - if (cgroups_memory_usage_observer) - { - double hard_limit_ratio = new_server_settings.cgroup_memory_watcher_hard_limit_ratio; - double soft_limit_ratio = new_server_settings.cgroup_memory_watcher_soft_limit_ratio; - cgroups_memory_usage_observer->setMemoryUsageLimits( - static_cast(max_server_memory_usage * hard_limit_ratio), - static_cast(max_server_memory_usage * soft_limit_ratio)); - } - size_t merges_mutations_memory_usage_soft_limit = new_server_settings.merges_mutations_memory_usage_soft_limit; size_t default_merges_mutations_server_memory_usage = static_cast(current_physical_server_memory * new_server_settings.merges_mutations_memory_usage_to_ram_ratio); diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index 6309f6079f6..0953ad88697 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -57,10 +57,12 @@ static std::unique_ptr openFileIfExists(const std::stri AsynchronousMetrics::AsynchronousMetrics( unsigned update_period_seconds, - const ProtocolServerMetricsFunc & protocol_server_metrics_func_) + const ProtocolServerMetricsFunc & protocol_server_metrics_func_, + std::shared_ptr cgroups_reader_) : update_period(update_period_seconds) , log(getLogger("AsynchronousMetrics")) , protocol_server_metrics_func(protocol_server_metrics_func_) + , cgroups_reader(std::move(cgroups_reader_)) { #if defined(OS_LINUX) openFileIfExists("/proc/meminfo", meminfo); @@ -669,6 +671,13 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) free_memory_in_allocator_arenas = je_malloc_pdirty * getPageSize(); #endif + if (cgroups_reader != nullptr) + { + rss = cgroups_reader->readMemoryUsage(); + new_values["CgroupsMemoryUsage"] = { rss, + "The amount of physical memory used by the server process, reported by cgroups." }; + } + Int64 difference = rss - amount; /// Log only if difference is high. This is for convenience. The threshold is arbitrary. @@ -681,7 +690,7 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) ReadableSize(rss), ReadableSize(difference)); - MemoryTracker::setRSS(rss, free_memory_in_allocator_arenas); + MemoryTracker::setRSS(rss, /*has_free_memory_in_allocator_arenas_=*/free_memory_in_allocator_arenas > 0); } } diff --git a/src/Common/AsynchronousMetrics.h b/src/Common/AsynchronousMetrics.h index 10a972d2458..0b110f41fc3 100644 --- a/src/Common/AsynchronousMetrics.h +++ b/src/Common/AsynchronousMetrics.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -68,7 +69,8 @@ public: AsynchronousMetrics( unsigned update_period_seconds, - const ProtocolServerMetricsFunc & protocol_server_metrics_func_); + const ProtocolServerMetricsFunc & protocol_server_metrics_func_, + std::shared_ptr cgroups_reader_); virtual ~AsynchronousMetrics(); @@ -91,6 +93,7 @@ private: virtual void logImpl(AsynchronousMetricValues &) {} ProtocolServerMetricsFunc protocol_server_metrics_func; + std::shared_ptr cgroups_reader; std::unique_ptr thread; diff --git a/src/Common/CgroupsMemoryUsageObserver.cpp b/src/Common/CgroupsMemoryUsageObserver.cpp index 02bde0d80b7..b12845df098 100644 --- a/src/Common/CgroupsMemoryUsageObserver.cpp +++ b/src/Common/CgroupsMemoryUsageObserver.cpp @@ -17,13 +17,6 @@ #include #include -#include "config.h" -#if USE_JEMALLOC -# include -#define STRINGIFY_HELPER(x) #x -#define STRINGIFY(x) STRINGIFY_HELPER(x) -#endif - using namespace DB; namespace fs = std::filesystem; @@ -155,15 +148,21 @@ std::optional getCgroupsV1Path() return {default_cgroups_mount / "memory"}; } -std::pair getCgroupsPath() +enum class CgroupsVersion : uint8_t +{ + V1, + V2 +}; + +std::pair getCgroupsPath() { auto v2_path = getCgroupsV2Path(); if (v2_path.has_value()) - return {*v2_path, CgroupsMemoryUsageObserver::CgroupsVersion::V2}; + return {*v2_path, CgroupsVersion::V2}; auto v1_path = getCgroupsV1Path(); if (v1_path.has_value()) - return {*v1_path, CgroupsMemoryUsageObserver::CgroupsVersion::V1}; + return {*v1_path, CgroupsVersion::V1}; throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Cannot find cgroups v1 or v2 current memory file"); } @@ -173,22 +172,29 @@ std::pair getCgroupsPat namespace DB { -CgroupsMemoryUsageObserver::CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_) - : log(getLogger("CgroupsMemoryUsageObserver")), wait_time(wait_time_) +std::shared_ptr createCgroupsReader() { const auto [cgroup_path, version] = getCgroupsPath(); + LOG_INFO( + getLogger("CgroupsReader"), + "Will create cgroup reader from '{}' (cgroups version: {})", + cgroup_path, + (version == CgroupsVersion::V1) ? "v1" : "v2"); if (version == CgroupsVersion::V2) - cgroup_reader = std::make_unique(cgroup_path); + return std::make_shared(cgroup_path); else - cgroup_reader = std::make_unique(cgroup_path); + { + chassert(version == CgroupsVersion::V1); + return std::make_shared(cgroup_path); + } - LOG_INFO( - log, - "Will read the current memory usage from '{}' (cgroups version: {}), wait time is {} sec", - cgroup_path, - (version == CgroupsVersion::V1) ? "v1" : "v2", - wait_time.count()); +} + +CgroupsMemoryUsageObserver::CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_, std::shared_ptr cgroups_reader_) + : log(getLogger("CgroupsMemoryUsageObserver")), wait_time(wait_time_), cgroups_reader(std::move(cgroups_reader_)) +{ + cgroups_reader = createCgroupsReader(); } CgroupsMemoryUsageObserver::~CgroupsMemoryUsageObserver() @@ -196,58 +202,6 @@ CgroupsMemoryUsageObserver::~CgroupsMemoryUsageObserver() stopThread(); } -void CgroupsMemoryUsageObserver::setMemoryUsageLimits(uint64_t hard_limit_, uint64_t soft_limit_) -{ - std::lock_guard limit_lock(limit_mutex); - - if (hard_limit_ == hard_limit && soft_limit_ == soft_limit) - return; - - hard_limit = hard_limit_; - soft_limit = soft_limit_; - - on_hard_limit = [this, hard_limit_](bool up) - { - if (up) - { - LOG_WARNING(log, "Exceeded hard memory limit ({})", ReadableSize(hard_limit_)); - - /// Update current usage in memory tracker. Also reset free_memory_in_allocator_arenas to zero though we don't know if they are - /// really zero. Trying to avoid OOM ... - MemoryTracker::setRSS(hard_limit_, 0); - } - else - { - LOG_INFO(log, "Dropped below hard memory limit ({})", ReadableSize(hard_limit_)); - } - }; - - on_soft_limit = [this, soft_limit_](bool up) - { - if (up) - { - LOG_WARNING(log, "Exceeded soft memory limit ({})", ReadableSize(soft_limit_)); - -# if USE_JEMALLOC - LOG_INFO(log, "Purging jemalloc arenas"); - mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0); -# endif - /// Reset current usage in memory tracker. Expect zero for free_memory_in_allocator_arenas as we just purged them. - uint64_t memory_usage = cgroup_reader->readMemoryUsage(); - LOG_TRACE(log, "Read current memory usage {} bytes ({}) from cgroups", memory_usage, ReadableSize(memory_usage)); - MemoryTracker::setRSS(memory_usage, 0); - - LOG_INFO(log, "Purged jemalloc arenas. Current memory usage is {}", ReadableSize(memory_usage)); - } - else - { - LOG_INFO(log, "Dropped below soft memory limit ({})", ReadableSize(soft_limit_)); - } - }; - - LOG_INFO(log, "Set new limits, soft limit: {}, hard limit: {}", ReadableSize(soft_limit_), ReadableSize(hard_limit_)); -} - void CgroupsMemoryUsageObserver::setOnMemoryAmountAvailableChangedFn(OnMemoryAmountAvailableChangedFn on_memory_amount_available_changed_) { std::lock_guard memory_amount_available_changed_lock(memory_amount_available_changed_mutex); @@ -301,35 +255,6 @@ void CgroupsMemoryUsageObserver::runThread() std::lock_guard memory_amount_available_changed_lock(memory_amount_available_changed_mutex); on_memory_amount_available_changed(); } - - std::lock_guard limit_lock(limit_mutex); - if (soft_limit > 0 && hard_limit > 0) - { - uint64_t memory_usage = cgroup_reader->readMemoryUsage(); - LOG_TRACE(log, "Read current memory usage {} bytes ({}) from cgroups", memory_usage, ReadableSize(memory_usage)); - if (memory_usage > hard_limit) - { - if (last_memory_usage <= hard_limit) - on_hard_limit(true); - } - else - { - if (last_memory_usage > hard_limit) - on_hard_limit(false); - } - - if (memory_usage > soft_limit) - { - if (last_memory_usage <= soft_limit) - on_soft_limit(true); - } - else - { - if (last_memory_usage > soft_limit) - on_soft_limit(false); - } - last_memory_usage = memory_usage; - } } catch (...) { diff --git a/src/Common/CgroupsMemoryUsageObserver.h b/src/Common/CgroupsMemoryUsageObserver.h index b848a2bff3c..078307a6fa0 100644 --- a/src/Common/CgroupsMemoryUsageObserver.h +++ b/src/Common/CgroupsMemoryUsageObserver.h @@ -16,6 +16,8 @@ struct ICgroupsReader virtual uint64_t readMemoryUsage() = 0; }; +std::shared_ptr createCgroupsReader(); + /// Does two things: /// 1. Periodically reads the memory usage of the process from Linux cgroups. /// You can specify soft or hard memory limits: @@ -35,19 +37,11 @@ struct ICgroupsReader class CgroupsMemoryUsageObserver { public: - using OnMemoryLimitFn = std::function; using OnMemoryAmountAvailableChangedFn = std::function; - enum class CgroupsVersion : uint8_t - { - V1, - V2 - }; - - explicit CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_); + explicit CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_, std::shared_ptr cgroups_reader_); ~CgroupsMemoryUsageObserver(); - void setMemoryUsageLimits(uint64_t hard_limit_, uint64_t soft_limit_); void setOnMemoryAmountAvailableChangedFn(OnMemoryAmountAvailableChangedFn on_memory_amount_available_changed_); void startThread(); @@ -58,22 +52,17 @@ private: const std::chrono::seconds wait_time; std::mutex limit_mutex; - size_t hard_limit TSA_GUARDED_BY(limit_mutex) = 0; - size_t soft_limit TSA_GUARDED_BY(limit_mutex) = 0; - OnMemoryLimitFn on_hard_limit TSA_GUARDED_BY(limit_mutex); - OnMemoryLimitFn on_soft_limit TSA_GUARDED_BY(limit_mutex); std::mutex memory_amount_available_changed_mutex; OnMemoryAmountAvailableChangedFn on_memory_amount_available_changed TSA_GUARDED_BY(memory_amount_available_changed_mutex); - uint64_t last_memory_usage = 0; /// how much memory does the process use uint64_t last_available_memory_amount; /// how much memory can the process use void stopThread(); void runThread(); - std::unique_ptr cgroup_reader; + std::shared_ptr cgroups_reader; std::mutex thread_mutex; std::condition_variable cond; diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 28cfa98666a..eaf34d87ec5 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -26,7 +26,6 @@ #endif #include -#include #include #include #include @@ -123,7 +122,7 @@ static constexpr size_t log_peak_memory_usage_every = 1ULL << 30; MemoryTracker total_memory_tracker(nullptr, VariableContext::Global); MemoryTracker background_memory_tracker(&total_memory_tracker, VariableContext::User, false); -std::atomic MemoryTracker::free_memory_in_allocator_arenas; +std::atomic MemoryTracker::has_free_memory_in_allocator_arenas; MemoryTracker::MemoryTracker(VariableContext level_) : parent(&total_memory_tracker), level(level_) {} MemoryTracker::MemoryTracker(MemoryTracker * parent_, VariableContext level_) : parent(parent_), level(level_) {} @@ -204,7 +203,7 @@ void MemoryTracker::debugLogBigAllocationWithoutCheck(Int64 size [[maybe_unused] LOG_TEST(getLogger("MemoryTracker"), "Too big allocation ({} bytes) without checking memory limits, " "it may lead to OOM. Stack trace: {}", size, StackTrace().toString()); #else - return; /// Avoid trash logging in release builds + /// Avoid trash logging in release builds #endif } @@ -294,33 +293,18 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed } } - Int64 limit_to_check = current_hard_limit; - #if USE_JEMALLOC - if (level == VariableContext::Global && allow_use_jemalloc_memory.load(std::memory_order_relaxed)) + if (level == VariableContext::Global && will_be > soft_limit.load(std::memory_order_relaxed) + && has_free_memory_in_allocator_arenas.exchange(false)) { - /// Jemalloc arenas may keep some extra memory. - /// This memory was substucted from RSS to decrease memory drift. - /// In case memory is close to limit, try to pugre the arenas. - /// This is needed to avoid OOM, because some allocations are directly done with mmap. - Int64 current_free_memory_in_allocator_arenas = free_memory_in_allocator_arenas.load(std::memory_order_relaxed); - - if (current_free_memory_in_allocator_arenas > 0 && current_hard_limit && current_free_memory_in_allocator_arenas + will_be > current_hard_limit) - { - if (free_memory_in_allocator_arenas.exchange(-current_free_memory_in_allocator_arenas) > 0) - { - Stopwatch watch; - mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0); - ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurge); - ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, watch.elapsedMicroseconds()); - } - } - - limit_to_check += abs(current_free_memory_in_allocator_arenas); + Stopwatch watch; + mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0); + ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurge); + ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, watch.elapsedMicroseconds()); } #endif - if (unlikely(current_hard_limit && will_be > limit_to_check)) + if (unlikely(current_hard_limit && will_be > current_hard_limit)) { if (memoryTrackerCanThrow(level, false) && throw_if_memory_exceeded) { @@ -526,11 +510,11 @@ void MemoryTracker::reset() } -void MemoryTracker::setRSS(Int64 rss_, Int64 free_memory_in_allocator_arenas_) +void MemoryTracker::setRSS(Int64 rss_, bool has_free_memory_in_allocator_arenas_) { Int64 new_amount = rss_; total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed); - free_memory_in_allocator_arenas.store(free_memory_in_allocator_arenas_, std::memory_order_relaxed); + has_free_memory_in_allocator_arenas.store(has_free_memory_in_allocator_arenas_, std::memory_order_relaxed); auto metric_loaded = total_memory_tracker.metric.load(std::memory_order_relaxed); if (metric_loaded != CurrentMetrics::end()) diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index fd32b631774..48d02fd1fc6 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -59,7 +59,7 @@ private: std::atomic profiler_limit {0}; std::atomic_bool allow_use_jemalloc_memory {true}; - static std::atomic free_memory_in_allocator_arenas; + static std::atomic has_free_memory_in_allocator_arenas; Int64 profiler_step = 0; @@ -252,7 +252,7 @@ public: /// Reset current counter to an RSS value. /// Jemalloc may have pre-allocated arenas, they are accounted in RSS. /// We can free this arenas in case of exception to avoid OOM. - static void setRSS(Int64 rss_, Int64 free_memory_in_allocator_arenas_); + static void setRSS(Int64 rss_, bool has_free_memory_in_allocator_arenas_); /// Prints info about peak memory consumption into log. void logPeakMemoryUsage(); diff --git a/src/Coordination/KeeperAsynchronousMetrics.cpp b/src/Coordination/KeeperAsynchronousMetrics.cpp index 86166ffe31b..3e404b7152b 100644 --- a/src/Coordination/KeeperAsynchronousMetrics.cpp +++ b/src/Coordination/KeeperAsynchronousMetrics.cpp @@ -115,7 +115,7 @@ void updateKeeperInformation(KeeperDispatcher & keeper_dispatcher, AsynchronousM KeeperAsynchronousMetrics::KeeperAsynchronousMetrics( ContextPtr context_, unsigned update_period_seconds, const ProtocolServerMetricsFunc & protocol_server_metrics_func_) - : AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_), context(std::move(context_)) + : AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_, context_->getCgroupsReader()), context(std::move(context_)) { } diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 2602afd8b78..771b8e9e558 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -405,6 +406,8 @@ struct ContextSharedPart : boost::noncopyable std::unique_ptr cluster_discovery TSA_GUARDED_BY(clusters_mutex); size_t clusters_version TSA_GUARDED_BY(clusters_mutex) = 0; + std::shared_ptr cgroups_reader; + /// No lock required for async_insert_queue modified only during initialization std::shared_ptr async_insert_queue; @@ -5635,6 +5638,16 @@ const ServerSettings & Context::getServerSettings() const return shared->server_settings; } +void Context::setCgroupsReader(std::shared_ptr cgroups_reader_) +{ + shared->cgroups_reader = std::move(cgroups_reader_); +} + +std::shared_ptr Context::getCgroupsReader() const +{ + return shared->cgroups_reader; +} + uint64_t HTTPContext::getMaxHstsAge() const { return context->getSettingsRef().hsts_max_age; diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 284cac50769..f183a72e8e2 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -150,6 +150,7 @@ class ServerType; template class MergeTreeBackgroundExecutor; class AsyncLoader; +struct ICgroupsReader; struct TemporaryTableHolder; using TemporaryTablesMapping = std::map>; @@ -1344,6 +1345,9 @@ public: const ServerSettings & getServerSettings() const; + void setCgroupsReader(std::shared_ptr cgroups_reader_); + std::shared_ptr getCgroupsReader() const; + private: std::shared_ptr getSettingsConstraintsAndCurrentProfilesWithLock() const; diff --git a/src/Interpreters/ServerAsynchronousMetrics.cpp b/src/Interpreters/ServerAsynchronousMetrics.cpp index 872a9f864df..6ee0168bede 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.cpp +++ b/src/Interpreters/ServerAsynchronousMetrics.cpp @@ -57,7 +57,7 @@ ServerAsynchronousMetrics::ServerAsynchronousMetrics( unsigned heavy_metrics_update_period_seconds, const ProtocolServerMetricsFunc & protocol_server_metrics_func_) : WithContext(global_context_) - , AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_) + , AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_, getContext()->getCgroupsReader()) , heavy_metric_update_period(heavy_metrics_update_period_seconds) { /// sanity check From 21009577d867d493a6ceda48987c060498774f94 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 18 Jul 2024 09:01:08 +0200 Subject: [PATCH 069/215] Dedicated memory background thread --- programs/server/Server.cpp | 39 ++++++++++++++++++++++++--- src/CMakeLists.txt | 2 +- src/Common/AsynchronousMetrics.cpp | 43 ------------------------------ src/Common/Jemalloc.cpp | 16 ----------- src/Common/Jemalloc.h | 29 ++++++++++++++++++++ src/Common/MemoryTracker.cpp | 33 +++++++++-------------- src/Common/MemoryTracker.h | 1 - 7 files changed, 78 insertions(+), 85 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index c52b1e037ec..ca46338d1c1 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -447,9 +447,12 @@ void checkForUsersNotInMainConfig( } } +namespace +{ + /// Unused in other builds #if defined(OS_LINUX) -static String readLine(const String & path) +String readLine(const String & path) { ReadBufferFromFile in(path); String contents; @@ -457,7 +460,7 @@ static String readLine(const String & path) return contents; } -static int readNumber(const String & path) +int readNumber(const String & path) { ReadBufferFromFile in(path); int result; @@ -467,7 +470,7 @@ static int readNumber(const String & path) #endif -static void sanityChecks(Server & server) +void sanityChecks(Server & server) { std::string data_path = getCanonicalPath(server.config().getString("path", DBMS_DEFAULT_PATH)); std::string logs_path = server.config().getString("logger.log", ""); @@ -588,6 +591,31 @@ static void sanityChecks(Server & server) } } +[[noreturn]] void backgroundMemoryThread() +{ + std::mutex mutex; + std::condition_variable cv; + + std::unique_lock lock(mutex); + while (true) + { + cv.wait_for(lock, std::chrono::microseconds(200)); + uint64_t epoch = 0; + mallctl("epoch", nullptr, nullptr, &epoch, sizeof(epoch)); + auto maybe_resident = getJemallocValue("stats.resident"); + if (!maybe_resident.has_value()) + continue; + + Int64 resident = *maybe_resident; + //LOG_INFO(getLogger("JEmalloc"), "Resident {}", ReadableSize(resident)); + MemoryTracker::setRSS(resident, false); + if (resident > total_memory_tracker.getHardLimit()) + purgeJemallocArenas(); + } +} + +} + void loadStartupScripts(const Poco::Util::AbstractConfiguration & config, ContextMutablePtr context, Poco::Logger * log) { try @@ -877,6 +905,11 @@ try total_memory_tracker.setSampleMaxAllocationSize(server_settings.total_memory_profiler_sample_max_allocation_size); } + ThreadFromGlobalPool background_memory_thread([] + { + backgroundMemoryThread(); + }); + Poco::ThreadPool server_pool( /* minCapacity */3, /* maxCapacity */server_settings.max_connections, diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d985595154c..bfa41eacea1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -174,7 +174,7 @@ add_library (clickhouse_new_delete STATIC Common/new_delete.cpp) target_link_libraries (clickhouse_new_delete PRIVATE clickhouse_common_io) if (TARGET ch_contrib::jemalloc) target_link_libraries (clickhouse_new_delete PRIVATE ch_contrib::jemalloc) - target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::jemalloc) + target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::jemalloc) target_link_libraries (clickhouse_storages_system PRIVATE ch_contrib::jemalloc) endif() diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index 0953ad88697..b3e53c29d4a 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -649,49 +649,6 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) "The amount of virtual memory mapped for the use of stack and for the allocated memory, in bytes." " It is unspecified whether it includes the per-thread stacks and most of the allocated memory, that is allocated with the 'mmap' system call." " This metric exists only for completeness reasons. I recommend to use the `MemoryResident` metric for monitoring."}; - - /// We must update the value of total_memory_tracker periodically. - /// Otherwise it might be calculated incorrectly - it can include a "drift" of memory amount. - /// See https://github.com/ClickHouse/ClickHouse/issues/10293 - { - Int64 amount = total_memory_tracker.get(); - Int64 peak = total_memory_tracker.getPeak(); - Int64 rss = data.resident; - Int64 free_memory_in_allocator_arenas = 0; - -#if USE_JEMALLOC - /// According to jemalloc man, pdirty is: - /// - /// Number of pages within unused extents that are potentially - /// dirty, and for which madvise() or similar has not been called. - /// - /// So they will be subtracted from RSS to make accounting more - /// accurate, since those pages are not really RSS but a memory - /// that can be used at anytime via jemalloc. - free_memory_in_allocator_arenas = je_malloc_pdirty * getPageSize(); -#endif - - if (cgroups_reader != nullptr) - { - rss = cgroups_reader->readMemoryUsage(); - new_values["CgroupsMemoryUsage"] = { rss, - "The amount of physical memory used by the server process, reported by cgroups." }; - } - - Int64 difference = rss - amount; - - /// Log only if difference is high. This is for convenience. The threshold is arbitrary. - if (difference >= 1048576 || difference <= -1048576) - LOG_TRACE(log, - "MemoryTracking: was {}, peak {}, free memory in arenas {}, will set to {} (RSS), difference: {}", - ReadableSize(amount), - ReadableSize(peak), - ReadableSize(free_memory_in_allocator_arenas), - ReadableSize(rss), - ReadableSize(difference)); - - MemoryTracker::setRSS(rss, /*has_free_memory_in_allocator_arenas_=*/free_memory_in_allocator_arenas > 0); - } } { diff --git a/src/Common/Jemalloc.cpp b/src/Common/Jemalloc.cpp index d7cc246db6a..d8ff9268cca 100644 --- a/src/Common/Jemalloc.cpp +++ b/src/Common/Jemalloc.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #define STRINGIFY_HELPER(x) #x #define STRINGIFY(x) STRINGIFY_HELPER(x) @@ -26,7 +25,6 @@ namespace ErrorCodes void purgeJemallocArenas() { - LOG_TRACE(getLogger("SystemJemalloc"), "Purging unused memory"); Stopwatch watch; mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0); ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurge); @@ -46,20 +44,6 @@ void checkJemallocProfilingEnabled() "set: MALLOC_CONF=background_thread:true,prof:true"); } -template -void setJemallocValue(const char * name, T value) -{ - T old_value; - size_t old_value_size = sizeof(T); - if (mallctl(name, &old_value, &old_value_size, reinterpret_cast(&value), sizeof(T))) - { - LOG_WARNING(getLogger("Jemalloc"), "mallctl for {} failed", name); - return; - } - - LOG_INFO(getLogger("Jemalloc"), "Value for {} set to {} (from {})", name, value, old_value); -} - void setJemallocProfileActive(bool value) { checkJemallocProfilingEnabled(); diff --git a/src/Common/Jemalloc.h b/src/Common/Jemalloc.h index 499a906fd3d..0c533711f78 100644 --- a/src/Common/Jemalloc.h +++ b/src/Common/Jemalloc.h @@ -5,6 +5,8 @@ #if USE_JEMALLOC #include +#include +#include namespace DB { @@ -21,6 +23,33 @@ void setJemallocBackgroundThreads(bool enabled); void setJemallocMaxBackgroundThreads(size_t max_threads); +template +void setJemallocValue(const char * name, T value) +{ + T old_value; + size_t old_value_size = sizeof(T); + if (mallctl(name, &old_value, &old_value_size, reinterpret_cast(&value), sizeof(T))) + { + LOG_WARNING(getLogger("Jemalloc"), "mallctl for {} failed", name); + return; + } + + LOG_INFO(getLogger("Jemalloc"), "Value for {} set to {} (from {})", name, value, old_value); +} + +template +std::optional getJemallocValue(const char * name) +{ + T value; + size_t value_size = sizeof(T); + if (mallctl(name, &value, &value_size, nullptr, 0)) + { + LOG_WARNING(getLogger("Jemalloc"), "mallctl for {} failed", name); + return std::nullopt; + } + return value; +} + } #endif diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index eaf34d87ec5..a541eeeb25f 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -20,9 +20,6 @@ #if USE_JEMALLOC # include -#define STRINGIFY_HELPER(x) #x -#define STRINGIFY(x) STRINGIFY_HELPER(x) - #endif #include @@ -126,11 +123,11 @@ std::atomic MemoryTracker::has_free_memory_in_allocator_arenas; MemoryTracker::MemoryTracker(VariableContext level_) : parent(&total_memory_tracker), level(level_) {} MemoryTracker::MemoryTracker(MemoryTracker * parent_, VariableContext level_) : parent(parent_), level(level_) {} + MemoryTracker::MemoryTracker(MemoryTracker * parent_, VariableContext level_, bool log_peak_memory_usage_in_destructor_) - : parent(parent_) - , log_peak_memory_usage_in_destructor(log_peak_memory_usage_in_destructor_) - , level(level_) -{} + : parent(parent_), log_peak_memory_usage_in_destructor(log_peak_memory_usage_in_destructor_), level(level_) +{ +} MemoryTracker::~MemoryTracker() { @@ -200,8 +197,12 @@ void MemoryTracker::debugLogBigAllocationWithoutCheck(Int64 size [[maybe_unused] return; MemoryTrackerBlockerInThread blocker(VariableContext::Global); - LOG_TEST(getLogger("MemoryTracker"), "Too big allocation ({} bytes) without checking memory limits, " - "it may lead to OOM. Stack trace: {}", size, StackTrace().toString()); + LOG_TEST( + getLogger("MemoryTracker"), + "Too big allocation ({} bytes) without checking memory limits, " + "it may lead to OOM. Stack trace: {}", + size, + StackTrace().toString()); #else /// Avoid trash logging in release builds #endif @@ -293,17 +294,6 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed } } -#if USE_JEMALLOC - if (level == VariableContext::Global && will_be > soft_limit.load(std::memory_order_relaxed) - && has_free_memory_in_allocator_arenas.exchange(false)) - { - Stopwatch watch; - mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0); - ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurge); - ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, watch.elapsedMicroseconds()); - } -#endif - if (unlikely(current_hard_limit && will_be > current_hard_limit)) { if (memoryTrackerCanThrow(level, false) && throw_if_memory_exceeded) @@ -513,7 +503,8 @@ void MemoryTracker::reset() void MemoryTracker::setRSS(Int64 rss_, bool has_free_memory_in_allocator_arenas_) { Int64 new_amount = rss_; - total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed); + if (rss_) + total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed); has_free_memory_in_allocator_arenas.store(has_free_memory_in_allocator_arenas_, std::memory_order_relaxed); auto metric_loaded = total_memory_tracker.metric.load(std::memory_order_relaxed); diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index 48d02fd1fc6..257ed7d0629 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -2,7 +2,6 @@ #include #include -#include #include #include #include From 9a43183eb39dc056186432e6478e050db5045ecf Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 18 Jul 2024 10:31:24 +0200 Subject: [PATCH 070/215] Finish background memory thread --- programs/keeper/Keeper.cpp | 30 ++++------ programs/server/Server.cpp | 59 ++++--------------- src/Common/AsynchronousMetrics.cpp | 19 ++---- src/Common/AsynchronousMetrics.h | 3 +- src/Common/CgroupsMemoryUsageObserver.cpp | 8 +-- src/Common/CgroupsMemoryUsageObserver.h | 24 +++----- src/Common/Jemalloc.h | 42 +++++++++---- src/Common/MemoryTracker.cpp | 10 +--- src/Common/MemoryTracker.h | 13 +--- src/Common/MemoryWorker.cpp | 49 +++++++++++++++ src/Common/MemoryWorker.h | 34 +++++++++++ .../KeeperAsynchronousMetrics.cpp | 2 +- src/Core/ServerSettings.h | 1 + src/Interpreters/Context.cpp | 12 ---- src/Interpreters/Context.h | 3 - .../ServerAsynchronousMetrics.cpp | 2 +- .../System/StorageSystemServerSettings.cpp | 1 - tests/integration/test_memory_limit/test.py | 1 - 18 files changed, 158 insertions(+), 155 deletions(-) create mode 100644 src/Common/MemoryWorker.cpp create mode 100644 src/Common/MemoryWorker.h diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index 3f6020ad48c..87f126a0046 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -371,6 +372,8 @@ try LOG_INFO(log, "Background threads finished in {} ms", watch.elapsedMilliseconds()); }); + MemoryWorker memory_worker(config().getUInt64("memory_worker_period_ms", 100)); + static ServerErrorHandler error_handler; Poco::ErrorHandler::set(&error_handler); @@ -399,18 +402,6 @@ try registerDisks(/*global_skip_access_check=*/false); - auto cgroups_memory_observer_wait_time = config().getUInt64("keeper_server.cgroups_memory_observer_wait_time", 15); - try - { - auto cgroups_reader = createCgroupsReader(); - global_context->setCgroupsReader(createCgroupsReader()); - } - catch (...) - { - if (cgroups_memory_observer_wait_time != 0) - tryLogCurrentException(log, "Failed to create cgroups reader"); - } - /// This object will periodically calculate some metrics. KeeperAsynchronousMetrics async_metrics( global_context, @@ -634,21 +625,22 @@ try main_config_reloader->start(); std::optional cgroups_memory_usage_observer; - if (cgroups_memory_observer_wait_time != 0) + try { - auto cgroups_reader = global_context->getCgroupsReader(); - if (cgroups_reader) + auto wait_time = config().getUInt64("keeper_server.cgroups_memory_observer_wait_time", 15); + if (wait_time != 0) { - cgroups_memory_usage_observer.emplace(std::chrono::seconds(cgroups_memory_observer_wait_time), global_context->getCgroupsReader()); + cgroups_memory_usage_observer.emplace(std::chrono::seconds(wait_time)); /// Not calling cgroups_memory_usage_observer->setLimits() here (as for the normal ClickHouse server) because Keeper controls /// its memory usage by other means (via setting 'max_memory_usage_soft_limit'). cgroups_memory_usage_observer->setOnMemoryAmountAvailableChangedFn([&]() { main_config_reloader->reload(); }); cgroups_memory_usage_observer->startThread(); } - else - LOG_ERROR(log, "Disabling cgroup memory observer because of an error during initialization of cgroups reader"); } - + catch (Exception &) + { + tryLogCurrentException(log, "Disabling cgroup memory observer because of an error during initialization"); + } LOG_INFO(log, "Ready for connections."); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index ca46338d1c1..ae445477c3a 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -24,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -110,6 +110,8 @@ #include #include +#include + #include "config.h" #include @@ -591,29 +593,6 @@ void sanityChecks(Server & server) } } -[[noreturn]] void backgroundMemoryThread() -{ - std::mutex mutex; - std::condition_variable cv; - - std::unique_lock lock(mutex); - while (true) - { - cv.wait_for(lock, std::chrono::microseconds(200)); - uint64_t epoch = 0; - mallctl("epoch", nullptr, nullptr, &epoch, sizeof(epoch)); - auto maybe_resident = getJemallocValue("stats.resident"); - if (!maybe_resident.has_value()) - continue; - - Int64 resident = *maybe_resident; - //LOG_INFO(getLogger("JEmalloc"), "Resident {}", ReadableSize(resident)); - MemoryTracker::setRSS(resident, false); - if (resident > total_memory_tracker.getHardLimit()) - purgeJemallocArenas(); - } -} - } void loadStartupScripts(const Poco::Util::AbstractConfiguration & config, ContextMutablePtr context, Poco::Logger * log) @@ -905,11 +884,6 @@ try total_memory_tracker.setSampleMaxAllocationSize(server_settings.total_memory_profiler_sample_max_allocation_size); } - ThreadFromGlobalPool background_memory_thread([] - { - backgroundMemoryThread(); - }); - Poco::ThreadPool server_pool( /* minCapacity */3, /* maxCapacity */server_settings.max_connections, @@ -930,16 +904,7 @@ try LOG_INFO(log, "Background threads finished in {} ms", watch.elapsedMilliseconds()); }); - try - { - auto cgroups_reader = createCgroupsReader(); - global_context->setCgroupsReader(createCgroupsReader()); - } - catch (...) - { - if (server_settings.cgroups_memory_usage_observer_wait_time != 0) - tryLogCurrentException(log, "Failed to create cgroups reader"); - } + MemoryWorker memory_worker(global_context->getServerSettings().memory_worker_period_ms); /// This object will periodically calculate some metrics. ServerAsynchronousMetrics async_metrics( @@ -1500,13 +1465,15 @@ try } std::optional cgroups_memory_usage_observer; - if (auto wait_time = server_settings.cgroups_memory_usage_observer_wait_time; wait_time != 0) + try { - auto cgroups_reader = global_context->getCgroupsReader(); - if (cgroups_reader) - cgroups_memory_usage_observer.emplace(std::chrono::seconds(wait_time), std::move(cgroups_reader)); - else - LOG_ERROR(log, "Disabling cgroup memory observer because of an error during initialization of cgroups reader"); + auto wait_time = server_settings.cgroups_memory_usage_observer_wait_time; + if (wait_time != 0) + cgroups_memory_usage_observer.emplace(std::chrono::seconds(wait_time)); + } + catch (Exception &) + { + tryLogCurrentException(log, "Disabling cgroup memory observer because of an error during initialization"); } std::string cert_path = config().getString("openSSL.server.certificateFile", ""); @@ -1602,8 +1569,6 @@ try background_memory_tracker.setDescription("(background)"); background_memory_tracker.setMetric(CurrentMetrics::MergesMutationsMemoryTracking); - total_memory_tracker.setAllowUseJemallocMemory(new_server_settings.allow_use_jemalloc_memory); - auto * global_overcommit_tracker = global_context->getGlobalOvercommitTracker(); total_memory_tracker.setOvercommitTracker(global_overcommit_tracker); diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index b3e53c29d4a..a5c9875188b 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -57,12 +58,10 @@ static std::unique_ptr openFileIfExists(const std::stri AsynchronousMetrics::AsynchronousMetrics( unsigned update_period_seconds, - const ProtocolServerMetricsFunc & protocol_server_metrics_func_, - std::shared_ptr cgroups_reader_) + const ProtocolServerMetricsFunc & protocol_server_metrics_func_) : update_period(update_period_seconds) , log(getLogger("AsynchronousMetrics")) , protocol_server_metrics_func(protocol_server_metrics_func_) - , cgroups_reader(std::move(cgroups_reader_)) { #if defined(OS_LINUX) openFileIfExists("/proc/meminfo", meminfo); @@ -378,23 +377,13 @@ void AsynchronousMetrics::run() namespace { -uint64_t updateJemallocEpoch() -{ - uint64_t value = 0; - size_t size = sizeof(value); - mallctl("epoch", &value, &size, &value, size); - return value; -} - template Value saveJemallocMetricImpl( AsynchronousMetricValues & values, const std::string & jemalloc_full_name, const std::string & clickhouse_full_name) { - Value value{}; - size_t size = sizeof(value); - mallctl(jemalloc_full_name.c_str(), &value, &size, nullptr, 0); + auto value = getJemallocValue(jemalloc_full_name.c_str()); values[clickhouse_full_name] = AsynchronousMetricValue(value, "An internal metric of the low-level memory allocator (jemalloc). See https://jemalloc.net/jemalloc.3.html"); return value; } @@ -604,7 +593,7 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) // 'epoch' is a special mallctl -- it updates the statistics. Without it, all // the following calls will return stale values. It increments and returns // the current epoch number, which might be useful to log as a sanity check. - auto epoch = updateJemallocEpoch(); + auto epoch = getJemallocValue("epoch"); new_values["jemalloc.epoch"] = { epoch, "An internal incremental update number of the statistics of jemalloc (Jason Evans' memory allocator), used in all other `jemalloc` metrics." }; // Collect the statistics themselves. diff --git a/src/Common/AsynchronousMetrics.h b/src/Common/AsynchronousMetrics.h index 0b110f41fc3..bc379d4e92b 100644 --- a/src/Common/AsynchronousMetrics.h +++ b/src/Common/AsynchronousMetrics.h @@ -69,8 +69,7 @@ public: AsynchronousMetrics( unsigned update_period_seconds, - const ProtocolServerMetricsFunc & protocol_server_metrics_func_, - std::shared_ptr cgroups_reader_); + const ProtocolServerMetricsFunc & protocol_server_metrics_func_); virtual ~AsynchronousMetrics(); diff --git a/src/Common/CgroupsMemoryUsageObserver.cpp b/src/Common/CgroupsMemoryUsageObserver.cpp index b12845df098..ab7ca69ca04 100644 --- a/src/Common/CgroupsMemoryUsageObserver.cpp +++ b/src/Common/CgroupsMemoryUsageObserver.cpp @@ -191,11 +191,9 @@ std::shared_ptr createCgroupsReader() } -CgroupsMemoryUsageObserver::CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_, std::shared_ptr cgroups_reader_) - : log(getLogger("CgroupsMemoryUsageObserver")), wait_time(wait_time_), cgroups_reader(std::move(cgroups_reader_)) -{ - cgroups_reader = createCgroupsReader(); -} +CgroupsMemoryUsageObserver::CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_) + : log(getLogger("CgroupsMemoryUsageObserver")), wait_time(wait_time_), cgroups_reader(createCgroupsReader()) +{} CgroupsMemoryUsageObserver::~CgroupsMemoryUsageObserver() { diff --git a/src/Common/CgroupsMemoryUsageObserver.h b/src/Common/CgroupsMemoryUsageObserver.h index 078307a6fa0..33e0f167a59 100644 --- a/src/Common/CgroupsMemoryUsageObserver.h +++ b/src/Common/CgroupsMemoryUsageObserver.h @@ -18,28 +18,20 @@ struct ICgroupsReader std::shared_ptr createCgroupsReader(); -/// Does two things: -/// 1. Periodically reads the memory usage of the process from Linux cgroups. -/// You can specify soft or hard memory limits: -/// - When the soft memory limit is hit, drop jemalloc cache. -/// - When the hard memory limit is hit, update MemoryTracking metric to throw memory exceptions faster. -/// The goal of this is to avoid that the process hits the maximum allowed memory limit at which there is a good -/// chance that the Limux OOM killer terminates it. All of this is done is because internal memory tracking in -/// ClickHouse can unfortunately under-estimate the actually used memory. -/// 2. Periodically reads the the maximum memory available to the process (which can change due to cgroups settings). -/// You can specify a callback to react on changes. The callback typically reloads the configuration, i.e. Server -/// or Keeper configuration file. This reloads settings 'max_server_memory_usage' (Server) and 'max_memory_usage_soft_limit' -/// (Keeper) from which various other internal limits are calculated, including the soft and hard limits for (1.). -/// The goal of this is to provide elasticity when the container is scaled-up/scaled-down. The mechanism (polling -/// cgroups) is quite implicit, unfortunately there is currently no better way to communicate memory threshold changes -/// to the database. +/// Periodically reads the the maximum memory available to the process (which can change due to cgroups settings). +/// You can specify a callback to react on changes. The callback typically reloads the configuration, i.e. Server +/// or Keeper configuration file. This reloads settings 'max_server_memory_usage' (Server) and 'max_memory_usage_soft_limit' +/// (Keeper) from which various other internal limits are calculated, including the soft and hard limits for (1.). +/// The goal of this is to provide elasticity when the container is scaled-up/scaled-down. The mechanism (polling +/// cgroups) is quite implicit, unfortunately there is currently no better way to communicate memory threshold changes +/// to the database. #if defined(OS_LINUX) class CgroupsMemoryUsageObserver { public: using OnMemoryAmountAvailableChangedFn = std::function; - explicit CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_, std::shared_ptr cgroups_reader_); + explicit CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_); ~CgroupsMemoryUsageObserver(); void setOnMemoryAmountAvailableChangedFn(OnMemoryAmountAvailableChangedFn on_memory_amount_available_changed_); diff --git a/src/Common/Jemalloc.h b/src/Common/Jemalloc.h index 0c533711f78..629d039b483 100644 --- a/src/Common/Jemalloc.h +++ b/src/Common/Jemalloc.h @@ -28,28 +28,46 @@ void setJemallocValue(const char * name, T value) { T old_value; size_t old_value_size = sizeof(T); - if (mallctl(name, &old_value, &old_value_size, reinterpret_cast(&value), sizeof(T))) - { - LOG_WARNING(getLogger("Jemalloc"), "mallctl for {} failed", name); - return; - } - + mallctl(name, &old_value, &old_value_size, reinterpret_cast(&value), sizeof(T)); LOG_INFO(getLogger("Jemalloc"), "Value for {} set to {} (from {})", name, value, old_value); } template -std::optional getJemallocValue(const char * name) +T getJemallocValue(const char * name) { T value; size_t value_size = sizeof(T); - if (mallctl(name, &value, &value_size, nullptr, 0)) - { - LOG_WARNING(getLogger("Jemalloc"), "mallctl for {} failed", name); - return std::nullopt; - } + mallctl(name, &value, &value_size, nullptr, 0); return value; } +template +struct JemallocMibCache +{ + explicit JemallocMibCache(const char * name) + { + mallctlnametomib(name, mib, &mib_length); + } + + void setValue(T value) + { + mallctlbymib(mib, mib_length, nullptr, nullptr, reinterpret_cast(&value), sizeof(T)); + } + + T getValue() + { + T value; + size_t value_size = sizeof(T); + mallctlbymib(mib, mib_length, &value, &value_size, nullptr, 0); + return value; + } + +private: + static constexpr size_t max_mib_length = 4; + size_t mib[max_mib_length]; + size_t mib_length = max_mib_length; +}; + } #endif diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index a541eeeb25f..e237c3a0d33 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -108,8 +108,6 @@ void AllocationTrace::onFreeImpl(void * ptr, size_t size) const namespace ProfileEvents { extern const Event QueryMemoryLimitExceeded; - extern const Event MemoryAllocatorPurge; - extern const Event MemoryAllocatorPurgeTimeMicroseconds; } using namespace std::chrono_literals; @@ -119,8 +117,6 @@ static constexpr size_t log_peak_memory_usage_every = 1ULL << 30; MemoryTracker total_memory_tracker(nullptr, VariableContext::Global); MemoryTracker background_memory_tracker(&total_memory_tracker, VariableContext::User, false); -std::atomic MemoryTracker::has_free_memory_in_allocator_arenas; - MemoryTracker::MemoryTracker(VariableContext level_) : parent(&total_memory_tracker), level(level_) {} MemoryTracker::MemoryTracker(MemoryTracker * parent_, VariableContext level_) : parent(parent_), level(level_) {} @@ -500,12 +496,10 @@ void MemoryTracker::reset() } -void MemoryTracker::setRSS(Int64 rss_, bool has_free_memory_in_allocator_arenas_) +void MemoryTracker::setRSS(Int64 rss_) { Int64 new_amount = rss_; - if (rss_) - total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed); - has_free_memory_in_allocator_arenas.store(has_free_memory_in_allocator_arenas_, std::memory_order_relaxed); + total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed); auto metric_loaded = total_memory_tracker.metric.load(std::memory_order_relaxed); if (metric_loaded != CurrentMetrics::end()) diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index 257ed7d0629..4085bb321ed 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -56,9 +56,6 @@ private: std::atomic soft_limit {0}; std::atomic hard_limit {0}; std::atomic profiler_limit {0}; - std::atomic_bool allow_use_jemalloc_memory {true}; - - static std::atomic has_free_memory_in_allocator_arenas; Int64 profiler_step = 0; @@ -153,14 +150,6 @@ public: { return soft_limit.load(std::memory_order_relaxed); } - void setAllowUseJemallocMemory(bool value) - { - allow_use_jemalloc_memory.store(value, std::memory_order_relaxed); - } - bool getAllowUseJemallocMmemory() const - { - return allow_use_jemalloc_memory.load(std::memory_order_relaxed); - } /** Set limit if it was not set. * Otherwise, set limit to new value, if new value is greater than previous limit. @@ -251,7 +240,7 @@ public: /// Reset current counter to an RSS value. /// Jemalloc may have pre-allocated arenas, they are accounted in RSS. /// We can free this arenas in case of exception to avoid OOM. - static void setRSS(Int64 rss_, bool has_free_memory_in_allocator_arenas_); + static void setRSS(Int64 rss_); /// Prints info about peak memory consumption into log. void logPeakMemoryUsage(); diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp new file mode 100644 index 00000000000..dce47b83667 --- /dev/null +++ b/src/Common/MemoryWorker.cpp @@ -0,0 +1,49 @@ +#include "Common/ThreadPool.h" +#include + +#include +#include + +namespace DB +{ + +#if USE_JEMALLOC +MemoryWorker::MemoryWorker(uint64_t period_ms_) + : period_ms(period_ms_) +{ + background_thread = ThreadFromGlobalPool([this] { backgroundThread(); }); +} + +MemoryWorker::~MemoryWorker() +{ + { + std::unique_lock lock(mutex); + shutdown = true; + } + cv.notify_all(); + + if (background_thread.joinable()) + background_thread.join(); +} + +void MemoryWorker::backgroundThread() +{ + JemallocMibCache epoch_mib("epoch"); + JemallocMibCache resident_mib("stats.resident"); + std::unique_lock lock(mutex); + while (true) + { + cv.wait_for(lock, period_ms, [this] { return shutdown; }); + if (shutdown) + return; + + epoch_mib.setValue(0); + Int64 resident = resident_mib.getValue(); + MemoryTracker::setRSS(resident); + if (resident > total_memory_tracker.getHardLimit()) + purgeJemallocArenas(); + } +} +#endif + +} diff --git a/src/Common/MemoryWorker.h b/src/Common/MemoryWorker.h new file mode 100644 index 00000000000..8048194d9cd --- /dev/null +++ b/src/Common/MemoryWorker.h @@ -0,0 +1,34 @@ +#pragma once + +#include + +#include "config.h" + +namespace DB +{ + +#if USE_JEMALLOC +class MemoryWorker +{ +public: + explicit MemoryWorker(uint64_t period_ms_); + + ~MemoryWorker(); +private: + void backgroundThread(); + + ThreadFromGlobalPool background_thread; + + std::mutex mutex; + std::condition_variable cv; + bool shutdown = false; + + std::chrono::milliseconds period_ms; +}; +#else +class MemoryWorker +{ +}; +#endif + +} diff --git a/src/Coordination/KeeperAsynchronousMetrics.cpp b/src/Coordination/KeeperAsynchronousMetrics.cpp index 3e404b7152b..86166ffe31b 100644 --- a/src/Coordination/KeeperAsynchronousMetrics.cpp +++ b/src/Coordination/KeeperAsynchronousMetrics.cpp @@ -115,7 +115,7 @@ void updateKeeperInformation(KeeperDispatcher & keeper_dispatcher, AsynchronousM KeeperAsynchronousMetrics::KeeperAsynchronousMetrics( ContextPtr context_, unsigned update_period_seconds, const ProtocolServerMetricsFunc & protocol_server_metrics_func_) - : AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_, context_->getCgroupsReader()), context(std::move(context_)) + : AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_), context(std::move(context_)) { } diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index 28b32a6e6a5..aaea0388239 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -157,6 +157,7 @@ namespace DB M(Bool, prepare_system_log_tables_on_startup, false, "If true, ClickHouse creates all configured `system.*_log` tables before the startup. It can be helpful if some startup scripts depend on these tables.", 0) \ M(Double, gwp_asan_force_sample_probability, 0.0003, "Probability that an allocation from specific places will be sampled by GWP Asan (i.e. PODArray allocations)", 0) \ M(UInt64, config_reload_interval_ms, 2000, "How often clickhouse will reload config and check for new changes", 0) \ + M(UInt64, memory_worker_period_ms, 100, "Period of background memory worker which corrects memory tracker memory usages and cleans up unused pages during higher memory usage.", 0) \ /// If you add a setting which can be updated at runtime, please update 'changeable_settings' map in StorageSystemServerSettings.cpp diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 6a24e049998..f70ccfd77be 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -406,8 +406,6 @@ struct ContextSharedPart : boost::noncopyable std::unique_ptr cluster_discovery TSA_GUARDED_BY(clusters_mutex); size_t clusters_version TSA_GUARDED_BY(clusters_mutex) = 0; - std::shared_ptr cgroups_reader; - /// No lock required for async_insert_queue modified only during initialization std::shared_ptr async_insert_queue; @@ -5631,16 +5629,6 @@ const ServerSettings & Context::getServerSettings() const return shared->server_settings; } -void Context::setCgroupsReader(std::shared_ptr cgroups_reader_) -{ - shared->cgroups_reader = std::move(cgroups_reader_); -} - -std::shared_ptr Context::getCgroupsReader() const -{ - return shared->cgroups_reader; -} - uint64_t HTTPContext::getMaxHstsAge() const { return context->getSettingsRef().hsts_max_age; diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 33b742d20ad..0e3c0591c12 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1344,9 +1344,6 @@ public: const ServerSettings & getServerSettings() const; - void setCgroupsReader(std::shared_ptr cgroups_reader_); - std::shared_ptr getCgroupsReader() const; - private: std::shared_ptr getSettingsConstraintsAndCurrentProfilesWithLock() const; diff --git a/src/Interpreters/ServerAsynchronousMetrics.cpp b/src/Interpreters/ServerAsynchronousMetrics.cpp index 6ee0168bede..872a9f864df 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.cpp +++ b/src/Interpreters/ServerAsynchronousMetrics.cpp @@ -57,7 +57,7 @@ ServerAsynchronousMetrics::ServerAsynchronousMetrics( unsigned heavy_metrics_update_period_seconds, const ProtocolServerMetricsFunc & protocol_server_metrics_func_) : WithContext(global_context_) - , AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_, getContext()->getCgroupsReader()) + , AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_) , heavy_metric_update_period(heavy_metrics_update_period_seconds) { /// sanity check diff --git a/src/Storages/System/StorageSystemServerSettings.cpp b/src/Storages/System/StorageSystemServerSettings.cpp index d242b6de4ec..ee99c472620 100644 --- a/src/Storages/System/StorageSystemServerSettings.cpp +++ b/src/Storages/System/StorageSystemServerSettings.cpp @@ -63,7 +63,6 @@ void StorageSystemServerSettings::fillData(MutableColumns & res_columns, Context /// current setting values, one needs to ask the components directly. std::unordered_map> changeable_settings = { {"max_server_memory_usage", {std::to_string(total_memory_tracker.getHardLimit()), ChangeableWithoutRestart::Yes}}, - {"allow_use_jemalloc_memory", {std::to_string(total_memory_tracker.getAllowUseJemallocMmemory()), ChangeableWithoutRestart::Yes}}, {"max_table_size_to_drop", {std::to_string(context->getMaxTableSizeToDrop()), ChangeableWithoutRestart::Yes}}, {"max_partition_size_to_drop", {std::to_string(context->getMaxPartitionSizeToDrop()), ChangeableWithoutRestart::Yes}}, diff --git a/tests/integration/test_memory_limit/test.py b/tests/integration/test_memory_limit/test.py index 6d6745711da..db68a38c1b1 100644 --- a/tests/integration/test_memory_limit/test.py +++ b/tests/integration/test_memory_limit/test.py @@ -13,7 +13,6 @@ node = cluster.add_instance( "configs/async_metrics_no.xml", ], mem_limit="4g", - env_variables={"MALLOC_CONF": "dirty_decay_ms:0"}, ) From 9ec1fd1ab769b2f6c6ad713b4e747a89bde48b78 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 18 Jul 2024 11:29:03 +0200 Subject: [PATCH 071/215] Fix non-jemalloc builds --- src/Common/MemoryWorker.cpp | 2 +- src/Common/MemoryWorker.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index dce47b83667..e5ebbe3b979 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -1,8 +1,8 @@ -#include "Common/ThreadPool.h" #include #include #include +#include namespace DB { diff --git a/src/Common/MemoryWorker.h b/src/Common/MemoryWorker.h index 8048194d9cd..5f02fd0b1d0 100644 --- a/src/Common/MemoryWorker.h +++ b/src/Common/MemoryWorker.h @@ -28,6 +28,8 @@ private: #else class MemoryWorker { +public: + explicit MemoryWorker(uint64_t /*period_ms_*/) {} }; #endif From 05c7dc582a48d738bed82bcb24d3a7619fec8bc9 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 18 Jul 2024 13:40:03 +0200 Subject: [PATCH 072/215] Add some comments --- src/Common/Jemalloc.h | 3 +++ src/Common/MemoryWorker.h | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/src/Common/Jemalloc.h b/src/Common/Jemalloc.h index 629d039b483..dfa265c5e59 100644 --- a/src/Common/Jemalloc.h +++ b/src/Common/Jemalloc.h @@ -41,6 +41,9 @@ T getJemallocValue(const char * name) return value; } +/// Each mallctl call consists of string name lookup which can be expensive. +/// This can be avoided by translating name to "Management Information Base" (MIB) +/// and using it in mallctlbymib calls template struct JemallocMibCache { diff --git a/src/Common/MemoryWorker.h b/src/Common/MemoryWorker.h index 5f02fd0b1d0..6c0a578aa61 100644 --- a/src/Common/MemoryWorker.h +++ b/src/Common/MemoryWorker.h @@ -8,6 +8,12 @@ namespace DB { #if USE_JEMALLOC +/// Correct MemoryTracker based on stats.resident read from jemalloc. +/// This requires jemalloc built with --enable-stats which we use. +/// The worker spawns a background thread which moves the jemalloc epoch (updates internal stats), +/// and fetches the current stats.resident whose value is sent to global MemoryTracker. +/// Additionally, if the current memory usage is higher than global hard limit, +/// jemalloc's dirty pages are forcefully purged. class MemoryWorker { public: From e47fe15968b84a3f9ea5a911bbad2a942fd27188 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 18 Jul 2024 12:22:03 +0000 Subject: [PATCH 073/215] Simplify condition --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 06455526f1b..e967b575acb 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -591,10 +591,9 @@ Pipe ReadFromMergeTree::readInOrder( /// If parallel replicas enabled, set total rows in progress here only on initiator with local plan /// Otherwise rows will counted multiple times const UInt64 in_order_limit = query_info.input_order_info ? query_info.input_order_info->limit : 0; - const bool parallel_replicas_remote_plan_for_initiator = is_parallel_reading_from_replicas - && !context->getSettingsRef().parallel_replicas_local_plan && context->canUseParallelReplicasOnInitiator(); - const bool parallel_replicas_follower = is_parallel_reading_from_replicas && context->canUseParallelReplicasOnFollower(); - const bool set_total_rows_approx = !parallel_replicas_follower && !parallel_replicas_remote_plan_for_initiator; + const bool parallel_replicas_local_plan_for_initiator = is_parallel_reading_from_replicas + && context->getSettingsRef().parallel_replicas_local_plan && context->canUseParallelReplicasOnInitiator(); + const bool set_total_rows_approx = !is_parallel_reading_from_replicas || parallel_replicas_local_plan_for_initiator; Pipes pipes; for (size_t i = 0; i < parts_with_ranges.size(); ++i) From 7523dafc067c000ffe329808f1f409c2a52d3e9e Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 18 Jul 2024 13:09:07 +0000 Subject: [PATCH 074/215] Fix after incorrect merge conflict resolution --- src/Processors/QueryPlan/ReadFromRemote.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 4599343f5c1..fdf35be6460 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -493,7 +493,7 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder /// when using local plan for local replica, local replica has 0 number size_t offset = (exclude_local_replica ? 1 : 0); - for (size_t i = 0; i < max_replicas_to_use; ++i) + for (size_t i = 0 + offset; i < pools_to_use.size(); ++i) { IConnections::ReplicaInfo replica_info{ /// we should use this number specifically because efficiency of data distribution by consistent hash depends on it. From 7ad07657a32415bd1b9df5541452d46e3b1e0c3d Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 18 Jul 2024 14:16:44 +0000 Subject: [PATCH 075/215] Fix --- src/Processors/QueryPlan/ReadFromRemote.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index fdf35be6460..79ad436c37e 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -493,7 +493,7 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder /// when using local plan for local replica, local replica has 0 number size_t offset = (exclude_local_replica ? 1 : 0); - for (size_t i = 0 + offset; i < pools_to_use.size(); ++i) + for (size_t i = 0 + offset; i < max_replicas_to_use; ++i) { IConnections::ReplicaInfo replica_info{ /// we should use this number specifically because efficiency of data distribution by consistent hash depends on it. From 7d66f400b25a5bf0f2f43f06c6ed432d1c572fb6 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 18 Jul 2024 16:51:49 +0200 Subject: [PATCH 076/215] Better --- src/Common/Jemalloc.h | 5 +++++ src/Common/MemoryTracker.cpp | 13 ++++++++----- src/Common/MemoryTracker.h | 8 ++++---- src/Common/MemoryWorker.cpp | 27 +++++++++++++++++++++++++-- src/Common/ProfileEvents.cpp | 3 +++ 5 files changed, 45 insertions(+), 11 deletions(-) diff --git a/src/Common/Jemalloc.h b/src/Common/Jemalloc.h index dfa265c5e59..22a94a44eba 100644 --- a/src/Common/Jemalloc.h +++ b/src/Common/Jemalloc.h @@ -65,6 +65,11 @@ struct JemallocMibCache return value; } + void run() + { + mallctlbymib(mib, mib_length, nullptr, nullptr, nullptr, 0); + } + private: static constexpr size_t max_mib_length = 4; size_t mib[max_mib_length]; diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index e237c3a0d33..49a3a6ef7ef 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -242,6 +242,7 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed * So, we allow over-allocations. */ Int64 will_be = size ? size + amount.fetch_add(size, std::memory_order_relaxed) : amount.load(std::memory_order_relaxed); + Int64 will_be_rss = size + rss.load(std::memory_order_relaxed); auto metric_loaded = metric.load(std::memory_order_relaxed); if (metric_loaded != CurrentMetrics::end() && size) @@ -290,7 +291,7 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed } } - if (unlikely(current_hard_limit && will_be > current_hard_limit)) + if (unlikely(current_hard_limit && (will_be > current_hard_limit || will_be_rss > current_hard_limit))) { if (memoryTrackerCanThrow(level, false) && throw_if_memory_exceeded) { @@ -310,12 +311,13 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed throw DB::Exception( DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED, "Memory limit{}{} exceeded: " - "would use {} (attempt to allocate chunk of {} bytes), maximum: {}." + "would use {} (attempt to allocate chunk of {} bytes), current RSS {}, maximum: {}." "{}{}", description ? " " : "", description ? description : "", formatReadableSizeWithBinarySuffix(will_be), size, + formatReadableSizeWithBinarySuffix(rss.load(std::memory_order_relaxed)), formatReadableSizeWithBinarySuffix(current_hard_limit), overcommit_result == OvercommitResult::NONE ? "" : " OvercommitTracker decision: ", toDescription(overcommit_result)); @@ -496,17 +498,18 @@ void MemoryTracker::reset() } -void MemoryTracker::setRSS(Int64 rss_) +void MemoryTracker::updateValues(Int64 rss_, Int64 allocated_) { - Int64 new_amount = rss_; + Int64 new_amount = allocated_; total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed); + total_memory_tracker.rss.store(rss_, std::memory_order_relaxed); auto metric_loaded = total_memory_tracker.metric.load(std::memory_order_relaxed); if (metric_loaded != CurrentMetrics::end()) CurrentMetrics::set(metric_loaded, new_amount); bool log_memory_usage = true; - total_memory_tracker.updatePeak(rss_, log_memory_usage); + total_memory_tracker.updatePeak(new_amount, log_memory_usage); } diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index 4085bb321ed..add8bcb43d2 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -57,6 +57,8 @@ private: std::atomic hard_limit {0}; std::atomic profiler_limit {0}; + std::atomic rss{0}; + Int64 profiler_step = 0; /// To test exception safety of calling code, memory tracker throws an exception on each memory allocation with specified probability. @@ -237,10 +239,8 @@ public: /// Reset the accumulated data. void reset(); - /// Reset current counter to an RSS value. - /// Jemalloc may have pre-allocated arenas, they are accounted in RSS. - /// We can free this arenas in case of exception to avoid OOM. - static void setRSS(Int64 rss_); + /// update values based on external information (e.g. jemalloc's stat) + static void updateValues(Int64 rss_, Int64 allocated_); /// Prints info about peak memory consumption into log. void logPeakMemoryUsage(); diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index e5ebbe3b979..ae488a47b67 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -3,11 +3,23 @@ #include #include #include +#include + +namespace ProfileEvents +{ + extern const Event MemoryAllocatorPurge; + extern const Event MemoryAllocatorPurgeTimeMicroseconds; + extern const Event MemoryWorkerRun; + extern const Event MemoryWorkerRunElapsedMicroseconds; +} namespace DB { #if USE_JEMALLOC +#define STRINGIFY_HELPER(x) #x +#define STRINGIFY(x) STRINGIFY_HELPER(x) + MemoryWorker::MemoryWorker(uint64_t period_ms_) : period_ms(period_ms_) { @@ -30,6 +42,8 @@ void MemoryWorker::backgroundThread() { JemallocMibCache epoch_mib("epoch"); JemallocMibCache resident_mib("stats.resident"); + JemallocMibCache allocated_mib("stats.allocated"); + JemallocMibCache purge_mib("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge"); std::unique_lock lock(mutex); while (true) { @@ -37,11 +51,20 @@ void MemoryWorker::backgroundThread() if (shutdown) return; + Stopwatch total_watch; epoch_mib.setValue(0); Int64 resident = resident_mib.getValue(); - MemoryTracker::setRSS(resident); if (resident > total_memory_tracker.getHardLimit()) - purgeJemallocArenas(); + { + Stopwatch purge_watch; + purge_mib.run(); + ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurge); + ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, purge_watch.elapsedMicroseconds()); + } + + MemoryTracker::updateValues(resident, allocated_mib.getValue()); + ProfileEvents::increment(ProfileEvents::MemoryWorkerRun); + ProfileEvents::increment(ProfileEvents::MemoryWorkerRunElapsedMicroseconds, total_watch.elapsedMicroseconds()); } } #endif diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 871ba7cab8b..d85c21fcded 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -778,6 +778,9 @@ The server successfully detected this situation and will download merged part fr M(GWPAsanAllocateSuccess, "Number of successful allocations done by GWPAsan") \ M(GWPAsanAllocateFailed, "Number of failed allocations done by GWPAsan (i.e. filled pool)") \ M(GWPAsanFree, "Number of free operations done by GWPAsan") \ + \ + M(MemoryWorkerRun, "Number of runs done by MemoryWorker in background") \ + M(MemoryWorkerRunElapsedMicroseconds, "Total time spent by MemoryWorker for background work") \ #ifdef APPLY_FOR_EXTERNAL_EVENTS From 6db09e884b9fcc516c6c4c0afe1dd3a33ec1ada9 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 19 Jul 2024 08:24:29 +0000 Subject: [PATCH 077/215] Simplify code --- src/Processors/QueryPlan/ReadFromRemote.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 79ad436c37e..8a2d773696b 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -491,16 +491,17 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder addresses.emplace_back(pool->getAddress()); LOG_DEBUG(getLogger("ReadFromParallelRemoteReplicasStep"), "Addresses to use: {}", fmt::join(addresses, ", ")); - /// when using local plan for local replica, local replica has 0 number - size_t offset = (exclude_local_replica ? 1 : 0); - for (size_t i = 0 + offset; i < max_replicas_to_use; ++i) + /// when using local plan for local replica, 0 is assigned to local replica as replica num, - in this case, starting from 1 here + size_t replica_num = (exclude_local_replica ? 1 : 0); + for (const auto & pool : pools_to_use) { IConnections::ReplicaInfo replica_info{ /// we should use this number specifically because efficiency of data distribution by consistent hash depends on it. - .number_of_current_replica = i, + .number_of_current_replica = replica_num, }; + ++replica_num; - addPipeForSingeReplica(pipes, pools_to_use[i - offset], replica_info); + addPipeForSingeReplica(pipes, pool, replica_info); } auto pipe = Pipe::unitePipes(std::move(pipes)); From daabf2275e3b04bc40949aade73f544f56d78931 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 19 Jul 2024 21:46:14 +0000 Subject: [PATCH 078/215] Remove wrong comment --- src/Processors/QueryPlan/ReadFromRemote.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 8a2d773696b..46d4aa29e70 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -393,7 +393,6 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep( for (const auto & addr : cluster->getShardsAddresses().front()) { - /// skip first local if (exclude_local_replica && addr.is_local) continue; From 4e69cd0d5262fed5bbebaf30fd2f34d6cb2ad830 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Sat, 20 Jul 2024 20:53:18 +0000 Subject: [PATCH 079/215] Avoid getting callbacks from context for local plan --- .../QueryPlan/ParallelReplicasLocalPlan.cpp | 2 +- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 15 +++++++++++---- src/Processors/QueryPlan/ReadFromMergeTree.h | 4 ++-- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp index e9d5ef90e30..e8ff0f417dc 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -85,7 +85,7 @@ std::pair, bool> createLocalPlanForParallelReplicas( { return coordinator->handleRequest(std::move(req)); }; auto read_from_merge_tree_parallel_replicas - = reading->createLocalParallelReplicasReadingStep(analyzed_result_ptr, all_ranges_cb, read_task_cb); + = reading->createLocalParallelReplicasReadingStep(analyzed_result_ptr, std::move(all_ranges_cb), std::move(read_task_cb)); node->step = std::move(read_from_merge_tree_parallel_replicas); addConvertingActions(*query_plan, header, /*has_missing_objects=*/false); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index e967b575acb..6f0037684e9 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -303,8 +303,15 @@ ReadFromMergeTree::ReadFromMergeTree( { if (is_parallel_reading_from_replicas) { - all_ranges_callback = all_ranges_callback_.value_or(context->getMergeTreeAllRangesCallback()); - read_task_callback = read_task_callback_.value_or(context->getMergeTreeReadTaskCallback()); + if (all_ranges_callback_.has_value()) + all_ranges_callback = all_ranges_callback_.value(); + else + all_ranges_callback = context->getMergeTreeAllRangesCallback(); + + if (read_task_callback_.has_value()) + read_task_callback = read_task_callback_.value(); + else + read_task_callback = context->getMergeTreeReadTaskCallback(); } const auto & settings = context->getSettingsRef(); @@ -340,8 +347,8 @@ ReadFromMergeTree::ReadFromMergeTree( std::unique_ptr ReadFromMergeTree::createLocalParallelReplicasReadingStep( AnalysisResultPtr analyzed_result_ptr_, - std::optional all_ranges_callback_, - std::optional read_task_callback_) + MergeTreeAllRangesCallback all_ranges_callback_, + MergeTreeReadTaskCallback read_task_callback_) { const auto number_of_local_replica = 0; const bool enable_parallel_reading = true; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index c9c6030d207..307b605c01c 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -128,8 +128,8 @@ public: std::unique_ptr createLocalParallelReplicasReadingStep( AnalysisResultPtr analyzed_result_ptr_, - std::optional all_ranges_callback_, - std::optional read_task_callback_); + MergeTreeAllRangesCallback all_ranges_callback_, + MergeTreeReadTaskCallback read_task_callback_); static constexpr auto name = "ReadFromMergeTree"; String getName() const override { return name; } From 2147a96475717f0af53dd62f487c011a5b9b933a Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Sun, 21 Jul 2024 11:32:57 +0200 Subject: [PATCH 080/215] Better --- src/Common/AsynchronousMetrics.cpp | 4 +++ src/Common/MemoryTracker.cpp | 31 +++++++++++++++---- src/Common/MemoryTracker.h | 2 +- src/Common/MemoryWorker.cpp | 7 ++++- src/Coordination/KeeperDispatcher.cpp | 8 ++++- .../configs/keeper_config2.xml | 2 +- .../configs/keeper_config3.xml | 2 +- 7 files changed, 45 insertions(+), 11 deletions(-) diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index a5c9875188b..dc2f687004b 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -638,6 +638,10 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) "The amount of virtual memory mapped for the use of stack and for the allocated memory, in bytes." " It is unspecified whether it includes the per-thread stacks and most of the allocated memory, that is allocated with the 'mmap' system call." " This metric exists only for completeness reasons. I recommend to use the `MemoryResident` metric for monitoring."}; + +#if !USE_JEMALLOC + MemoryTracker::updateValues(data.resident, data.resident, /*force_update=*/true); +#endif } { diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 49a3a6ef7ef..07d6ba98745 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -221,6 +221,7 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed { /// For global memory tracker always update memory usage. amount.fetch_add(size, std::memory_order_relaxed); + rss.fetch_add(size, std::memory_order_relaxed); auto metric_loaded = metric.load(std::memory_order_relaxed); if (metric_loaded != CurrentMetrics::end()) @@ -242,7 +243,7 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed * So, we allow over-allocations. */ Int64 will_be = size ? size + amount.fetch_add(size, std::memory_order_relaxed) : amount.load(std::memory_order_relaxed); - Int64 will_be_rss = size + rss.load(std::memory_order_relaxed); + Int64 will_be_rss = size ? size + rss.fetch_add(size, std::memory_order_relaxed) : rss.load(std::memory_order_relaxed); auto metric_loaded = metric.load(std::memory_order_relaxed); if (metric_loaded != CurrentMetrics::end() && size) @@ -269,6 +270,7 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed { /// Revert amount.fetch_sub(size, std::memory_order_relaxed); + rss.fetch_sub(size, std::memory_order_relaxed); /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc MemoryTrackerBlockerInThread untrack_lock(VariableContext::Global); @@ -291,7 +293,8 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed } } - if (unlikely(current_hard_limit && (will_be > current_hard_limit || will_be_rss > current_hard_limit))) + if (unlikely( + current_hard_limit && (will_be > current_hard_limit || (level == VariableContext::Global && will_be_rss > current_hard_limit)))) { if (memoryTrackerCanThrow(level, false) && throw_if_memory_exceeded) { @@ -303,6 +306,7 @@ AllocationTrace MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceed { /// Revert amount.fetch_sub(size, std::memory_order_relaxed); + rss.fetch_sub(size, std::memory_order_relaxed); /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc MemoryTrackerBlockerInThread untrack_lock(VariableContext::Global); @@ -411,6 +415,7 @@ AllocationTrace MemoryTracker::free(Int64 size, double _sample_probability) { /// For global memory tracker always update memory usage. amount.fetch_sub(size, std::memory_order_relaxed); + rss.fetch_sub(size, std::memory_order_relaxed); auto metric_loaded = metric.load(std::memory_order_relaxed); if (metric_loaded != CurrentMetrics::end()) CurrentMetrics::sub(metric_loaded, size); @@ -424,7 +429,12 @@ AllocationTrace MemoryTracker::free(Int64 size, double _sample_probability) } Int64 accounted_size = size; - if (level == VariableContext::Thread || level == VariableContext::Global) + if (level == VariableContext::Global) + { + amount.fetch_sub(accounted_size, std::memory_order_relaxed); + rss.fetch_sub(accounted_size, std::memory_order_relaxed); + } + else if (level == VariableContext::Thread) { /// Could become negative if memory allocated in this thread is freed in another one amount.fetch_sub(accounted_size, std::memory_order_relaxed); @@ -498,12 +508,21 @@ void MemoryTracker::reset() } -void MemoryTracker::updateValues(Int64 rss_, Int64 allocated_) +void MemoryTracker::updateValues(Int64 rss_, Int64 allocated_, bool force_update) { - Int64 new_amount = allocated_; - total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed); total_memory_tracker.rss.store(rss_, std::memory_order_relaxed); + if (likely(!force_update && total_memory_tracker.amount.load(std::memory_order_relaxed) >= 0)) + return; + + Int64 new_amount = allocated_; + LOG_INFO( + getLogger("MemoryTracker"), + "Correcting the value of global memory tracker from {} to {}", + ReadableSize(total_memory_tracker.amount.load(std::memory_order_relaxed)), + ReadableSize(allocated_)); + total_memory_tracker.amount.store(new_amount, std::memory_order_relaxed); + auto metric_loaded = total_memory_tracker.metric.load(std::memory_order_relaxed); if (metric_loaded != CurrentMetrics::end()) CurrentMetrics::set(metric_loaded, new_amount); diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index add8bcb43d2..4913be9781f 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -240,7 +240,7 @@ public: void reset(); /// update values based on external information (e.g. jemalloc's stat) - static void updateValues(Int64 rss_, Int64 allocated_); + static void updateValues(Int64 rss_, Int64 allocated_, bool force_update); /// Prints info about peak memory consumption into log. void logPeakMemoryUsage(); diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index ae488a47b67..23cd90178ff 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -44,6 +44,7 @@ void MemoryWorker::backgroundThread() JemallocMibCache resident_mib("stats.resident"); JemallocMibCache allocated_mib("stats.allocated"); JemallocMibCache purge_mib("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge"); + bool first_run = false; std::unique_lock lock(mutex); while (true) { @@ -62,9 +63,13 @@ void MemoryWorker::backgroundThread() ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, purge_watch.elapsedMicroseconds()); } - MemoryTracker::updateValues(resident, allocated_mib.getValue()); + /// force update the allocated stat from jemalloc for the first run to cover the allocations we missed + /// during initialization + MemoryTracker::updateValues(resident, allocated_mib.getValue(), first_run); ProfileEvents::increment(ProfileEvents::MemoryWorkerRun); ProfileEvents::increment(ProfileEvents::MemoryWorkerRunElapsedMicroseconds, total_watch.elapsedMicroseconds()); + + first_run = false; } } #endif diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index 8c7e6405153..332662117d8 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -148,7 +148,13 @@ void KeeperDispatcher::requestThread() Int64 mem_soft_limit = keeper_context->getKeeperMemorySoftLimit(); if (configuration_and_settings->standalone_keeper && isExceedingMemorySoftLimit() && checkIfRequestIncreaseMem(request.request)) { - LOG_WARNING(log, "Processing requests refused because of max_memory_usage_soft_limit {}, the total used memory is {}, request type is {}", ReadableSize(mem_soft_limit), ReadableSize(total_memory_tracker.get()), request.request->getOpNum()); + LOG_WARNING( + log, + "Processing requests refused because of max_memory_usage_soft_limit {}, the total used memory is {}, request type " + "is {}", + ReadableSize(mem_soft_limit), + ReadableSize(total_memory_tracker.get()), + request.request->getOpNum()); addErrorResponses({request}, Coordination::Error::ZCONNECTIONLOSS); continue; } diff --git a/tests/integration/test_keeper_memory_soft_limit/configs/keeper_config2.xml b/tests/integration/test_keeper_memory_soft_limit/configs/keeper_config2.xml index 25ececea3e8..e71b93379d0 100644 --- a/tests/integration/test_keeper_memory_soft_limit/configs/keeper_config2.xml +++ b/tests/integration/test_keeper_memory_soft_limit/configs/keeper_config2.xml @@ -16,7 +16,7 @@ az-zoo2 1 - 20000000 + 200000000 10000 diff --git a/tests/integration/test_keeper_memory_soft_limit/configs/keeper_config3.xml b/tests/integration/test_keeper_memory_soft_limit/configs/keeper_config3.xml index 81e343b77c9..cf4a4686f2c 100644 --- a/tests/integration/test_keeper_memory_soft_limit/configs/keeper_config3.xml +++ b/tests/integration/test_keeper_memory_soft_limit/configs/keeper_config3.xml @@ -13,7 +13,7 @@ 2181 3 - 20000000 + 200000000 10000 From 1c3f7d0fd0fbf27692ff29d8309382eae7b7a598 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 22 Jul 2024 14:58:00 +0200 Subject: [PATCH 081/215] Small fix --- programs/server/Server.cpp | 4 ++-- src/Common/MemoryWorker.cpp | 15 ++++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 04480f0bfe9..5691d82e216 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -904,8 +904,6 @@ try LOG_INFO(log, "Background threads finished in {} ms", watch.elapsedMilliseconds()); }); - MemoryWorker memory_worker(global_context->getServerSettings().memory_worker_period_ms); - /// This object will periodically calculate some metrics. ServerAsynchronousMetrics async_metrics( global_context, @@ -1198,6 +1196,8 @@ try FailPointInjection::enableFromGlobalConfig(config()); + MemoryWorker memory_worker(global_context->getServerSettings().memory_worker_period_ms); + int default_oom_score = 0; #if !defined(NDEBUG) diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index 23cd90178ff..2b945a30d3d 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -2,8 +2,9 @@ #include #include -#include #include +#include +#include namespace ProfileEvents { @@ -23,6 +24,7 @@ namespace DB MemoryWorker::MemoryWorker(uint64_t period_ms_) : period_ms(period_ms_) { + LOG_INFO(getLogger("MemoryWorker"), "Starting background memory thread with period of {}ms", period_ms.count()); background_thread = ThreadFromGlobalPool([this] { backgroundThread(); }); } @@ -42,9 +44,10 @@ void MemoryWorker::backgroundThread() { JemallocMibCache epoch_mib("epoch"); JemallocMibCache resident_mib("stats.resident"); + JemallocMibCache active_mib("stats.active"); JemallocMibCache allocated_mib("stats.allocated"); JemallocMibCache purge_mib("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge"); - bool first_run = false; + bool first_run = true; std::unique_lock lock(mutex); while (true) { @@ -55,6 +58,11 @@ void MemoryWorker::backgroundThread() Stopwatch total_watch; epoch_mib.setValue(0); Int64 resident = resident_mib.getValue(); + + /// force update the allocated stat from jemalloc for the first run to cover the allocations we missed + /// during initialization + MemoryTracker::updateValues(resident, allocated_mib.getValue(), first_run); + if (resident > total_memory_tracker.getHardLimit()) { Stopwatch purge_watch; @@ -63,9 +71,6 @@ void MemoryWorker::backgroundThread() ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, purge_watch.elapsedMicroseconds()); } - /// force update the allocated stat from jemalloc for the first run to cover the allocations we missed - /// during initialization - MemoryTracker::updateValues(resident, allocated_mib.getValue(), first_run); ProfileEvents::increment(ProfileEvents::MemoryWorkerRun); ProfileEvents::increment(ProfileEvents::MemoryWorkerRunElapsedMicroseconds, total_watch.elapsedMicroseconds()); From fa0f4543b688861a97843a8767d895f52b6dee9d Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 22 Jul 2024 13:44:25 +0000 Subject: [PATCH 082/215] Fix marks multiplier for local replica to avoid overflow with fuzzer --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 6f0037684e9..5922550eaf3 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -384,7 +384,11 @@ Pipe ReadFromMergeTree::readFromPoolParallelReplicas(RangesInDataParts parts_wit /// We have a special logic for local replica. It has to read less data, because in some cases it should /// merge states of aggregate functions or do some other important stuff other than reading from Disk. - const auto multiplier = context->getSettingsRef().parallel_replicas_single_task_marks_count_multiplier; + auto multiplier = context->getSettingsRef().parallel_replicas_single_task_marks_count_multiplier; + // avoid using multiplier if min marks to read is already too big (to avoid overflow) + if (pool_settings.min_marks_for_concurrent_read >= std::numeric_limits::max()) + multiplier = 1.0f; + if (auto result = pool_settings.min_marks_for_concurrent_read * multiplier; canConvertTo(result)) pool_settings.min_marks_for_concurrent_read = static_cast(result); else @@ -553,7 +557,11 @@ Pipe ReadFromMergeTree::readInOrder( .number_of_current_replica = number_of_current_replica.value_or(client_info.number_of_current_replica), }; - const auto multiplier = context->getSettingsRef().parallel_replicas_single_task_marks_count_multiplier; + auto multiplier = context->getSettingsRef().parallel_replicas_single_task_marks_count_multiplier; + // avoid using multiplier if min marks to read is already too big (to avoid overflow) + if (pool_settings.min_marks_for_concurrent_read >= std::numeric_limits::max()) + multiplier = 1.0f; + if (auto result = pool_settings.min_marks_for_concurrent_read * multiplier; canConvertTo(result)) pool_settings.min_marks_for_concurrent_read = static_cast(result); else From 4483f4b3894e846deb43fa4e7d0962eba2a2224b Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 22 Jul 2024 16:39:07 +0000 Subject: [PATCH 083/215] Better logging for announcement --- .../MergeTree/ParallelReplicasReadingCoordinator.cpp | 7 +++---- src/Storages/MergeTree/RequestResponse.cpp | 5 +---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp index e1c0d87837a..4e3c9b08fb0 100644 --- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp +++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp @@ -433,12 +433,11 @@ void DefaultCoordinator::setProgressCallback() void DefaultCoordinator::doHandleInitialAllRangesAnnouncement(InitialAllRangesAnnouncement announcement) { - const auto replica_num = announcement.replica_num; - - LOG_DEBUG(log, "Initial request from replica {}: {}", announcement.replica_num, announcement.describe()); + LOG_DEBUG(log, "Initial request: {}", announcement.describe()); initializeReadingState(std::move(announcement)); + const auto replica_num = announcement.replica_num; if (replica_num >= stats.size()) throw Exception( ErrorCodes::LOGICAL_ERROR, "Replica number ({}) is bigger than total replicas count ({})", replica_num, stats.size()); @@ -859,7 +858,7 @@ void InOrderCoordinator::markReplicaAsUnavailable(size_t replica_number) template void InOrderCoordinator::doHandleInitialAllRangesAnnouncement(InitialAllRangesAnnouncement announcement) { - LOG_TRACE(log, "Received an announcement {}", announcement.describe()); + LOG_TRACE(log, "Received an announcement : {}", announcement.describe()); ++stats[announcement.replica_num].number_of_requests; diff --git a/src/Storages/MergeTree/RequestResponse.cpp b/src/Storages/MergeTree/RequestResponse.cpp index 33cd935c88b..bcdeb443a0b 100644 --- a/src/Storages/MergeTree/RequestResponse.cpp +++ b/src/Storages/MergeTree/RequestResponse.cpp @@ -129,10 +129,7 @@ void InitialAllRangesAnnouncement::serialize(WriteBuffer & out) const String InitialAllRangesAnnouncement::describe() { - String result; - result += description.describe(); - result += fmt::format("----------\nReceived from {} replica\n", replica_num); - return result; + return fmt::format("replica {}, mode {}, {}", replica_num, mode, description.describe()); } InitialAllRangesAnnouncement InitialAllRangesAnnouncement::deserialize(ReadBuffer & in) From 420716d702daf8885796ed1b9b2cf83c437d6e32 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 23 Jul 2024 06:38:37 +0000 Subject: [PATCH 084/215] Fix clang-tidy warning --- src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp index 4e3c9b08fb0..a1b36fe1299 100644 --- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp +++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp @@ -435,9 +435,10 @@ void DefaultCoordinator::doHandleInitialAllRangesAnnouncement(InitialAllRangesAn { LOG_DEBUG(log, "Initial request: {}", announcement.describe()); + const auto replica_num = announcement.replica_num; + initializeReadingState(std::move(announcement)); - const auto replica_num = announcement.replica_num; if (replica_num >= stats.size()) throw Exception( ErrorCodes::LOGICAL_ERROR, "Replica number ({}) is bigger than total replicas count ({})", replica_num, stats.size()); From d78cfd030fa8364456ac5283a6a1469703c53b40 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Mon, 22 Jul 2024 21:47:46 +0200 Subject: [PATCH 085/215] Use cgroups as source --- programs/keeper/Keeper.cpp | 8 +- programs/server/Server.cpp | 9 +- src/Common/AsynchronousMetrics.cpp | 32 +- src/Common/AsynchronousMetrics.h | 10 +- src/Common/CgroupsMemoryUsageObserver.cpp | 168 +---------- src/Common/CgroupsMemoryUsageObserver.h | 12 - src/Common/MemoryTracker.cpp | 9 +- src/Common/MemoryTracker.h | 3 +- src/Common/MemoryWorker.cpp | 280 ++++++++++++++++-- src/Common/MemoryWorker.h | 49 ++- .../KeeperAsynchronousMetrics.cpp | 9 +- src/Coordination/KeeperAsynchronousMetrics.h | 8 +- src/Core/ServerSettings.h | 2 +- .../ServerAsynchronousMetrics.cpp | 6 +- src/Interpreters/ServerAsynchronousMetrics.h | 5 +- 15 files changed, 367 insertions(+), 243 deletions(-) diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index b10d3f34623..d308e741311 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -376,7 +376,8 @@ try LOG_INFO(log, "Background threads finished in {} ms", watch.elapsedMilliseconds()); }); - MemoryWorker memory_worker(config().getUInt64("memory_worker_period_ms", 100)); + MemoryWorker memory_worker(config().getUInt64("memory_worker_period_ms", 0)); + memory_worker.start(); static ServerErrorHandler error_handler; Poco::ErrorHandler::set(&error_handler); @@ -419,8 +420,9 @@ try for (const auto & server : *servers) metrics.emplace_back(ProtocolServerMetrics{server.getPortName(), server.currentThreads()}); return metrics; - } - ); + }, + /*update_jemalloc_epoch_=*/memory_worker.getSource() != MemoryWorker::MemoryUsageSource::Jemalloc, + /*update_rss_=*/memory_worker.getSource() == MemoryWorker::MemoryUsageSource::None); std::vector listen_hosts = DB::getMultipleValuesFromConfig(config(), "", "listen_host"); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 5691d82e216..1fc1df1494c 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -904,6 +904,8 @@ try LOG_INFO(log, "Background threads finished in {} ms", watch.elapsedMilliseconds()); }); + MemoryWorker memory_worker(global_context->getServerSettings().memory_worker_period_ms); + /// This object will periodically calculate some metrics. ServerAsynchronousMetrics async_metrics( global_context, @@ -922,8 +924,9 @@ try for (const auto & server : servers) metrics.emplace_back(ProtocolServerMetrics{server.getPortName(), server.currentThreads()}); return metrics; - } - ); + }, + /*update_jemalloc_epoch_=*/memory_worker.getSource() != MemoryWorker::MemoryUsageSource::Jemalloc, + /*update_rss_=*/memory_worker.getSource() == MemoryWorker::MemoryUsageSource::None); /// NOTE: global context should be destroyed *before* GlobalThreadPool::shutdown() /// Otherwise GlobalThreadPool::shutdown() will hang, since Context holds some threads. @@ -1196,7 +1199,7 @@ try FailPointInjection::enableFromGlobalConfig(config()); - MemoryWorker memory_worker(global_context->getServerSettings().memory_worker_period_ms); + memory_worker.start(); int default_oom_score = 0; diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index dc2f687004b..53b8e13eaaa 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -58,10 +58,14 @@ static std::unique_ptr openFileIfExists(const std::stri AsynchronousMetrics::AsynchronousMetrics( unsigned update_period_seconds, - const ProtocolServerMetricsFunc & protocol_server_metrics_func_) + const ProtocolServerMetricsFunc & protocol_server_metrics_func_, + bool update_jemalloc_epoch_, + bool update_rss_) : update_period(update_period_seconds) , log(getLogger("AsynchronousMetrics")) , protocol_server_metrics_func(protocol_server_metrics_func_) + , update_jemalloc_epoch(update_jemalloc_epoch_) + , update_rss(update_rss_) { #if defined(OS_LINUX) openFileIfExists("/proc/meminfo", meminfo); @@ -377,6 +381,14 @@ void AsynchronousMetrics::run() namespace { +uint64_t updateJemallocEpoch() +{ + uint64_t value = 0; + size_t size = sizeof(value); + mallctl("epoch", &value, &size, &value, size); + return value; +} + template Value saveJemallocMetricImpl( AsynchronousMetricValues & values, @@ -593,8 +605,11 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) // 'epoch' is a special mallctl -- it updates the statistics. Without it, all // the following calls will return stale values. It increments and returns // the current epoch number, which might be useful to log as a sanity check. - auto epoch = getJemallocValue("epoch"); - new_values["jemalloc.epoch"] = { epoch, "An internal incremental update number of the statistics of jemalloc (Jason Evans' memory allocator), used in all other `jemalloc` metrics." }; + auto epoch = update_jemalloc_epoch ? updateJemallocEpoch() : getJemallocValue("epoch"); + new_values["jemalloc.epoch"] + = {epoch, + "An internal incremental update number of the statistics of jemalloc (Jason Evans' memory allocator), used in all other " + "`jemalloc` metrics."}; // Collect the statistics themselves. saveJemallocMetric(new_values, "allocated"); @@ -607,10 +622,10 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) saveJemallocMetric(new_values, "background_thread.num_threads"); saveJemallocMetric(new_values, "background_thread.num_runs"); saveJemallocMetric(new_values, "background_thread.run_intervals"); - saveJemallocProf(new_values, "active"); + saveJemallocProf(new_values, "active"); saveAllArenasMetric(new_values, "pactive"); - [[maybe_unused]] size_t je_malloc_pdirty = saveAllArenasMetric(new_values, "pdirty"); - [[maybe_unused]] size_t je_malloc_pmuzzy = saveAllArenasMetric(new_values, "pmuzzy"); + saveAllArenasMetric(new_values, "pdirty"); + saveAllArenasMetric(new_values, "pmuzzy"); saveAllArenasMetric(new_values, "dirty_purged"); saveAllArenasMetric(new_values, "muzzy_purged"); #endif @@ -639,9 +654,8 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update) " It is unspecified whether it includes the per-thread stacks and most of the allocated memory, that is allocated with the 'mmap' system call." " This metric exists only for completeness reasons. I recommend to use the `MemoryResident` metric for monitoring."}; -#if !USE_JEMALLOC - MemoryTracker::updateValues(data.resident, data.resident, /*force_update=*/true); -#endif + if (update_rss) + MemoryTracker::updateRSS(data.resident); } { diff --git a/src/Common/AsynchronousMetrics.h b/src/Common/AsynchronousMetrics.h index bc379d4e92b..eb6ede7a558 100644 --- a/src/Common/AsynchronousMetrics.h +++ b/src/Common/AsynchronousMetrics.h @@ -7,10 +7,8 @@ #include #include -#include #include #include -#include #include #include #include @@ -69,7 +67,9 @@ public: AsynchronousMetrics( unsigned update_period_seconds, - const ProtocolServerMetricsFunc & protocol_server_metrics_func_); + const ProtocolServerMetricsFunc & protocol_server_metrics_func_, + bool update_jemalloc_epoch_, + bool update_rss_); virtual ~AsynchronousMetrics(); @@ -92,7 +92,6 @@ private: virtual void logImpl(AsynchronousMetricValues &) {} ProtocolServerMetricsFunc protocol_server_metrics_func; - std::shared_ptr cgroups_reader; std::unique_ptr thread; @@ -113,6 +112,9 @@ private: MemoryStatisticsOS memory_stat TSA_GUARDED_BY(data_mutex); #endif + const bool update_jemalloc_epoch; + const bool update_rss; + #if defined(OS_LINUX) std::optional meminfo TSA_GUARDED_BY(data_mutex); std::optional loadavg TSA_GUARDED_BY(data_mutex); diff --git a/src/Common/CgroupsMemoryUsageObserver.cpp b/src/Common/CgroupsMemoryUsageObserver.cpp index ab7ca69ca04..afeac1808b2 100644 --- a/src/Common/CgroupsMemoryUsageObserver.cpp +++ b/src/Common/CgroupsMemoryUsageObserver.cpp @@ -13,12 +13,8 @@ #include #include -#include -#include -#include using namespace DB; -namespace fs = std::filesystem; namespace DB { @@ -29,170 +25,8 @@ extern const int FILE_DOESNT_EXIST; extern const int INCORRECT_DATA; } -} - -namespace -{ - -/// Format is -/// kernel 5 -/// rss 15 -/// [...] -uint64_t readMetricFromStatFile(ReadBufferFromFile & buf, const std::string & key) -{ - while (!buf.eof()) - { - std::string current_key; - readStringUntilWhitespace(current_key, buf); - if (current_key != key) - { - std::string dummy; - readStringUntilNewlineInto(dummy, buf); - buf.ignore(); - continue; - } - - assertChar(' ', buf); - uint64_t value = 0; - readIntText(value, buf); - return value; - } - - throw Exception(ErrorCodes::INCORRECT_DATA, "Cannot find '{}' in '{}'", key, buf.getFileName()); -} - -struct CgroupsV1Reader : ICgroupsReader -{ - explicit CgroupsV1Reader(const fs::path & stat_file_dir) : buf(stat_file_dir / "memory.stat") { } - - uint64_t readMemoryUsage() override - { - std::lock_guard lock(mutex); - buf.rewind(); - return readMetricFromStatFile(buf, "rss"); - } - -private: - std::mutex mutex; - ReadBufferFromFile buf TSA_GUARDED_BY(mutex); -}; - -struct CgroupsV2Reader : ICgroupsReader -{ - explicit CgroupsV2Reader(const fs::path & stat_file_dir) - : current_buf(stat_file_dir / "memory.current"), stat_buf(stat_file_dir / "memory.stat") - { - } - - uint64_t readMemoryUsage() override - { - std::lock_guard lock(mutex); - current_buf.rewind(); - stat_buf.rewind(); - - int64_t mem_usage = 0; - /// memory.current contains a single number - /// the reason why we subtract it described here: https://github.com/ClickHouse/ClickHouse/issues/64652#issuecomment-2149630667 - readIntText(mem_usage, current_buf); - mem_usage -= readMetricFromStatFile(stat_buf, "inactive_file"); - chassert(mem_usage >= 0, "Negative memory usage"); - return mem_usage; - } - -private: - std::mutex mutex; - ReadBufferFromFile current_buf TSA_GUARDED_BY(mutex); - ReadBufferFromFile stat_buf TSA_GUARDED_BY(mutex); -}; - -/// Caveats: -/// - All of the logic in this file assumes that the current process is the only process in the -/// containing cgroup (or more precisely: the only process with significant memory consumption). -/// If this is not the case, then other processe's memory consumption may affect the internal -/// memory tracker ... -/// - Cgroups v1 and v2 allow nested cgroup hierarchies. As v1 is deprecated for over half a -/// decade and will go away at some point, hierarchical detection is only implemented for v2. -/// - I did not test what happens if a host has v1 and v2 simultaneously enabled. I believe such -/// systems existed only for a short transition period. - -std::optional getCgroupsV2Path() -{ - if (!cgroupsV2Enabled()) - return {}; - - if (!cgroupsV2MemoryControllerEnabled()) - return {}; - - fs::path current_cgroup = cgroupV2PathOfProcess(); - if (current_cgroup.empty()) - return {}; - - /// Return the bottom-most nested current memory file. If there is no such file at the current - /// level, try again at the parent level as memory settings are inherited. - while (current_cgroup != default_cgroups_mount.parent_path()) - { - const auto current_path = current_cgroup / "memory.current"; - const auto stat_path = current_cgroup / "memory.stat"; - if (fs::exists(current_path) && fs::exists(stat_path)) - return {current_cgroup}; - current_cgroup = current_cgroup.parent_path(); - } - return {}; -} - -std::optional getCgroupsV1Path() -{ - auto path = default_cgroups_mount / "memory/memory.stat"; - if (!fs::exists(path)) - return {}; - return {default_cgroups_mount / "memory"}; -} - -enum class CgroupsVersion : uint8_t -{ - V1, - V2 -}; - -std::pair getCgroupsPath() -{ - auto v2_path = getCgroupsV2Path(); - if (v2_path.has_value()) - return {*v2_path, CgroupsVersion::V2}; - - auto v1_path = getCgroupsV1Path(); - if (v1_path.has_value()) - return {*v1_path, CgroupsVersion::V1}; - - throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Cannot find cgroups v1 or v2 current memory file"); -} - -} - -namespace DB -{ - -std::shared_ptr createCgroupsReader() -{ - const auto [cgroup_path, version] = getCgroupsPath(); - LOG_INFO( - getLogger("CgroupsReader"), - "Will create cgroup reader from '{}' (cgroups version: {})", - cgroup_path, - (version == CgroupsVersion::V1) ? "v1" : "v2"); - - if (version == CgroupsVersion::V2) - return std::make_shared(cgroup_path); - else - { - chassert(version == CgroupsVersion::V1); - return std::make_shared(cgroup_path); - } - -} - CgroupsMemoryUsageObserver::CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_) - : log(getLogger("CgroupsMemoryUsageObserver")), wait_time(wait_time_), cgroups_reader(createCgroupsReader()) + : log(getLogger("CgroupsMemoryUsageObserver")), wait_time(wait_time_) {} CgroupsMemoryUsageObserver::~CgroupsMemoryUsageObserver() diff --git a/src/Common/CgroupsMemoryUsageObserver.h b/src/Common/CgroupsMemoryUsageObserver.h index 33e0f167a59..3de83d6b437 100644 --- a/src/Common/CgroupsMemoryUsageObserver.h +++ b/src/Common/CgroupsMemoryUsageObserver.h @@ -3,21 +3,11 @@ #include #include -#include #include namespace DB { -struct ICgroupsReader -{ - virtual ~ICgroupsReader() = default; - - virtual uint64_t readMemoryUsage() = 0; -}; - -std::shared_ptr createCgroupsReader(); - /// Periodically reads the the maximum memory available to the process (which can change due to cgroups settings). /// You can specify a callback to react on changes. The callback typically reloads the configuration, i.e. Server /// or Keeper configuration file. This reloads settings 'max_server_memory_usage' (Server) and 'max_memory_usage_soft_limit' @@ -54,8 +44,6 @@ private: void runThread(); - std::shared_ptr cgroups_reader; - std::mutex thread_mutex; std::condition_variable cond; ThreadFromGlobalPool thread; diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 07d6ba98745..0ffae89ffa6 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -508,13 +508,13 @@ void MemoryTracker::reset() } -void MemoryTracker::updateValues(Int64 rss_, Int64 allocated_, bool force_update) +void MemoryTracker::updateRSS(Int64 rss_) { total_memory_tracker.rss.store(rss_, std::memory_order_relaxed); +} - if (likely(!force_update && total_memory_tracker.amount.load(std::memory_order_relaxed) >= 0)) - return; - +void MemoryTracker::updateAllocated(Int64 allocated_) +{ Int64 new_amount = allocated_; LOG_INFO( getLogger("MemoryTracker"), @@ -531,7 +531,6 @@ void MemoryTracker::updateValues(Int64 rss_, Int64 allocated_, bool force_update total_memory_tracker.updatePeak(new_amount, log_memory_usage); } - void MemoryTracker::setSoftLimit(Int64 value) { soft_limit.store(value, std::memory_order_relaxed); diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index 4913be9781f..d2db8489f19 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -240,7 +240,8 @@ public: void reset(); /// update values based on external information (e.g. jemalloc's stat) - static void updateValues(Int64 rss_, Int64 allocated_, bool force_update); + static void updateRSS(Int64 rss_); + static void updateAllocated(Int64 allocated_); /// Prints info about peak memory consumption into log. void logPeakMemoryUsage(); diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index 2b945a30d3d..42e797a80d6 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -1,11 +1,21 @@ #include +#include +#include +#include +#include #include #include #include #include #include +#include +#include +#include + +namespace fs = std::filesystem; + namespace ProfileEvents { extern const Event MemoryAllocatorPurge; @@ -17,14 +27,227 @@ namespace ProfileEvents namespace DB { -#if USE_JEMALLOC -#define STRINGIFY_HELPER(x) #x -#define STRINGIFY(x) STRINGIFY_HELPER(x) - -MemoryWorker::MemoryWorker(uint64_t period_ms_) - : period_ms(period_ms_) +namespace ErrorCodes { - LOG_INFO(getLogger("MemoryWorker"), "Starting background memory thread with period of {}ms", period_ms.count()); + extern const int FILE_DOESNT_EXIST; + extern const int INCORRECT_DATA; +} + +#if defined(OS_LINUX) +struct ICgroupsReader +{ + virtual ~ICgroupsReader() = default; + + virtual uint64_t readMemoryUsage() = 0; +}; + +namespace +{ + +/// Format is +/// kernel 5 +/// rss 15 +/// [...] +uint64_t readMetricFromStatFile(ReadBufferFromFile & buf, const std::string & key) +{ + while (!buf.eof()) + { + std::string current_key; + readStringUntilWhitespace(current_key, buf); + if (current_key != key) + { + std::string dummy; + readStringUntilNewlineInto(dummy, buf); + buf.ignore(); + continue; + } + + assertChar(' ', buf); + uint64_t value = 0; + readIntText(value, buf); + return value; + } + + LOG_ERROR(getLogger("CgroupsReader"), "Cannot find '{}' in '{}'", key, buf.getFileName()); + return 0; +} + +struct CgroupsV1Reader : ICgroupsReader +{ + explicit CgroupsV1Reader(const fs::path & stat_file_dir) : buf(stat_file_dir / "memory.stat") { } + + uint64_t readMemoryUsage() override + { + std::lock_guard lock(mutex); + buf.rewind(); + return readMetricFromStatFile(buf, "rss"); + } + +private: + std::mutex mutex; + ReadBufferFromFile buf TSA_GUARDED_BY(mutex); +}; + +struct CgroupsV2Reader : ICgroupsReader +{ + explicit CgroupsV2Reader(const fs::path & stat_file_dir) : stat_buf(stat_file_dir / "memory.stat") { } + + uint64_t readMemoryUsage() override + { + std::lock_guard lock(mutex); + stat_buf.rewind(); + return readMetricFromStatFile(stat_buf, "anon"); + } + +private: + std::mutex mutex; + ReadBufferFromFile stat_buf TSA_GUARDED_BY(mutex); +}; + +/// Caveats: +/// - All of the logic in this file assumes that the current process is the only process in the +/// containing cgroup (or more precisely: the only process with significant memory consumption). +/// If this is not the case, then other processe's memory consumption may affect the internal +/// memory tracker ... +/// - Cgroups v1 and v2 allow nested cgroup hierarchies. As v1 is deprecated for over half a +/// decade and will go away at some point, hierarchical detection is only implemented for v2. +/// - I did not test what happens if a host has v1 and v2 simultaneously enabled. I believe such +/// systems existed only for a short transition period. + +std::optional getCgroupsV2Path() +{ + if (!cgroupsV2Enabled()) + return {}; + + if (!cgroupsV2MemoryControllerEnabled()) + return {}; + + fs::path current_cgroup = cgroupV2PathOfProcess(); + if (current_cgroup.empty()) + return {}; + + /// Return the bottom-most nested current memory file. If there is no such file at the current + /// level, try again at the parent level as memory settings are inherited. + while (current_cgroup != default_cgroups_mount.parent_path()) + { + const auto current_path = current_cgroup / "memory.current"; + const auto stat_path = current_cgroup / "memory.stat"; + if (fs::exists(current_path) && fs::exists(stat_path)) + return {current_cgroup}; + current_cgroup = current_cgroup.parent_path(); + } + return {}; +} + +std::optional getCgroupsV1Path() +{ + auto path = default_cgroups_mount / "memory/memory.stat"; + if (!fs::exists(path)) + return {}; + return {default_cgroups_mount / "memory"}; +} + +enum class CgroupsVersion : uint8_t +{ + V1, + V2 +}; + +std::pair getCgroupsPath() +{ + auto v2_path = getCgroupsV2Path(); + if (v2_path.has_value()) + return {*v2_path, CgroupsVersion::V2}; + + auto v1_path = getCgroupsV1Path(); + if (v1_path.has_value()) + return {*v1_path, CgroupsVersion::V1}; + + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Cannot find cgroups v1 or v2 current memory file"); +} + +std::shared_ptr createCgroupsReader() +{ + const auto [cgroup_path, version] = getCgroupsPath(); + LOG_INFO( + getLogger("CgroupsReader"), + "Will create cgroup reader from '{}' (cgroups version: {})", + cgroup_path, + (version == CgroupsVersion::V1) ? "v1" : "v2"); + + if (version == CgroupsVersion::V2) + return std::make_shared(cgroup_path); + else + { + chassert(version == CgroupsVersion::V1); + return std::make_shared(cgroup_path); + } + +} +#endif + +constexpr uint64_t cgroups_memory_usage_tick_ms{50}; +constexpr uint64_t jemalloc_memory_usage_tick_ms{100}; + +std::string_view sourceToString(MemoryWorker::MemoryUsageSource source) +{ + switch (source) + { + case MemoryWorker::MemoryUsageSource::Cgroups: return "Cgroups"; + case MemoryWorker::MemoryUsageSource::Jemalloc: return "Jemalloc"; + case MemoryWorker::MemoryUsageSource::None: return "None"; + } +} + +} + +/// We try to pick the best possible supported source for reading memory usage. +/// Supported sources in order of priority +/// - reading from cgroups' pseudo-files (fastest and most accurate) +/// - reading jemalloc's resident stat (doesn't take into account allocations that didn't use jemalloc) +/// Also, different tick rates are used because not all options are equally fast +MemoryWorker::MemoryWorker(uint64_t period_ms_) + : log(getLogger("MemoryWorker")) + , period_ms(period_ms_) +{ +#if defined(OS_LINUX) + try + { + cgroups_reader = createCgroupsReader(); + source = MemoryUsageSource::Cgroups; + if (period_ms == 0) + period_ms = cgroups_memory_usage_tick_ms; + + return; + } + catch (...) + { + tryLogCurrentException(log, "Cannot use cgroups reader"); + } +#endif + +#if USE_JEMALLOC + source = MemoryUsageSource::Jemalloc; + if (period_ms == 0) + period_ms = jemalloc_memory_usage_tick_ms; +#endif +} + +MemoryWorker::MemoryUsageSource MemoryWorker::getSource() +{ + return source; +} + +void MemoryWorker::start() +{ + if (source == MemoryUsageSource::None) + return; + + LOG_INFO( + getLogger("MemoryWorker"), + "Starting background memory thread with period of {}ms, using {} as source", + period_ms, + sourceToString(source)); background_thread = ThreadFromGlobalPool([this] { backgroundThread(); }); } @@ -40,29 +263,39 @@ MemoryWorker::~MemoryWorker() background_thread.join(); } +uint64_t MemoryWorker::getMemoryUsage() +{ + switch (source) + { + case MemoryUsageSource::Cgroups: + return cgroups_reader->readMemoryUsage(); + case MemoryUsageSource::Jemalloc: + return resident_mib.getValue(); + case MemoryUsageSource::None: + throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Trying to fetch memory usage while no memory source can be used"); + } +} + void MemoryWorker::backgroundThread() { - JemallocMibCache epoch_mib("epoch"); - JemallocMibCache resident_mib("stats.resident"); - JemallocMibCache active_mib("stats.active"); - JemallocMibCache allocated_mib("stats.allocated"); - JemallocMibCache purge_mib("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge"); + std::chrono::milliseconds chrono_period_ms{period_ms}; bool first_run = true; std::unique_lock lock(mutex); while (true) { - cv.wait_for(lock, period_ms, [this] { return shutdown; }); + cv.wait_for(lock, chrono_period_ms, [this] { return shutdown; }); if (shutdown) return; Stopwatch total_watch; - epoch_mib.setValue(0); - Int64 resident = resident_mib.getValue(); - /// force update the allocated stat from jemalloc for the first run to cover the allocations we missed - /// during initialization - MemoryTracker::updateValues(resident, allocated_mib.getValue(), first_run); + if (source == MemoryUsageSource::Jemalloc) + epoch_mib.setValue(0); + Int64 resident = getMemoryUsage(); + MemoryTracker::updateRSS(resident); + +#if USE_JEMALLOC if (resident > total_memory_tracker.getHardLimit()) { Stopwatch purge_watch; @@ -71,12 +304,19 @@ void MemoryWorker::backgroundThread() ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, purge_watch.elapsedMicroseconds()); } + if (unlikely(first_run || total_memory_tracker.get() < 0)) + { + if (source != MemoryUsageSource::Jemalloc) + epoch_mib.setValue(0); + + MemoryTracker::updateAllocated(allocated_mib.getValue()); + } +#endif + ProfileEvents::increment(ProfileEvents::MemoryWorkerRun); ProfileEvents::increment(ProfileEvents::MemoryWorkerRunElapsedMicroseconds, total_watch.elapsedMicroseconds()); - first_run = false; } } -#endif } diff --git a/src/Common/MemoryWorker.h b/src/Common/MemoryWorker.h index 6c0a578aa61..6fde93d63ad 100644 --- a/src/Common/MemoryWorker.h +++ b/src/Common/MemoryWorker.h @@ -1,13 +1,14 @@ #pragma once +#include #include - -#include "config.h" +#include namespace DB { -#if USE_JEMALLOC +struct ICgroupsReader; + /// Correct MemoryTracker based on stats.resident read from jemalloc. /// This requires jemalloc built with --enable-stats which we use. /// The worker spawns a background thread which moves the jemalloc epoch (updates internal stats), @@ -19,8 +20,21 @@ class MemoryWorker public: explicit MemoryWorker(uint64_t period_ms_); + enum class MemoryUsageSource : uint8_t + { + None, + Cgroups, + Jemalloc + }; + + MemoryUsageSource getSource(); + + void start(); + ~MemoryWorker(); private: + uint64_t getMemoryUsage(); + void backgroundThread(); ThreadFromGlobalPool background_thread; @@ -29,14 +43,27 @@ private: std::condition_variable cv; bool shutdown = false; - std::chrono::milliseconds period_ms; -}; -#else -class MemoryWorker -{ -public: - explicit MemoryWorker(uint64_t /*period_ms_*/) {} -}; + LoggerPtr log; + + uint64_t period_ms; + + MemoryUsageSource source{MemoryUsageSource::None}; + +#if defined(OS_LINUX) + std::shared_ptr cgroups_reader; #endif +#if USE_JEMALLOC + JemallocMibCache epoch_mib{"epoch"}; + JemallocMibCache resident_mib{"stats.resident"}; + JemallocMibCache allocated_mib{"stats.allocated"}; + +#define STRINGIFY_HELPER(x) #x +#define STRINGIFY(x) STRINGIFY_HELPER(x) + JemallocMibCache purge_mib{"arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge"}; +#undef STRINGIFY +#undef STRINGIFY_HELPER +#endif +}; + } diff --git a/src/Coordination/KeeperAsynchronousMetrics.cpp b/src/Coordination/KeeperAsynchronousMetrics.cpp index 86166ffe31b..157858f3c44 100644 --- a/src/Coordination/KeeperAsynchronousMetrics.cpp +++ b/src/Coordination/KeeperAsynchronousMetrics.cpp @@ -114,8 +114,13 @@ void updateKeeperInformation(KeeperDispatcher & keeper_dispatcher, AsynchronousM } KeeperAsynchronousMetrics::KeeperAsynchronousMetrics( - ContextPtr context_, unsigned update_period_seconds, const ProtocolServerMetricsFunc & protocol_server_metrics_func_) - : AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_), context(std::move(context_)) + ContextPtr context_, + unsigned update_period_seconds, + const ProtocolServerMetricsFunc & protocol_server_metrics_func_, + bool update_jemalloc_epoch_, + bool update_rss_) + : AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_, update_jemalloc_epoch_, update_rss_) + , context(std::move(context_)) { } diff --git a/src/Coordination/KeeperAsynchronousMetrics.h b/src/Coordination/KeeperAsynchronousMetrics.h index ec0e60cbb6e..a2ab7cab756 100644 --- a/src/Coordination/KeeperAsynchronousMetrics.h +++ b/src/Coordination/KeeperAsynchronousMetrics.h @@ -13,9 +13,13 @@ class KeeperAsynchronousMetrics : public AsynchronousMetrics { public: KeeperAsynchronousMetrics( - ContextPtr context_, unsigned update_period_seconds, const ProtocolServerMetricsFunc & protocol_server_metrics_func_); - ~KeeperAsynchronousMetrics() override; + ContextPtr context_, + unsigned update_period_seconds, + const ProtocolServerMetricsFunc & protocol_server_metrics_func_, + bool update_jemalloc_epoch_, + bool update_rss_); + ~KeeperAsynchronousMetrics() override; private: ContextPtr context; diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index aaea0388239..ea5a3f19638 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -157,7 +157,7 @@ namespace DB M(Bool, prepare_system_log_tables_on_startup, false, "If true, ClickHouse creates all configured `system.*_log` tables before the startup. It can be helpful if some startup scripts depend on these tables.", 0) \ M(Double, gwp_asan_force_sample_probability, 0.0003, "Probability that an allocation from specific places will be sampled by GWP Asan (i.e. PODArray allocations)", 0) \ M(UInt64, config_reload_interval_ms, 2000, "How often clickhouse will reload config and check for new changes", 0) \ - M(UInt64, memory_worker_period_ms, 100, "Period of background memory worker which corrects memory tracker memory usages and cleans up unused pages during higher memory usage.", 0) \ + M(UInt64, memory_worker_period_ms, 0, "Tick period of background memory worker which corrects memory tracker memory usages and cleans up unused pages during higher memory usage. If set to 0, default value will be used depending on the memory usage source", 0) \ /// If you add a setting which can be updated at runtime, please update 'changeable_settings' map in StorageSystemServerSettings.cpp diff --git a/src/Interpreters/ServerAsynchronousMetrics.cpp b/src/Interpreters/ServerAsynchronousMetrics.cpp index 872a9f864df..079029695c9 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.cpp +++ b/src/Interpreters/ServerAsynchronousMetrics.cpp @@ -55,9 +55,11 @@ ServerAsynchronousMetrics::ServerAsynchronousMetrics( ContextPtr global_context_, unsigned update_period_seconds, unsigned heavy_metrics_update_period_seconds, - const ProtocolServerMetricsFunc & protocol_server_metrics_func_) + const ProtocolServerMetricsFunc & protocol_server_metrics_func_, + bool update_jemalloc_epoch_, + bool update_rss_) : WithContext(global_context_) - , AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_) + , AsynchronousMetrics(update_period_seconds, protocol_server_metrics_func_, update_jemalloc_epoch_, update_rss_) , heavy_metric_update_period(heavy_metrics_update_period_seconds) { /// sanity check diff --git a/src/Interpreters/ServerAsynchronousMetrics.h b/src/Interpreters/ServerAsynchronousMetrics.h index e3c83dc748e..5fab419a32b 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.h +++ b/src/Interpreters/ServerAsynchronousMetrics.h @@ -14,7 +14,10 @@ public: ContextPtr global_context_, unsigned update_period_seconds, unsigned heavy_metrics_update_period_seconds, - const ProtocolServerMetricsFunc & protocol_server_metrics_func_); + const ProtocolServerMetricsFunc & protocol_server_metrics_func_, + bool update_jemalloc_epoch_, + bool update_rss_); + ~ServerAsynchronousMetrics() override; private: From 5a1b96ac8453f73de5e891e3a9f235fd96270b50 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 23 Jul 2024 10:52:14 +0200 Subject: [PATCH 086/215] Style fix --- src/Common/CgroupsMemoryUsageObserver.cpp | 6 ------ src/Common/MemoryWorker.cpp | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/Common/CgroupsMemoryUsageObserver.cpp b/src/Common/CgroupsMemoryUsageObserver.cpp index afeac1808b2..16d5d1cccde 100644 --- a/src/Common/CgroupsMemoryUsageObserver.cpp +++ b/src/Common/CgroupsMemoryUsageObserver.cpp @@ -19,12 +19,6 @@ using namespace DB; namespace DB { -namespace ErrorCodes -{ -extern const int FILE_DOESNT_EXIST; -extern const int INCORRECT_DATA; -} - CgroupsMemoryUsageObserver::CgroupsMemoryUsageObserver(std::chrono::seconds wait_time_) : log(getLogger("CgroupsMemoryUsageObserver")), wait_time(wait_time_) {} diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index 42e797a80d6..ddc3fd783f4 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -30,7 +30,7 @@ namespace DB namespace ErrorCodes { extern const int FILE_DOESNT_EXIST; - extern const int INCORRECT_DATA; + extern const int LOGICAL_ERROR; } #if defined(OS_LINUX) From 5b51a35e015336227dcc5b25445fefbc3da3059c Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 23 Jul 2024 11:31:34 +0200 Subject: [PATCH 087/215] Add unused variables --- src/Common/AsynchronousMetrics.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/AsynchronousMetrics.h b/src/Common/AsynchronousMetrics.h index eb6ede7a558..fedba4e55be 100644 --- a/src/Common/AsynchronousMetrics.h +++ b/src/Common/AsynchronousMetrics.h @@ -112,8 +112,8 @@ private: MemoryStatisticsOS memory_stat TSA_GUARDED_BY(data_mutex); #endif - const bool update_jemalloc_epoch; - const bool update_rss; + [[maybe_unused]] const bool update_jemalloc_epoch; + [[maybe_unused]] const bool update_rss; #if defined(OS_LINUX) std::optional meminfo TSA_GUARDED_BY(data_mutex); From ade79cfd7a8465bf6e332f3b2ca6143ba652251a Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 23 Jul 2024 11:50:56 +0200 Subject: [PATCH 088/215] More fixes --- src/Common/MemoryWorker.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index ddc3fd783f4..8169bc7d177 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -186,9 +186,6 @@ std::shared_ptr createCgroupsReader() } #endif -constexpr uint64_t cgroups_memory_usage_tick_ms{50}; -constexpr uint64_t jemalloc_memory_usage_tick_ms{100}; - std::string_view sourceToString(MemoryWorker::MemoryUsageSource source) { switch (source) @@ -213,6 +210,8 @@ MemoryWorker::MemoryWorker(uint64_t period_ms_) #if defined(OS_LINUX) try { + static constexpr uint64_t cgroups_memory_usage_tick_ms{50}; + cgroups_reader = createCgroupsReader(); source = MemoryUsageSource::Cgroups; if (period_ms == 0) @@ -227,6 +226,8 @@ MemoryWorker::MemoryWorker(uint64_t period_ms_) #endif #if USE_JEMALLOC + static constexpr uint64_t jemalloc_memory_usage_tick_ms{100}; + source = MemoryUsageSource::Jemalloc; if (period_ms == 0) period_ms = jemalloc_memory_usage_tick_ms; @@ -270,7 +271,11 @@ uint64_t MemoryWorker::getMemoryUsage() case MemoryUsageSource::Cgroups: return cgroups_reader->readMemoryUsage(); case MemoryUsageSource::Jemalloc: +#if USE_JEMALLOC return resident_mib.getValue(); +#else + return 0; +#endif case MemoryUsageSource::None: throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Trying to fetch memory usage while no memory source can be used"); } @@ -279,7 +284,7 @@ uint64_t MemoryWorker::getMemoryUsage() void MemoryWorker::backgroundThread() { std::chrono::milliseconds chrono_period_ms{period_ms}; - bool first_run = true; + [[maybe_unused]] bool first_run = true; std::unique_lock lock(mutex); while (true) { @@ -289,8 +294,10 @@ void MemoryWorker::backgroundThread() Stopwatch total_watch; +#if USE_JEMALLOC if (source == MemoryUsageSource::Jemalloc) epoch_mib.setValue(0); +#endif Int64 resident = getMemoryUsage(); MemoryTracker::updateRSS(resident); From 842b51c782f0cc5cf9bd7b695c466933310a7e6c Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 23 Jul 2024 13:30:40 +0000 Subject: [PATCH 089/215] Try assigning replica numbers consistently independent of initiator --- .../ClusterProxy/executeQuery.cpp | 67 +++++++++++- .../QueryPlan/ParallelReplicasLocalPlan.cpp | 7 +- .../QueryPlan/ParallelReplicasLocalPlan.h | 3 +- .../QueryPlan/ReadFromMergeTree.cpp | 8 +- src/Processors/QueryPlan/ReadFromMergeTree.h | 3 +- src/Processors/QueryPlan/ReadFromRemote.cpp | 102 ++++-------------- src/Processors/QueryPlan/ReadFromRemote.h | 6 +- .../ParallelReplicasReadingCoordinator.cpp | 3 +- 8 files changed, 101 insertions(+), 98 deletions(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index a8cf3022a61..80f6d7a864b 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -40,6 +40,7 @@ namespace ErrorCodes extern const int TOO_LARGE_DISTRIBUTED_DEPTH; extern const int LOGICAL_ERROR; extern const int UNEXPECTED_CLUSTER; + extern const int INCONSISTENT_CLUSTER_DEFINITION; } namespace ClusterProxy @@ -519,22 +520,75 @@ void executeQueryWithParallelReplicas( "`cluster_for_parallel_replicas` setting refers to cluster with several shards. Expected a cluster with one shard"); } - const auto replica_count = std::min(settings.max_parallel_replicas.value, new_cluster->getShardsInfo().begin()->getAllNodeCount()); + const auto & shard = new_cluster->getShardsInfo().at(0); + size_t max_replicas_to_use = settings.max_parallel_replicas; + if (max_replicas_to_use > shard.getAllNodeCount()) + { + LOG_INFO( + getLogger("ReadFromParallelRemoteReplicasStep"), + "The number of replicas requested ({}) is bigger than the real number available in the cluster ({}). " + "Will use the latter number to execute the query.", + settings.max_parallel_replicas, + shard.getAllNodeCount()); + max_replicas_to_use = shard.getAllNodeCount(); + } - auto coordinator = std::make_shared(replica_count, settings.parallel_replicas_mark_segment_size); + auto coordinator = std::make_shared(max_replicas_to_use, settings.parallel_replicas_mark_segment_size); auto external_tables = new_context->getExternalTables(); + std::vector shuffled_pool; + if (max_replicas_to_use < shard.getAllNodeCount()) + { + shuffled_pool = shard.pool->getShuffledPools(settings); + shuffled_pool.resize(max_replicas_to_use); + } + else + { + /// if all replicas in cluster are used for query execution + /// try to preserve replicas order as in cluster definition + /// it's important for data locality during query execution + auto priority_func = [](size_t i) { return Priority{static_cast(i)}; }; + shuffled_pool = shard.pool->getShuffledPools(settings, priority_func); + } + + std::vector pools_to_use; + pools_to_use.reserve(shuffled_pool.size()); + for (auto & pool : shuffled_pool) + pools_to_use.emplace_back(std::move(pool.pool)); + /// do not build local plan for distributed queries for now (address it later) if (settings.allow_experimental_analyzer && settings.parallel_replicas_local_plan && !shard_num) { + /// find local replica index in pool, to assign it as replica number + std::optional local_replica_number; + for (size_t i = 0, s = pools_to_use.size(); i < s; ++i) + { + const auto & hostname = pools_to_use[i]->getHost(); + const auto found = std::find_if( + begin(shard.local_addresses), + end(shard.local_addresses), + [&hostname](const Cluster::Address & local_addr) { return hostname == local_addr.host_name; }); + if (found != shard.local_addresses.end()) + { + local_replica_number = i; + break; + } + } + if (!local_replica_number) + throw Exception( + ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION, + "Local replica is not found in '{}' cluster definition, see 'cluster_for_parallel_replicas' setting", + new_cluster->getName()); + auto [local_plan, with_parallel_replicas] = createLocalPlanForParallelReplicas( query_ast, header, new_context, processed_stage, coordinator, - std::move(analyzed_read_from_merge_tree)); + std::move(analyzed_read_from_merge_tree), + local_replica_number.value()); if (!with_parallel_replicas) { @@ -542,6 +596,8 @@ void executeQueryWithParallelReplicas( return; } + LOG_DEBUG(logger, "Local replica got replica number {}", local_replica_number.value()); + auto read_from_remote = std::make_unique( query_ast, new_cluster, @@ -555,7 +611,8 @@ void executeQueryWithParallelReplicas( std::move(external_tables), getLogger("ReadFromParallelRemoteReplicasStep"), std::move(storage_limits), - /*exclude_local_replica*/ true); + std::move(pools_to_use), + local_replica_number); auto remote_plan = std::make_unique(); remote_plan->addStep(std::move(read_from_remote)); @@ -587,7 +644,7 @@ void executeQueryWithParallelReplicas( std::move(external_tables), getLogger("ReadFromParallelRemoteReplicasStep"), std::move(storage_limits), - /*exclude_local_replica*/ false); + std::move(pools_to_use)); query_plan.addStep(std::move(read_from_remote)); } diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp index e8ff0f417dc..050044edd3a 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp @@ -27,7 +27,8 @@ std::pair, bool> createLocalPlanForParallelReplicas( ContextPtr context, QueryProcessingStage::Enum processed_stage, ParallelReplicasReadingCoordinatorPtr coordinator, - QueryPlanStepPtr analyzed_read_from_merge_tree) + QueryPlanStepPtr analyzed_read_from_merge_tree, + size_t replica_number) { checkStackSize(); @@ -84,8 +85,8 @@ std::pair, bool> createLocalPlanForParallelReplicas( MergeTreeReadTaskCallback read_task_cb = [coordinator](ParallelReadRequest req) -> std::optional { return coordinator->handleRequest(std::move(req)); }; - auto read_from_merge_tree_parallel_replicas - = reading->createLocalParallelReplicasReadingStep(analyzed_result_ptr, std::move(all_ranges_cb), std::move(read_task_cb)); + auto read_from_merge_tree_parallel_replicas = reading->createLocalParallelReplicasReadingStep( + analyzed_result_ptr, std::move(all_ranges_cb), std::move(read_task_cb), replica_number); node->step = std::move(read_from_merge_tree_parallel_replicas); addConvertingActions(*query_plan, header, /*has_missing_objects=*/false); diff --git a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.h b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.h index 2a49be6347a..ab0bbeaeeff 100644 --- a/src/Processors/QueryPlan/ParallelReplicasLocalPlan.h +++ b/src/Processors/QueryPlan/ParallelReplicasLocalPlan.h @@ -14,5 +14,6 @@ std::pair, bool> createLocalPlanForParallelReplicas( ContextPtr context, QueryProcessingStage::Enum processed_stage, ParallelReplicasReadingCoordinatorPtr coordinator, - QueryPlanStepPtr read_from_merge_tree); + QueryPlanStepPtr read_from_merge_tree, + size_t replica_number); } diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 5922550eaf3..aea9d02fe01 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -348,9 +348,9 @@ ReadFromMergeTree::ReadFromMergeTree( std::unique_ptr ReadFromMergeTree::createLocalParallelReplicasReadingStep( AnalysisResultPtr analyzed_result_ptr_, MergeTreeAllRangesCallback all_ranges_callback_, - MergeTreeReadTaskCallback read_task_callback_) + MergeTreeReadTaskCallback read_task_callback_, + size_t replica_number) { - const auto number_of_local_replica = 0; const bool enable_parallel_reading = true; return std::make_unique( prepared_parts, @@ -364,11 +364,11 @@ std::unique_ptr ReadFromMergeTree::createLocalParallelReplica requested_num_streams, max_block_numbers_to_read, log, - analyzed_result_ptr_, + std::move(analyzed_result_ptr_), enable_parallel_reading, all_ranges_callback_, read_task_callback_, - number_of_local_replica); + replica_number); } Pipe ReadFromMergeTree::readFromPoolParallelReplicas(RangesInDataParts parts_with_range, Names required_columns, PoolSettings pool_settings) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 307b605c01c..69e20a7864d 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -129,7 +129,8 @@ public: std::unique_ptr createLocalParallelReplicasReadingStep( AnalysisResultPtr analyzed_result_ptr_, MergeTreeAllRangesCallback all_ranges_callback_, - MergeTreeReadTaskCallback read_task_callback_); + MergeTreeReadTaskCallback read_task_callback_, + size_t replica_number); static constexpr auto name = "ReadFromMergeTree"; String getName() const override { return name; } diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index 46d4aa29e70..3df46eb1987 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -371,7 +371,8 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep( Tables external_tables_, LoggerPtr log_, std::shared_ptr storage_limits_, - bool exclude_local_replica_) + std::vector pools_to_use_, + std::optional exclude_pool_index_) : ISourceStep(DataStream{.header = std::move(header_)}) , cluster(cluster_) , query_ast(query_ast_) @@ -384,24 +385,20 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep( , external_tables{external_tables_} , storage_limits(std::move(storage_limits_)) , log(log_) - , exclude_local_replica(exclude_local_replica_) + , pools_to_use(std::move(pools_to_use_)) + , exclude_pool_index(exclude_pool_index_) { chassert(cluster->getShardCount() == 1); std::vector replicas; - replicas.reserve(cluster->getShardsAddresses().front().size()); + replicas.reserve(pools_to_use.size()); - for (const auto & addr : cluster->getShardsAddresses().front()) + for (size_t i = 0, l = pools_to_use.size(); i < l; ++i) { - if (exclude_local_replica && addr.is_local) + if (exclude_pool_index.has_value() && i == exclude_pool_index) continue; - /// replace hostname with replica name if the hostname started with replica namespace, - /// it makes description shorter and more readable - if (!addr.database_replica_name.empty() && addr.host_name.starts_with(addr.database_replica_name)) - replicas.push_back(fmt::format("{}", addr.database_replica_name)); - else - replicas.push_back(fmt::format("{}", addr.host_name)); + replicas.push_back(pools_to_use[i]->getAddress()); } auto description = fmt::format("Query: {} Replicas: {}", formattedAST(query_ast), fmt::join(replicas, ", ")); @@ -421,86 +418,29 @@ void ReadFromParallelRemoteReplicasStep::enforceAggregationInOrder() void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { Pipes pipes; - const Settings & current_settings = context->getSettingsRef(); - auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(current_settings); - - const auto & shard = cluster->getShardsInfo().at(0); - size_t max_replicas_to_use = current_settings.max_parallel_replicas; - if (max_replicas_to_use > shard.getAllNodeCount()) - { - LOG_INFO( - getLogger("ReadFromParallelRemoteReplicasStep"), - "The number of replicas requested ({}) is bigger than the real number available in the cluster ({}). " - "Will use the latter number to execute the query.", - current_settings.max_parallel_replicas, - shard.getAllNodeCount()); - max_replicas_to_use = shard.getAllNodeCount(); - } - - std::vector shuffled_pool; - if (max_replicas_to_use < shard.getAllNodeCount()) - { - shuffled_pool = shard.pool->getShuffledPools(current_settings); - } - else - { - /// try to preserve replicas order if all replicas in cluster are used for query execution - /// it's important for data locality during query execution - auto priority_func = [](size_t i) { return Priority{static_cast(i)}; }; - shuffled_pool = shard.pool->getShuffledPools(current_settings, priority_func); - } - - std::vector pools_to_use; - pools_to_use.reserve(shuffled_pool.size()); - for (const auto & pool : shuffled_pool) - { - if (exclude_local_replica) - { - const auto & hostname = pool.pool->getHost(); - auto it = std::find_if( - begin(shard.local_addresses), - end(shard.local_addresses), - [&hostname](const Cluster::Address & local_addr) { return hostname == local_addr.host_name; }); - if (it == shard.local_addresses.end()) - pools_to_use.push_back(pool.pool); - } - else - { - pools_to_use.push_back(pool.pool); - } - } - - pools_to_use.resize(std::min(pools_to_use.size(), max_replicas_to_use)); - // if local plan is used for local replica, we should exclude one remote replica - if (exclude_local_replica && !pools_to_use.empty()) - pools_to_use.resize(max_replicas_to_use - 1); - - LOG_DEBUG( - getLogger("ReadFromParallelRemoteReplicasStep"), - "Number of pools to use is {}. Originally {}", - pools_to_use.size(), - shuffled_pool.size()); - - if (pools_to_use.empty()) - return; std::vector addresses; addresses.reserve(pools_to_use.size()); - for (const auto & pool : pools_to_use) - addresses.emplace_back(pool->getAddress()); + for (size_t i = 0, l = pools_to_use.size(); i < l; ++i) + { + if (exclude_pool_index.has_value() && i == exclude_pool_index) + continue; + + addresses.emplace_back(pools_to_use[i]->getAddress()); + } LOG_DEBUG(getLogger("ReadFromParallelRemoteReplicasStep"), "Addresses to use: {}", fmt::join(addresses, ", ")); - /// when using local plan for local replica, 0 is assigned to local replica as replica num, - in this case, starting from 1 here - size_t replica_num = (exclude_local_replica ? 1 : 0); - for (const auto & pool : pools_to_use) + for (size_t i = 0, l = pools_to_use.size(); i < l; ++i) { + if (exclude_pool_index.has_value() && i == exclude_pool_index) + continue; + IConnections::ReplicaInfo replica_info{ /// we should use this number specifically because efficiency of data distribution by consistent hash depends on it. - .number_of_current_replica = replica_num, + .number_of_current_replica = i, }; - ++replica_num; - addPipeForSingeReplica(pipes, pool, replica_info); + addPipeForSingeReplica(pipes, pools_to_use[i], replica_info); } auto pipe = Pipe::unitePipes(std::move(pipes)); diff --git a/src/Processors/QueryPlan/ReadFromRemote.h b/src/Processors/QueryPlan/ReadFromRemote.h index 442da098a17..74389c8f9eb 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.h +++ b/src/Processors/QueryPlan/ReadFromRemote.h @@ -79,7 +79,8 @@ public: Tables external_tables_, LoggerPtr log_, std::shared_ptr storage_limits_, - bool exclude_local_replica = false); + std::vector pools_to_use, + std::optional exclude_pool_index_ = std::nullopt); String getName() const override { return "ReadFromRemoteParallelReplicas"; } @@ -102,7 +103,8 @@ private: Tables external_tables; std::shared_ptr storage_limits; LoggerPtr log; - bool exclude_local_replica; + std::vector pools_to_use; + std::optional exclude_pool_index; }; } diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp index a1b36fe1299..a1264eabdea 100644 --- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp +++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp @@ -185,7 +185,8 @@ public: void handleInitialAllRangesAnnouncement(InitialAllRangesAnnouncement announcement) { if (++sent_initial_requests > replicas_count) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Initiator received more initial requests than there are replicas"); + throw Exception( + ErrorCodes::LOGICAL_ERROR, "Initiator received more initial requests than there are replicas: replica_num={}", announcement.replica_num); doHandleInitialAllRangesAnnouncement(std::move(announcement)); } From 04d80ec2763a0677f32e9f6190cf2bcda0ebf8b7 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 23 Jul 2024 16:13:07 +0200 Subject: [PATCH 090/215] Fix non-linux build --- src/Common/MemoryWorker.cpp | 4 ++-- src/Common/MemoryWorker.h | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index 8169bc7d177..c576772d303 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -33,7 +33,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -#if defined(OS_LINUX) struct ICgroupsReader { virtual ~ICgroupsReader() = default; @@ -44,6 +43,7 @@ struct ICgroupsReader namespace { +#if defined(OS_LINUX) /// Format is /// kernel 5 /// rss 15 @@ -269,7 +269,7 @@ uint64_t MemoryWorker::getMemoryUsage() switch (source) { case MemoryUsageSource::Cgroups: - return cgroups_reader->readMemoryUsage(); + return cgroups_reader != nullptr ? cgroups_reader->readMemoryUsage() : 0; case MemoryUsageSource::Jemalloc: #if USE_JEMALLOC return resident_mib.getValue(); diff --git a/src/Common/MemoryWorker.h b/src/Common/MemoryWorker.h index 6fde93d63ad..b1b0495bf14 100644 --- a/src/Common/MemoryWorker.h +++ b/src/Common/MemoryWorker.h @@ -49,9 +49,7 @@ private: MemoryUsageSource source{MemoryUsageSource::None}; -#if defined(OS_LINUX) std::shared_ptr cgroups_reader; -#endif #if USE_JEMALLOC JemallocMibCache epoch_mib{"epoch"}; From f5630f86e4853c06db1267628e4a8c0e80d62889 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 23 Jul 2024 15:43:25 +0000 Subject: [PATCH 091/215] Fix: resize pools_to_use correctly (keep local replica in it) --- .../ClusterProxy/executeQuery.cpp | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 80f6d7a864b..44474a11b50 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -540,14 +540,15 @@ void executeQueryWithParallelReplicas( std::vector shuffled_pool; if (max_replicas_to_use < shard.getAllNodeCount()) { + // will be shuffled according to `load_balancing` setting shuffled_pool = shard.pool->getShuffledPools(settings); - shuffled_pool.resize(max_replicas_to_use); } else { - /// if all replicas in cluster are used for query execution - /// try to preserve replicas order as in cluster definition - /// it's important for data locality during query execution + /// If all replicas in cluster are used for query execution, + /// try to preserve replicas order as in cluster definition. + /// It's important for data locality during query execution + /// independently of the query initiator auto priority_func = [](size_t i) { return Priority{static_cast(i)}; }; shuffled_pool = shard.pool->getShuffledPools(settings, priority_func); } @@ -560,8 +561,8 @@ void executeQueryWithParallelReplicas( /// do not build local plan for distributed queries for now (address it later) if (settings.allow_experimental_analyzer && settings.parallel_replicas_local_plan && !shard_num) { - /// find local replica index in pool, to assign it as replica number - std::optional local_replica_number; + /// find local replica index in pool + std::optional local_replica_index; for (size_t i = 0, s = pools_to_use.size(); i < s; ++i) { const auto & hostname = pools_to_use[i]->getHost(); @@ -571,16 +572,25 @@ void executeQueryWithParallelReplicas( [&hostname](const Cluster::Address & local_addr) { return hostname == local_addr.host_name; }); if (found != shard.local_addresses.end()) { - local_replica_number = i; + local_replica_index = i; break; } } - if (!local_replica_number) + if (!local_replica_index) throw Exception( ErrorCodes::INCONSISTENT_CLUSTER_DEFINITION, "Local replica is not found in '{}' cluster definition, see 'cluster_for_parallel_replicas' setting", new_cluster->getName()); + // resize the pool but keep local replicas in it (and udpate its index) + chassert(max_replicas_to_use <= pools_to_use.size()); + if (local_replica_index >= max_replicas_to_use) + { + std::swap(pools_to_use[max_replicas_to_use - 1], pools_to_use[local_replica_index.value()]); + local_replica_index = max_replicas_to_use - 1; + } + pools_to_use.resize(max_replicas_to_use); + auto [local_plan, with_parallel_replicas] = createLocalPlanForParallelReplicas( query_ast, header, @@ -588,7 +598,7 @@ void executeQueryWithParallelReplicas( processed_stage, coordinator, std::move(analyzed_read_from_merge_tree), - local_replica_number.value()); + local_replica_index.value()); if (!with_parallel_replicas) { @@ -596,7 +606,7 @@ void executeQueryWithParallelReplicas( return; } - LOG_DEBUG(logger, "Local replica got replica number {}", local_replica_number.value()); + LOG_DEBUG(logger, "Local replica got replica number {}", local_replica_index.value()); auto read_from_remote = std::make_unique( query_ast, @@ -612,7 +622,7 @@ void executeQueryWithParallelReplicas( getLogger("ReadFromParallelRemoteReplicasStep"), std::move(storage_limits), std::move(pools_to_use), - local_replica_number); + local_replica_index); auto remote_plan = std::make_unique(); remote_plan->addStep(std::move(read_from_remote)); @@ -631,6 +641,9 @@ void executeQueryWithParallelReplicas( } else { + chassert(max_replicas_to_use <= pools_to_use.size()); + pools_to_use.resize(max_replicas_to_use); + auto read_from_remote = std::make_unique( query_ast, new_cluster, From 87ad12e5fceec5c412964fbf0a6336a4d3da7662 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 23 Jul 2024 17:16:11 +0000 Subject: [PATCH 092/215] Fix typo --- src/Interpreters/ClusterProxy/executeQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 44474a11b50..444a872f491 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -582,7 +582,7 @@ void executeQueryWithParallelReplicas( "Local replica is not found in '{}' cluster definition, see 'cluster_for_parallel_replicas' setting", new_cluster->getName()); - // resize the pool but keep local replicas in it (and udpate its index) + // resize the pool but keep local replicas in it (and update its index) chassert(max_replicas_to_use <= pools_to_use.size()); if (local_replica_index >= max_replicas_to_use) { From f49412e967ec549d8a49e53ce3bfba27e13994fc Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 23 Jul 2024 17:16:40 +0000 Subject: [PATCH 093/215] Change local replica position in `parallel_replicas` cluster --- tests/config/config.d/clusters.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/config/config.d/clusters.xml b/tests/config/config.d/clusters.xml index af26565d7e6..ff42b5828e3 100644 --- a/tests/config/config.d/clusters.xml +++ b/tests/config/config.d/clusters.xml @@ -115,10 +115,6 @@ false - - 127.0.0.1 - 9000 - 127.0.0.2 9000 @@ -147,6 +143,10 @@ 127.0.0.8 9000 + + 127.0.0.1 + 9000 + 127.0.0.9 9000 From ec5459a60d0be15e7bc100bd257e2b5c65baf594 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 24 Jul 2024 09:15:34 +0200 Subject: [PATCH 094/215] Update allocated with resident if no jemalloc --- src/Common/MemoryWorker.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index c576772d303..75d0e7c32d8 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -310,15 +310,19 @@ void MemoryWorker::backgroundThread() ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurge); ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, purge_watch.elapsedMicroseconds()); } +#endif if (unlikely(first_run || total_memory_tracker.get() < 0)) { +#if USE_JEMALLOC if (source != MemoryUsageSource::Jemalloc) epoch_mib.setValue(0); MemoryTracker::updateAllocated(allocated_mib.getValue()); - } +#elif defined(OS_LINUX) + MemoryTracker::updateAllocated(resident); #endif + } ProfileEvents::increment(ProfileEvents::MemoryWorkerRun); ProfileEvents::increment(ProfileEvents::MemoryWorkerRunElapsedMicroseconds, total_watch.elapsedMicroseconds()); From e2e4c8ee0f8fcf1d6ef4d566c2fa4f9ee2123a56 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 24 Jul 2024 10:21:09 +0200 Subject: [PATCH 095/215] Better --- src/Common/MemoryWorker.cpp | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index 84ccffb8e90..e148f7f8f49 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -65,11 +65,25 @@ Metrics readAllMetricsFromStatFile(ReadBufferFromFile & buf) return metrics; } -uint64_t readMetricFromStatFile(ReadBufferFromFile & buf, const std::string & key) +uint64_t readMetricFromStatFile(ReadBufferFromFile & buf, std::string_view key) { - const auto all_metrics = readAllMetricsFromStatFile(buf); - if (const auto it = all_metrics.find(key); it != all_metrics.end()) - return it->second; + while (!buf.eof()) + { + std::string current_key; + readStringUntilWhitespace(current_key, buf); + if (current_key != key) + { + std::string dummy; + readStringUntilNewlineInto(dummy, buf); + buf.ignore(); + continue; + } + + assertChar(' ', buf); + uint64_t value = 0; + readIntText(value, buf); + return value; + } LOG_ERROR(getLogger("CgroupsReader"), "Cannot find '{}' in '{}'", key, buf.getFileName()); return 0; } From f449c2fea0487abbe262c10fc5af9a99df1bc822 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 25 Jul 2024 08:45:08 +0200 Subject: [PATCH 096/215] Fix --- src/Common/MemoryTracker.h | 5 +++++ src/Common/MemoryWorker.cpp | 8 +++----- src/Coordination/KeeperDispatcher.cpp | 3 ++- src/Coordination/KeeperServer.cpp | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index d2db8489f19..f15465a20c1 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -120,6 +120,11 @@ public: return amount.load(std::memory_order_relaxed); } + Int64 getRSS() const + { + return rss.load(std::memory_order_relaxed); + } + // Merges and mutations may pass memory ownership to other threads thus in the end of execution // MemoryTracker for background task may have a non-zero counter. // This method is intended to fix the counter inside of background_memory_tracker. diff --git a/src/Common/MemoryWorker.cpp b/src/Common/MemoryWorker.cpp index e148f7f8f49..1b869ed9d6b 100644 --- a/src/Common/MemoryWorker.cpp +++ b/src/Common/MemoryWorker.cpp @@ -312,7 +312,7 @@ uint64_t MemoryWorker::getMemoryUsage() void MemoryWorker::backgroundThread() { std::chrono::milliseconds chrono_period_ms{period_ms}; - bool first_run = true; + [[maybe_unused]] bool first_run = true; std::unique_lock lock(mutex); while (true) { @@ -340,17 +340,15 @@ void MemoryWorker::backgroundThread() } #endif +#if USE_JEMALLOC if (unlikely(first_run || total_memory_tracker.get() < 0)) { -#if USE_JEMALLOC if (source != MemoryUsageSource::Jemalloc) epoch_mib.setValue(0); MemoryTracker::updateAllocated(allocated_mib.getValue()); -#elif defined(OS_LINUX) - MemoryTracker::updateAllocated(resident); -#endif } +#endif ProfileEvents::increment(ProfileEvents::MemoryWorkerRun); ProfileEvents::increment(ProfileEvents::MemoryWorkerRunElapsedMicroseconds, total_watch.elapsedMicroseconds()); diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index 4c2ccb8db64..893bb8e6082 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -150,10 +150,11 @@ void KeeperDispatcher::requestThread() { LOG_WARNING( log, - "Processing requests refused because of max_memory_usage_soft_limit {}, the total used memory is {}, request type " + "Processing requests refused because of max_memory_usage_soft_limit {}, the total allocated memory is {}, RSS is {}, request type " "is {}", ReadableSize(mem_soft_limit), ReadableSize(total_memory_tracker.get()), + ReadableSize(total_memory_tracker.getRSS()), request.request->getOpNum()); addErrorResponses({request}, Coordination::Error::ZCONNECTIONLOSS); continue; diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index d40e5ef2e50..ad9e8d32caa 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -599,7 +599,7 @@ bool KeeperServer::isLeaderAlive() const bool KeeperServer::isExceedingMemorySoftLimit() const { Int64 mem_soft_limit = keeper_context->getKeeperMemorySoftLimit(); - return mem_soft_limit > 0 && total_memory_tracker.get() >= mem_soft_limit; + return mem_soft_limit > 0 && std::max(total_memory_tracker.get(), total_memory_tracker.getRSS()) >= mem_soft_limit; } /// TODO test whether taking failed peer in count From ce134830b702b117724235de28afdea2b5db0dfc Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 29 Jul 2024 17:15:15 +0000 Subject: [PATCH 097/215] Disable pr local plan by default --- src/Core/Settings.h | 2 +- src/Core/SettingsChangesHistory.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 4a5dec8dcc8..e175d7f76af 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -934,7 +934,7 @@ class IColumn; M(UInt64, parallel_replicas_min_number_of_rows_per_replica, 0, "Limit the number of replicas used in a query to (estimated rows to read / min_number_of_rows_per_replica). The max is still limited by 'max_parallel_replicas'", 0) \ M(Bool, parallel_replicas_prefer_local_join, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN.", 0) \ M(UInt64, parallel_replicas_mark_segment_size, 128, "Parts virtually divided into segments to be distributed between replicas for parallel reading. This setting controls the size of these segments. Not recommended to change until you're absolutely sure in what you're doing", 0) \ - M(Bool, parallel_replicas_local_plan, true, "Build local plan for local replica", 0) \ + M(Bool, parallel_replicas_local_plan, false, "Build local plan for local replica", 0) \ \ M(Bool, allow_experimental_inverted_index, false, "If it is set to true, allow to use experimental inverted index.", 0) \ M(Bool, allow_experimental_full_text_index, false, "If it is set to true, allow to use experimental full-text index.", 0) \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 24427dab1f3..ee9cfd3f28c 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -58,7 +58,7 @@ String ClickHouseVersion::toString() const static std::initializer_list> settings_changes_history_initializer = { {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."}, - {"parallel_replicas_local_plan", false, true, "Use local plan for local replica in a query with parallel replicas"}, + {"parallel_replicas_local_plan", false, false, "Use local plan for local replica in a query with parallel replicas"}, {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"}, {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"}, {"output_format_native_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in Native output format"}, From 9501bc8a7551daa74cd37363b0047bb42c029697 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 29 Jul 2024 20:33:06 +0000 Subject: [PATCH 098/215] Fix forgotten conflict --- src/Processors/QueryPlan/ReadFromRemote.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index cec460ef3bc..3df46eb1987 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -21,13 +21,8 @@ #include #include #include -<<<<<<< HEAD #include -======= -#include -#include ->>>>>>> origin/master namespace DB { From d711ec118c41689d6f9694048bf279c3dbd2e1db Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 30 Jul 2024 09:04:59 +0000 Subject: [PATCH 099/215] Fix build --- src/Processors/QueryPlan/ConvertingActions.cpp | 2 +- src/Processors/QueryPlan/ReadFromRemote.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/ConvertingActions.cpp b/src/Processors/QueryPlan/ConvertingActions.cpp index ff106ff08c1..b9703ef59cd 100644 --- a/src/Processors/QueryPlan/ConvertingActions.cpp +++ b/src/Processors/QueryPlan/ConvertingActions.cpp @@ -25,7 +25,7 @@ void addConvertingActions(QueryPlan & plan, const Block & header, bool has_missi }; auto convert_actions_dag = get_converting_dag(plan.getCurrentDataStream().header, header); - auto converting = std::make_unique(plan.getCurrentDataStream(), convert_actions_dag); + auto converting = std::make_unique(plan.getCurrentDataStream(), std::move(convert_actions_dag)); plan.addStep(std::move(converting)); } diff --git a/src/Processors/QueryPlan/ReadFromRemote.h b/src/Processors/QueryPlan/ReadFromRemote.h index 2dc4d2f8ed5..74389c8f9eb 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.h +++ b/src/Processors/QueryPlan/ReadFromRemote.h @@ -70,6 +70,7 @@ public: ASTPtr query_ast_, ClusterPtr cluster_, const StorageID & storage_id_, + ParallelReplicasReadingCoordinatorPtr coordinator_, Block header_, QueryProcessingStage::Enum stage_, ContextMutablePtr context_, From bce0f46254d382f78c9a36be1371ed731aeb7880 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 30 Jul 2024 16:26:01 +0000 Subject: [PATCH 100/215] Simplify tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh + enable parallel_replicas_local_plan by default --- ...allel_replicas_join_algo_and_analyzer_3.sh | 52 ++++++------------- 1 file changed, 17 insertions(+), 35 deletions(-) diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh index e49a340ab67..81c6e8bed18 100755 --- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh +++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh @@ -21,23 +21,21 @@ insert into num_2 select number * 3, -number from numbers(1.5e6); echo echo "nested join with analyzer and parallel replicas, both global" -$CLICKHOUSE_CLIENT -q " -select * from (select key, value from num_1) l -inner join (select key, value from num_2 inner join - (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=0) r -on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS allow_experimental_analyzer=1, -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0" +PARALLEL_REPLICAS_SETTINGS="allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_local_plan=1" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=0) r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS allow_experimental_analyzer=1, send_logs_level='trace', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0" 2>&1 | +SETTINGS allow_experimental_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0" + +$CLICKHOUSE_CLIENT -q " +select * from (select key, value from num_1) l +inner join (select key, value from num_2 inner join + (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=0) r +on l.key = r.key order by l.key limit 10 offset 10000 +SETTINGS allow_experimental_analyzer=1, send_logs_level='trace', $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' @@ -51,18 +49,14 @@ select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=1) r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS allow_experimental_analyzer=1, -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0" +SETTINGS allow_experimental_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=1) r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS allow_experimental_analyzer=1, send_logs_level='trace', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0" 2>&1 | +SETTINGS allow_experimental_analyzer=1, send_logs_level='trace', $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' @@ -77,18 +71,14 @@ select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='full_sorting_merge') r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS allow_experimental_analyzer=1, parallel_replicas_prefer_local_join=0, -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='full_sorting_merge'" +SETTINGS allow_experimental_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, join_algorithm='full_sorting_merge'" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='full_sorting_merge') r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS allow_experimental_analyzer=1, parallel_replicas_prefer_local_join=0, send_logs_level='trace', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='full_sorting_merge'" 2>&1 | +SETTINGS allow_experimental_analyzer=1, send_logs_level='trace', $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, join_algorithm='full_sorting_merge'" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' @@ -102,18 +92,14 @@ select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='hash') r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS allow_experimental_analyzer=1, parallel_replicas_prefer_local_join=0, -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='full_sorting_merge'" +SETTINGS allow_experimental_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, join_algorithm='full_sorting_merge'" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='hash') r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS allow_experimental_analyzer=1, parallel_replicas_prefer_local_join=0, send_logs_level='trace', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='full_sorting_merge'" 2>&1 | +SETTINGS allow_experimental_analyzer=1, send_logs_level='trace', $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, join_algorithm='full_sorting_merge'" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' @@ -127,18 +113,14 @@ select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='full_sorting_merge') r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS allow_experimental_analyzer=1, parallel_replicas_prefer_local_join=0, -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='hash'" +SETTINGS allow_experimental_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, join_algorithm='hash'" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='full_sorting_merge') r on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS allow_experimental_analyzer=1, parallel_replicas_prefer_local_join=0, send_logs_level='trace', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='hash'" 2>&1 | +SETTINGS allow_experimental_analyzer=1, send_logs_level='trace', $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, join_algorithm='hash'" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' From c5a2f3aafbe6373a662f80f715e56a302a7fdd8b Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 1 Aug 2024 14:21:54 +0000 Subject: [PATCH 101/215] Limit min_marks_for_concurrent_read --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 772bf427892..ec64db4139f 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -383,9 +383,12 @@ Pipe ReadFromMergeTree::readFromPoolParallelReplicas(RangesInDataParts parts_wit /// We have a special logic for local replica. It has to read less data, because in some cases it should /// merge states of aggregate functions or do some other important stuff other than reading from Disk. auto multiplier = context->getSettingsRef().parallel_replicas_single_task_marks_count_multiplier; - // avoid using multiplier if min marks to read is already too big (to avoid overflow) - if (pool_settings.min_marks_for_concurrent_read >= std::numeric_limits::max()) + if (pool_settings.min_marks_for_concurrent_read > std::numeric_limits::max()) + { + /// limit min marks to read in case it's big, happened in test since due to settings randomzation + pool_settings.min_marks_for_concurrent_read = std::numeric_limits::max(); multiplier = 1.0f; + } if (auto result = pool_settings.min_marks_for_concurrent_read * multiplier; canConvertTo(result)) pool_settings.min_marks_for_concurrent_read = static_cast(result); @@ -556,9 +559,12 @@ Pipe ReadFromMergeTree::readInOrder( }; auto multiplier = context->getSettingsRef().parallel_replicas_single_task_marks_count_multiplier; - // avoid using multiplier if min marks to read is already too big (to avoid overflow) - if (pool_settings.min_marks_for_concurrent_read >= std::numeric_limits::max()) + if (pool_settings.min_marks_for_concurrent_read > std::numeric_limits::max()) + { + /// limit min marks to read in case it's big, happened in test since due to settings randomzation + pool_settings.min_marks_for_concurrent_read = std::numeric_limits::max(); multiplier = 1.0f; + } if (auto result = pool_settings.min_marks_for_concurrent_read * multiplier; canConvertTo(result)) pool_settings.min_marks_for_concurrent_read = static_cast(result); From 85b8c11175ff77176e6bbf4bd0b5821a34a14338 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 1 Aug 2024 20:32:03 +0000 Subject: [PATCH 102/215] Fix flaky 0_stateless/02784_parallel_replicas_automatic_decision - avoid max_threads randomzation --- .../0_stateless/02784_parallel_replicas_automatic_decision.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision.sh b/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision.sh index e5658f31a34..ef41a5d6277 100755 --- a/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision.sh +++ b/tests/queries/0_stateless/02784_parallel_replicas_automatic_decision.sh @@ -53,6 +53,7 @@ function run_query_with_pure_parallel_replicas () { --allow_experimental_parallel_reading_from_replicas 1 \ --parallel_replicas_for_non_replicated_merge_tree 1 \ --parallel_replicas_min_number_of_rows_per_replica "$2" \ + --max_threads 5 \ |& grep "It is enough work for" | awk '{ print substr($7, 2, length($7) - 2) "\t" $20 " estimated parallel replicas" }' } From b6f5eb12115ad54ea9cfade80b71fce2052ce6c0 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Sun, 11 Aug 2024 23:15:19 +0000 Subject: [PATCH 103/215] Fix screwed merge --- src/Core/SettingsChangesHistory.cpp | 261 +----------------- ...allel_replicas_join_algo_and_analyzer_3.sh | 89 +----- ...llel_replicas_joins_and_analyzer.reference | 16 +- 3 files changed, 15 insertions(+), 351 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index e81786a902e..7cfc119f734 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -57,266 +57,6 @@ String ClickHouseVersion::toString() const /// Note: please check if the key already exists to prevent duplicate entries. static std::initializer_list> settings_changes_history_initializer = { - {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."}, - {"parallel_replicas_local_plan", false, false, "Use local plan for local replica in a query with parallel replicas"}, - {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"}, - {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"}, - {"output_format_native_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in Native output format"}, - {"input_format_native_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in Native output format"}, - {"read_in_order_use_buffering", false, true, "Use buffering before merging while reading in order of primary key"}, - {"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."}, - {"input_format_json_case_insensitive_column_matching", false, false, "Ignore case when matching JSON keys with CH columns."}, - {"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."}, - {"dictionary_validate_primary_key_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."}, - {"collect_hash_table_stats_during_joins", false, true, "New setting."}, - {"max_size_to_preallocate_for_joins", 0, 100'000'000, "New setting."}, - {"input_format_orc_reader_time_zone_name", "GMT", "GMT", "The time zone name for ORC row reader, the default ORC row reader's time zone is GMT."}, - {"lightweight_mutation_projection_mode", "throw", "throw", "When lightweight delete happens on a table with projection(s), the possible operations include throw the exception as projection exists, or drop all projection related to this table then do lightweight delete."}, - {"database_replicated_allow_heavy_create", true, false, "Long-running DDL queries (CREATE AS SELECT and POPULATE) for Replicated database engine was forbidden"}, - {"query_plan_merge_filters", false, false, "Allow to merge filters in the query plan"}, - {"azure_sdk_max_retries", 10, 10, "Maximum number of retries in azure sdk"}, - {"azure_sdk_retry_initial_backoff_ms", 10, 10, "Minimal backoff between retries in azure sdk"}, - {"azure_sdk_retry_max_backoff_ms", 1000, 1000, "Maximal backoff between retries in azure sdk"}, - {"merge_tree_min_bytes_per_task_for_remote_reading", 4194304, 2097152, "Value is unified with `filesystem_prefetch_min_bytes_for_single_read_task`"}, - {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."}, - {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."}, - {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."}, - {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."} - }}, - {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"}, - {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"}, - {"input_format_parquet_use_native_reader", false, false, "When reading Parquet files, to use native reader instead of arrow reader."}, - {"hdfs_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in HDFS engine instead of empty query result"}, - {"azure_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in AzureBlobStorage engine instead of empty query result"}, - {"s3_validate_request_settings", true, true, "Allow to disable S3 request settings validation"}, - {"allow_experimental_full_text_index", false, false, "Enable experimental full-text index"}, - {"azure_skip_empty_files", false, false, "Allow to skip empty files in azure table engine"}, - {"hdfs_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in HDFS table engine"}, - {"azure_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in AzureBlobStorage table engine"}, - {"s3_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in S3 table engine"}, - {"s3_max_part_number", 10000, 10000, "Maximum part number number for s3 upload part"}, - {"s3_max_single_operation_copy_size", 32 * 1024 * 1024, 32 * 1024 * 1024, "Maximum size for a single copy operation in s3"}, - {"input_format_parquet_max_block_size", 8192, DEFAULT_BLOCK_SIZE, "Increase block size for parquet reader."}, - {"input_format_parquet_prefer_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Average block bytes output by parquet reader."}, - {"enable_blob_storage_log", true, true, "Write information about blob storage operations to system.blob_storage_log table"}, - {"allow_deprecated_snowflake_conversion_functions", true, false, "Disabled deprecated functions snowflakeToDateTime[64] and dateTime[64]ToSnowflake."}, - {"allow_statistic_optimize", false, false, "Old setting which popped up here being renamed."}, - {"allow_experimental_statistic", false, false, "Old setting which popped up here being renamed."}, - {"allow_statistics_optimize", false, false, "The setting was renamed. The previous name is `allow_statistic_optimize`."}, - {"allow_experimental_statistics", false, false, "The setting was renamed. The previous name is `allow_experimental_statistic`."}, - {"enable_vertical_final", false, true, "Enable vertical final by default again after fixing bug"}, - {"parallel_replicas_custom_key_range_lower", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards"}, - {"parallel_replicas_custom_key_range_upper", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards. A value of 0 disables the upper limit"}, - {"output_format_pretty_display_footer_column_names", 0, 1, "Add a setting to display column names in the footer if there are many rows. Threshold value is controlled by output_format_pretty_display_footer_column_names_min_rows."}, - {"output_format_pretty_display_footer_column_names_min_rows", 0, 50, "Add a setting to control the threshold value for setting output_format_pretty_display_footer_column_names_min_rows. Default 50."}, - {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."}, - {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."}, - {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."}, - }}, - {"24.5", {{"allow_deprecated_error_prone_window_functions", true, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)"}, - {"allow_experimental_join_condition", false, false, "Support join with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y."}, - {"input_format_tsv_crlf_end_of_line", false, false, "Enables reading of CRLF line endings with TSV formats"}, - {"output_format_parquet_use_custom_encoder", false, true, "Enable custom Parquet encoder."}, - {"cross_join_min_rows_to_compress", 0, 10000000, "Minimal count of rows to compress block in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."}, - {"cross_join_min_bytes_to_compress", 0, 1_GiB, "Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."}, - {"http_max_chunk_size", 0, 0, "Internal limitation"}, - {"prefer_external_sort_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging."}, - {"input_format_force_null_for_omitted_fields", false, false, "Disable type-defaults for omitted fields when needed"}, - {"cast_string_to_dynamic_use_inference", false, false, "Add setting to allow converting String to Dynamic through parsing"}, - {"allow_experimental_dynamic_type", false, false, "Add new experimental Dynamic type"}, - {"azure_max_blocks_in_multipart_upload", 50000, 50000, "Maximum number of blocks in multipart upload for Azure."}, - }}, - {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"}, - {"max_parsing_threads", 0, 0, "Add a separate setting to control number of threads in parallel parsing from files"}, - {"ignore_drop_queries_probability", 0, 0, "Allow to ignore drop queries in server with specified probability for testing purposes"}, - {"lightweight_deletes_sync", 2, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes"}, - {"query_cache_system_table_handling", "save", "throw", "The query cache no longer caches results of queries against system tables"}, - {"input_format_json_ignore_unnecessary_fields", false, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields"}, - {"input_format_hive_text_allow_variable_number_of_columns", false, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values."}, - {"allow_experimental_database_replicated", false, true, "Database engine Replicated is now in Beta stage"}, - {"temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds", (10 * 60 * 1000), (10 * 60 * 1000), "Wait time to lock cache for sapce reservation in temporary data in filesystem cache"}, - {"optimize_rewrite_sum_if_to_count_if", false, true, "Only available for the analyzer, where it works correctly"}, - {"azure_allow_parallel_part_upload", "true", "true", "Use multiple threads for azure multipart upload."}, - {"max_recursive_cte_evaluation_depth", DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, "Maximum limit on recursive CTE evaluation depth"}, - {"query_plan_convert_outer_join_to_inner_join", false, true, "Allow to convert OUTER JOIN to INNER JOIN if filter after JOIN always filters default values"}, - }}, - {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"}, - {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"}, - {"use_page_cache_for_disks_without_file_cache", false, false, "Added userspace page cache"}, - {"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"}, - {"page_cache_inject_eviction", false, false, "Added userspace page cache"}, - {"default_table_engine", "None", "MergeTree", "Set default table engine to MergeTree for better usability"}, - {"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"}, - {"traverse_shadow_remote_data_paths", false, false, "Traverse shadow directory when query system.remote_data_paths."}, - {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication in dependent materialized view cannot work together with async inserts."}, - {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"}, - {"log_processors_profiles", false, true, "Enable by default"}, - {"function_locate_has_mysql_compatible_argument_order", false, true, "Increase compatibility with MySQL's locate function."}, - {"allow_suspicious_primary_key", true, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)"}, - {"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"}, - {"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"}, - {"analyzer_compatibility_join_using_top_level_identifier", false, false, "Force to resolve identifier in JOIN USING from projection"}, - {"distributed_insert_skip_read_only_replicas", false, false, "If true, INSERT into Distributed will skip read-only replicas"}, - {"keeper_max_retries", 10, 10, "Max retries for general keeper operations"}, - {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"}, - {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"}, - {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"}, - {"allow_experimental_analyzer", false, true, "Enable analyzer and planner by default."}, - {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."}, - {"allow_get_client_http_header", false, false, "Introduced a new function."}, - {"output_format_pretty_row_numbers", false, true, "It is better for usability."}, - {"output_format_pretty_max_value_width_apply_for_single_value", true, false, "Single values in Pretty formats won't be cut."}, - {"output_format_parquet_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."}, - {"output_format_orc_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."}, - {"output_format_arrow_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."}, - {"output_format_parquet_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."}, - {"output_format_orc_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."}, - {"output_format_pretty_highlight_digit_groups", false, true, "If enabled and if output is a terminal, highlight every digit corresponding to the number of thousands, millions, etc. with underline."}, - {"geo_distance_returns_float64_on_float64_arguments", false, true, "Increase the default precision."}, - {"azure_max_inflight_parts_for_one_file", 20, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited."}, - {"azure_strict_upload_part_size", 0, 0, "The exact size of part to upload during multipart upload to Azure blob storage."}, - {"azure_min_upload_part_size", 16*1024*1024, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage."}, - {"azure_max_upload_part_size", 5ull*1024*1024*1024, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage."}, - {"azure_upload_part_size_multiply_factor", 2, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage."}, - {"azure_upload_part_size_multiply_parts_count_threshold", 500, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor."}, - {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."}, - {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."}, - {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."}, - }}, - {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"}, - {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"}, - {"output_format_values_escape_quote_with_quote", false, false, "If true escape ' with '', otherwise quoted with \\'"}, - {"output_format_pretty_single_large_number_tip_threshold", 0, 1'000'000, "Print a readable number tip on the right side of the table if the block consists of a single number which exceeds this value (except 0)"}, - {"input_format_try_infer_exponent_floats", true, false, "Don't infer floats in exponential notation by default"}, - {"query_plan_optimize_prewhere", true, true, "Allow to push down filter to PREWHERE expression for supported storages"}, - {"async_insert_max_data_size", 1000000, 10485760, "The previous value appeared to be too small."}, - {"async_insert_poll_timeout_ms", 10, 10, "Timeout in milliseconds for polling data from asynchronous insert queue"}, - {"async_insert_use_adaptive_busy_timeout", false, true, "Use adaptive asynchronous insert timeout"}, - {"async_insert_busy_timeout_min_ms", 50, 50, "The minimum value of the asynchronous insert timeout in milliseconds; it also serves as the initial value, which may be increased later by the adaptive algorithm"}, - {"async_insert_busy_timeout_max_ms", 200, 200, "The minimum value of the asynchronous insert timeout in milliseconds; async_insert_busy_timeout_ms is aliased to async_insert_busy_timeout_max_ms"}, - {"async_insert_busy_timeout_increase_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases"}, - {"async_insert_busy_timeout_decrease_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout decreases"}, - {"format_template_row_format", "", "", "Template row format string can be set directly in query"}, - {"format_template_resultset_format", "", "", "Template result set format string can be set in query"}, - {"split_parts_ranges_into_intersecting_and_non_intersecting_final", true, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"}, - {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}, - {"azure_max_single_part_copy_size", 256*1024*1024, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage."}, - {"min_external_table_block_size_rows", DEFAULT_INSERT_BLOCK_SIZE, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to external table to specified size in rows, if blocks are not big enough"}, - {"min_external_table_block_size_bytes", DEFAULT_INSERT_BLOCK_SIZE * 256, DEFAULT_INSERT_BLOCK_SIZE * 256, "Squash blocks passed to external table to specified size in bytes, if blocks are not big enough."}, - {"parallel_replicas_prefer_local_join", true, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN."}, - {"optimize_time_filter_with_preimage", true, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')"}, - {"extract_key_value_pairs_max_pairs_per_row", 0, 0, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory."}, - {"default_view_definer", "CURRENT_USER", "CURRENT_USER", "Allows to set default `DEFINER` option while creating a view"}, - {"default_materialized_view_sql_security", "DEFINER", "DEFINER", "Allows to set a default value for SQL SECURITY option when creating a materialized view"}, - {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"}, - {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, - {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, - }}, - {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."}, - {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"}, - {"output_format_arrow_use_signed_indexes_for_dictionary", false, true, "Use signed indexes type for Arrow dictionaries by default as it's recommended"}, - {"allow_experimental_variant_type", false, false, "Add new experimental Variant type"}, - {"use_variant_as_common_type", false, false, "Allow to use Variant in if/multiIf if there is no common type"}, - {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"}, - {"parallel_replicas_mark_segment_size", 128, 128, "Add new setting to control segment size in new parallel replicas coordinator implementation"}, - {"ignore_materialized_views_with_dropped_target_table", false, false, "Add new setting to allow to ignore materialized views with dropped target table"}, - {"output_format_compression_level", 3, 3, "Allow to change compression level in the query output"}, - {"output_format_compression_zstd_window_log", 0, 0, "Allow to change zstd window log in the query output when zstd compression is used"}, - {"enable_zstd_qat_codec", false, false, "Add new ZSTD_QAT codec"}, - {"enable_vertical_final", false, true, "Use vertical final by default"}, - {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"}, - {"max_rows_in_set_to_optimize_join", 100000, 0, "Disable join optimization as it prevents from read in order optimization"}, - {"output_format_pretty_color", true, "auto", "Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"}, - {"function_visible_width_behavior", 0, 1, "We changed the default behavior of `visibleWidth` to be more precise"}, - {"max_estimated_execution_time", 0, 0, "Separate max_execution_time and max_estimated_execution_time"}, - {"iceberg_engine_ignore_schema_evolution", false, false, "Allow to ignore schema evolution in Iceberg table engine"}, - {"optimize_injective_functions_in_group_by", false, true, "Replace injective functions by it's arguments in GROUP BY section in analyzer"}, - {"update_insert_deduplication_token_in_dependent_materialized_views", false, false, "Allow to update insert deduplication token with table identifier during insert in dependent materialized views"}, - {"azure_max_unexpected_write_error_retries", 4, 4, "The maximum number of retries in case of unexpected errors during Azure blob storage write"}, - {"split_parts_ranges_into_intersecting_and_non_intersecting_final", false, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"}, - {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}}}, - {"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."}, - {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"}, - {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"}, - {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}}, - {"23.11", {{"parsedatetime_parse_without_leading_zeros", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}}, - {"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"}, - {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"}, - {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"}, - {"input_format_json_read_arrays_as_strings", false, true, "Allow to read arrays as strings in JSON formats by default"}, - {"input_format_json_infer_incomplete_types_as_strings", false, true, "Allow to infer incomplete types as Strings in JSON formats by default"}, - {"input_format_json_try_infer_numbers_from_strings", true, false, "Don't infer numbers from strings in JSON formats by default to prevent possible parsing errors"}, - {"http_write_exception_in_output_format", false, true, "Output valid JSON/XML on exception in HTTP streaming."}}}, - {"23.8", {{"rewrite_count_distinct_if_with_count_distinct_implementation", false, true, "Rewrite countDistinctIf with count_distinct_implementation configuration"}}}, - {"23.7", {{"function_sleep_max_microseconds_per_block", 0, 3000000, "In previous versions, the maximum sleep time of 3 seconds was applied only for `sleep`, but not for `sleepEachRow` function. In the new version, we introduce this setting. If you set compatibility with the previous versions, we will disable the limit altogether."}}}, - {"23.6", {{"http_send_timeout", 180, 30, "3 minutes seems crazy long. Note that this is timeout for a single network write call, not for the whole upload operation."}, - {"http_receive_timeout", 180, 30, "See http_send_timeout."}}}, - {"23.5", {{"input_format_parquet_preserve_order", true, false, "Allow Parquet reader to reorder rows for better parallelism."}, - {"parallelize_output_from_storages", false, true, "Allow parallelism when executing queries that read from file/url/s3/etc. This may reorder rows."}, - {"use_with_fill_by_sorting_prefix", false, true, "Columns preceding WITH FILL columns in ORDER BY clause form sorting prefix. Rows with different values in sorting prefix are filled independently"}, - {"output_format_parquet_compliant_nested_types", false, true, "Change an internal field name in output Parquet file schema."}}}, - {"23.4", {{"allow_suspicious_indices", true, false, "If true, index can defined with identical expressions"}, - {"allow_nonconst_timezone_arguments", true, false, "Allow non-const timezone arguments in certain time-related functions like toTimeZone(), fromUnixTimestamp*(), snowflakeToDateTime*()."}, - {"connect_timeout_with_failover_ms", 50, 1000, "Increase default connect timeout because of async connect"}, - {"connect_timeout_with_failover_secure_ms", 100, 1000, "Increase default secure connect timeout because of async connect"}, - {"hedged_connection_timeout_ms", 100, 50, "Start new connection in hedged requests after 50 ms instead of 100 to correspond with previous connect timeout"}, - {"formatdatetime_f_prints_single_zero", true, false, "Improved compatibility with MySQL DATE_FORMAT()/STR_TO_DATE()"}, - {"formatdatetime_parsedatetime_m_is_month_name", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}}, - {"23.3", {{"output_format_parquet_version", "1.0", "2.latest", "Use latest Parquet format version for output format"}, - {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"}, - {"input_format_native_allow_types_conversion", false, true, "Allow types conversion in Native input forma"}, - {"output_format_arrow_compression_method", "none", "lz4_frame", "Use lz4 compression in Arrow output format by default"}, - {"output_format_parquet_compression_method", "snappy", "lz4", "Use lz4 compression in Parquet output format by default"}, - {"output_format_orc_compression_method", "none", "lz4_frame", "Use lz4 compression in ORC output format by default"}, - {"async_query_sending_for_remote", false, true, "Create connections and send query async across shards"}}}, - {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"}, - {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"}, - {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"}, - {"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"}, - {"insert_keeper_max_retries", 0, 20, "Enable reconnections to Keeper on INSERT, improve reliability"}}}, - {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"}, - {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"}, - {"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"}, - {"input_format_tsv_detect_header", false, true, "Detect header in TSV format by default"}, - {"input_format_custom_detect_header", false, true, "Detect header in CustomSeparated format by default"}, - {"query_plan_remove_redundant_sorting", false, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries"}}}, - {"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"}, - {"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"}, - {"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}}, - {"22.11", {{"use_structure_from_insertion_table_in_table_functions", 0, 2, "Improve using structure from insertion table in table functions"}}}, - {"22.9", {{"force_grouping_standard_compatibility", false, true, "Make GROUPING function output the same as in SQL standard and other DBMS"}}}, - {"22.7", {{"cross_to_inner_join_rewrite", 1, 2, "Force rewrite comma join to inner"}, - {"enable_positional_arguments", false, true, "Enable positional arguments feature by default"}, - {"format_csv_allow_single_quotes", true, false, "Most tools don't treat single quote in CSV specially, don't do it by default too"}}}, - {"22.6", {{"output_format_json_named_tuples_as_objects", false, true, "Allow to serialize named tuples as JSON objects in JSON formats by default"}, - {"input_format_skip_unknown_fields", false, true, "Optimize reading subset of columns for some input formats"}}}, - {"22.5", {{"memory_overcommit_ratio_denominator", 0, 1073741824, "Enable memory overcommit feature by default"}, - {"memory_overcommit_ratio_denominator_for_user", 0, 1073741824, "Enable memory overcommit feature by default"}}}, - {"22.4", {{"allow_settings_after_format_in_insert", true, false, "Do not allow SETTINGS after FORMAT for INSERT queries because ClickHouse interpret SETTINGS as some values, which is misleading"}}}, - {"22.3", {{"cast_ipv4_ipv6_default_on_conversion_error", true, false, "Make functions cast(value, 'IPv4') and cast(value, 'IPv6') behave same as toIPv4 and toIPv6 functions"}}}, - {"21.12", {{"stream_like_engine_allow_direct_select", true, false, "Do not allow direct select for Kafka/RabbitMQ/FileLog by default"}}}, - {"21.9", {{"output_format_decimal_trailing_zeros", true, false, "Do not output trailing zeros in text representation of Decimal types by default for better looking output"}, - {"use_hedged_requests", false, true, "Enable Hedged Requests feature by default"}}}, - {"21.7", {{"legacy_column_name_of_tuple_literal", true, false, "Add this setting only for compatibility reasons. It makes sense to set to 'true', while doing rolling update of cluster from version lower than 21.7 to higher"}}}, - {"21.5", {{"async_socket_for_remote", false, true, "Fix all problems and turn on asynchronous reads from socket for remote queries by default again"}}}, - {"21.3", {{"async_socket_for_remote", true, false, "Turn off asynchronous reads from socket for remote queries because of some problems"}, - {"optimize_normalize_count_variants", false, true, "Rewrite aggregate functions that semantically equals to count() as count() by default"}, - {"normalize_function_names", false, true, "Normalize function names to their canonical names, this was needed for projection query routing"}}}, - {"21.2", {{"enable_global_with_statement", false, true, "Propagate WITH statements to UNION queries and all subqueries by default"}}}, - {"21.1", {{"insert_quorum_parallel", false, true, "Use parallel quorum inserts by default. It is significantly more convenient to use than sequential quorum inserts"}, - {"input_format_null_as_default", false, true, "Allow to insert NULL as default for input formats by default"}, - {"optimize_on_insert", false, true, "Enable data optimization on INSERT by default for better user experience"}, - {"use_compact_format_in_distributed_parts_names", false, true, "Use compact format for async INSERT into Distributed tables by default"}}}, - {"20.10", {{"format_regexp_escaping_rule", "Escaped", "Raw", "Use Raw as default escaping rule for Regexp format to male the behaviour more like to what users expect"}}}, - {"20.7", {{"show_table_uuid_in_table_create_query_if_not_nil", true, false, "Stop showing UID of the table in its CREATE query for Engine=Atomic"}}}, - {"20.5", {{"input_format_with_names_use_header", false, true, "Enable using header with names for formats with WithNames/WithNamesAndTypes suffixes"}, - {"allow_suspicious_codecs", true, false, "Don't allow to specify meaningless compression codecs"}}}, - {"20.4", {{"validate_polygons", false, true, "Throw exception if polygon is invalid in function pointInPolygon by default instead of returning possibly wrong results"}}}, - {"19.18", {{"enable_scalar_subquery_optimization", false, true, "Prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once"}}}, - {"19.14", {{"any_join_distinct_right_table_keys", true, false, "Disable ANY RIGHT and ANY FULL JOINs by default to avoid inconsistency"}}}, - {"19.12", {{"input_format_defaults_for_omitted_fields", false, true, "Enable calculation of complex default expressions for omitted fields for some input formats, because it should be the expected behaviour"}}}, - {"19.5", {{"max_partitions_per_insert_block", 0, 100, "Add a limit for the number of partitions in one block"}}}, - {"18.12.17", {{"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}}}, {"24.12", { } @@ -345,6 +85,7 @@ static std::initializer_list>>>>>> origin/master +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=0) r on l.key = r.key order by l.key limit 10 offset 10000 -<<<<<<< HEAD -SETTINGS allow_experimental_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0" - -$CLICKHOUSE_CLIENT -q " -select * from (select key, value from num_1) l -inner join (select key, value from num_2 inner join - (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=0) r -on l.key = r.key order by l.key limit 10 offset 10000 -SETTINGS allow_experimental_analyzer=1, send_logs_level='trace', $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0" 2>&1 | -======= -SETTINGS enable_analyzer=1, send_logs_level='trace', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0" 2>&1 | ->>>>>>> origin/master +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, send_logs_level='trace', parallel_replicas_prefer_local_join=0" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' @@ -66,26 +49,14 @@ select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=1) r on l.key = r.key order by l.key limit 10 offset 10000 -<<<<<<< HEAD -SETTINGS allow_experimental_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0" -======= -SETTINGS enable_analyzer=1, -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0" ->>>>>>> origin/master +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=1) r on l.key = r.key order by l.key limit 10 offset 10000 -<<<<<<< HEAD -SETTINGS allow_experimental_analyzer=1, send_logs_level='trace', $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0" 2>&1 | -======= -SETTINGS enable_analyzer=1, send_logs_level='trace', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0" 2>&1 | ->>>>>>> origin/master +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, send_logs_level='trace', parallel_replicas_prefer_local_join=0" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' @@ -100,26 +71,14 @@ select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='full_sorting_merge') r on l.key = r.key order by l.key limit 10 offset 10000 -<<<<<<< HEAD -SETTINGS allow_experimental_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, join_algorithm='full_sorting_merge'" -======= -SETTINGS enable_analyzer=1, parallel_replicas_prefer_local_join=0, -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='full_sorting_merge'" ->>>>>>> origin/master +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, join_algorithm='full_sorting_merge'" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='full_sorting_merge') r on l.key = r.key order by l.key limit 10 offset 10000 -<<<<<<< HEAD -SETTINGS allow_experimental_analyzer=1, send_logs_level='trace', $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, join_algorithm='full_sorting_merge'" 2>&1 | -======= -SETTINGS enable_analyzer=1, parallel_replicas_prefer_local_join=0, send_logs_level='trace', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='full_sorting_merge'" 2>&1 | ->>>>>>> origin/master +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, send_logs_level='trace', join_algorithm='full_sorting_merge'" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' @@ -133,26 +92,14 @@ select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='hash') r on l.key = r.key order by l.key limit 10 offset 10000 -<<<<<<< HEAD -SETTINGS allow_experimental_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, join_algorithm='full_sorting_merge'" -======= -SETTINGS enable_analyzer=1, parallel_replicas_prefer_local_join=0, -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='full_sorting_merge'" ->>>>>>> origin/master +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, join_algorithm='full_sorting_merge'" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='hash') r on l.key = r.key order by l.key limit 10 offset 10000 -<<<<<<< HEAD -SETTINGS allow_experimental_analyzer=1, send_logs_level='trace', $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, join_algorithm='full_sorting_merge'" 2>&1 | -======= -SETTINGS enable_analyzer=1, parallel_replicas_prefer_local_join=0, send_logs_level='trace', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='full_sorting_merge'" 2>&1 | ->>>>>>> origin/master +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, send_logs_level='trace', join_algorithm='full_sorting_merge'" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' @@ -166,26 +113,14 @@ select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='full_sorting_merge') r on l.key = r.key order by l.key limit 10 offset 10000 -<<<<<<< HEAD -SETTINGS allow_experimental_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, join_algorithm='hash'" -======= -SETTINGS enable_analyzer=1, parallel_replicas_prefer_local_join=0, -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='hash'" ->>>>>>> origin/master +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, join_algorithm='hash'" $CLICKHOUSE_CLIENT -q " select * from (select key, value from num_1) l inner join (select key, value from num_2 inner join (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='full_sorting_merge') r on l.key = r.key order by l.key limit 10 offset 10000 -<<<<<<< HEAD -SETTINGS allow_experimental_analyzer=1, send_logs_level='trace', $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, join_algorithm='hash'" 2>&1 | -======= -SETTINGS enable_analyzer=1, parallel_replicas_prefer_local_join=0, send_logs_level='trace', -allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, -cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='hash'" 2>&1 | ->>>>>>> origin/master +SETTINGS enable_analyzer=1, $PARALLEL_REPLICAS_SETTINGS, parallel_replicas_prefer_local_join=0, send_logs_level='trace', join_algorithm='hash'" 2>&1 | grep "executeQuery\|.*Coordinator: Coordination done" | grep -o "SELECT.*WithMergeableState)\|.*Coordinator: Coordination done" | sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g' diff --git a/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.reference b/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.reference index 453caf1cffe..ba67acd7d0b 100644 --- a/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.reference +++ b/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.reference @@ -18,8 +18,7 @@ select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x 13 13 0 0 0 0 14 14 14 14 0 0 15 15 0 0 0 0 -<<<<<<< HEAD -explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1; +explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; Union Expression Join @@ -34,11 +33,6 @@ Union ReadFromMemoryStorage Expression ReadFromRemoteParallelReplicas -======= -explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; -Expression - ReadFromRemoteParallelReplicas ->>>>>>> origin/master -- -- The same query with cte; with sub1 as (select x, y from tab1 where x != 2), @@ -477,8 +471,7 @@ select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x 13 13 0 0 0 0 14 14 14 14 0 0 15 15 0 0 0 0 -<<<<<<< HEAD -explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1; +explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; Union Expression Join @@ -495,11 +488,6 @@ Union ReadFromMergeTree Expression ReadFromRemoteParallelReplicas -======= -explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1; -Expression - ReadFromRemoteParallelReplicas ->>>>>>> origin/master -- -- The same query with cte; with sub1 as (select x, y from tab1 where x != 2), From 2068fbb15759dcc6f6140c84e2e08b596da4f766 Mon Sep 17 00:00:00 2001 From: marco-vb Date: Mon, 12 Aug 2024 10:05:10 +0000 Subject: [PATCH 104/215] Added support for single wildcard usage in x509 SubjectAltName user identification (both DNS and URI). --- .../external-authenticators/ssl-x509.md | 8 +- src/Access/Authentication.cpp | 24 +++++ .../certs/ca-cert.pem | 56 +++++----- .../certs/ca-cert.srl | 2 +- .../certs/ca-key.pem | 100 +++++++++--------- .../certs/client1-cert.pem | 52 ++++----- .../certs/client1-key.pem | 100 +++++++++--------- .../certs/client1-req.pem | 46 ++++---- .../certs/client2-cert.pem | 52 ++++----- .../certs/client2-key.pem | 100 +++++++++--------- .../certs/client2-req.pem | 46 ++++---- .../certs/client3-cert.pem | 52 ++++----- .../certs/client3-key.pem | 100 +++++++++--------- .../certs/client3-req.pem | 46 ++++---- .../certs/client4-cert.pem | 53 +++++----- .../certs/client4-key.pem | 100 +++++++++--------- .../certs/client4-req.pem | 46 ++++---- .../certs/client5-cert.pem | 33 ++++++ .../certs/client5-ext.cnf | 1 + .../certs/client5-key.pem | 52 +++++++++ .../certs/client5-req.pem | 27 +++++ .../certs/client6-cert.pem | 33 ++++++ .../certs/client6-ext.cnf | 1 + .../certs/client6-key.pem | 52 +++++++++ .../certs/client6-req.pem | 27 +++++ .../certs/generate_certs.sh | 4 + .../certs/server-cert.pem | 56 +++++----- .../certs/server-key.pem | 100 +++++++++--------- .../certs/server-req.pem | 46 ++++---- .../certs/wrong-cert.pem | 52 ++++----- .../certs/wrong-key.pem | 100 +++++++++--------- .../configs/ssl_config.xml | 0 .../configs/users_with_ssl_auth.xml | 5 + .../test_ssl_cert_authentication/test.py | 30 ++++++ 34 files changed, 950 insertions(+), 652 deletions(-) create mode 100644 tests/integration/test_ssl_cert_authentication/certs/client5-cert.pem create mode 100644 tests/integration/test_ssl_cert_authentication/certs/client5-ext.cnf create mode 100644 tests/integration/test_ssl_cert_authentication/certs/client5-key.pem create mode 100644 tests/integration/test_ssl_cert_authentication/certs/client5-req.pem create mode 100644 tests/integration/test_ssl_cert_authentication/certs/client6-cert.pem create mode 100644 tests/integration/test_ssl_cert_authentication/certs/client6-ext.cnf create mode 100644 tests/integration/test_ssl_cert_authentication/certs/client6-key.pem create mode 100644 tests/integration/test_ssl_cert_authentication/certs/client6-req.pem mode change 100644 => 100755 tests/integration/test_ssl_cert_authentication/configs/ssl_config.xml mode change 100644 => 100755 tests/integration/test_ssl_cert_authentication/configs/users_with_ssl_auth.xml diff --git a/docs/en/operations/external-authenticators/ssl-x509.md b/docs/en/operations/external-authenticators/ssl-x509.md index 09fac45d7ae..435dd8227fc 100644 --- a/docs/en/operations/external-authenticators/ssl-x509.md +++ b/docs/en/operations/external-authenticators/ssl-x509.md @@ -6,7 +6,7 @@ import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.m -[SSL 'strict' option](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) enables mandatory certificate validation for the incoming connections. In this case, only connections with trusted certificates can be established. Connections with untrusted certificates will be rejected. Thus, certificate validation allows to uniquely authenticate an incoming connection. `Common Name` or `subjectAltName extension` field of the certificate is used to identify the connected user. This allows to associate multiple certificates with the same user. Additionally, reissuing and revoking of the certificates does not affect the ClickHouse configuration. +[SSL 'strict' option](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) enables mandatory certificate validation for the incoming connections. In this case, only connections with trusted certificates can be established. Connections with untrusted certificates will be rejected. Thus, certificate validation allows to uniquely authenticate an incoming connection. `Common Name` or `subjectAltName extension` field of the certificate is used to identify the connected user. `subjectAltName extension` supports the usage of one wildcard '*' in the server configuration: see [test_x509_san_wildcard_support](../../../../tests/integration/test_ssl_cert_authentication/test.py). This allows to associate multiple certificates with the same user. Additionally, reissuing and revoking of the certificates does not affect the ClickHouse configuration. To enable SSL certificate authentication, a list of `Common Name`'s or `Subject Alt Name`'s for each ClickHouse user must be specified in the settings file `users.xml `: @@ -30,6 +30,12 @@ To enable SSL certificate authentication, a list of `Common Name`'s or `Subject + + + + URI:spiffe://foo.com/*/bar + + ``` diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 6b9a6e05cf6..994aec57f0c 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -5,11 +6,13 @@ #include #include #include +#include "Common/LoggingFormatStringHelpers.h" #include #include #include #include +#include "base/types.h" #include "config.h" namespace DB @@ -239,15 +242,36 @@ bool Authentication::areCredentialsValid( throw Authentication::Require(auth_data.getKerberosRealm()); case AuthenticationType::SSL_CERTIFICATE: + { for (SSLCertificateSubjects::Type type : {SSLCertificateSubjects::Type::CN, SSLCertificateSubjects::Type::SAN}) { for (const auto & subject : auth_data.getSSLCertificateSubjects().at(type)) { if (ssl_certificate_credentials->getSSLCertificateSubjects().at(type).contains(subject)) return true; + + // Wildcard support (1 only) + if (subject.contains('*')) + { + auto prefix = std::string_view(subject).substr(0, subject.find('*')); + auto suffix = std::string_view(subject).substr(subject.find('*') + 1); + auto slashes = std::count(subject.begin(), subject.end(), '/'); + + for (const auto & certificate_subject : ssl_certificate_credentials->getSSLCertificateSubjects().at(type)) + { + bool matches_wildcard = certificate_subject.starts_with(prefix) && certificate_subject.ends_with(suffix); + + // '*' must not represent a '/' in URI, so check if the number of '/' are equal + bool matches_slashes = slashes == count(certificate_subject.begin(), certificate_subject.end(), '/'); + + if (matches_wildcard && matches_slashes) + return true; + } + } } } return false; + } case AuthenticationType::SSH_KEY: #if USE_SSH diff --git a/tests/integration/test_ssl_cert_authentication/certs/ca-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/ca-cert.pem index d1e4a3a88d9..bec7f132e23 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/ca-cert.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/ca-cert.pem @@ -1,32 +1,32 @@ -----BEGIN CERTIFICATE----- -MIIFhTCCA22gAwIBAgIUZmPYBB6vdp8uxKlJcS8mI0SArqQwDQYJKoZIhvcNAQEL +MIIFhTCCA22gAwIBAgIULizd839OeWFia09eiBiQGk7m/wYwDQYJKoZIhvcNAQEL BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwNjI2 -MTAyNTAwWhcNMzQwNjI0MTAyNTAwWjBSMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwODEy +MTAzOTI4WhcNMzQwODEwMTAzOTI4WjBSMQswCQYDVQQGEwJSVTETMBEGA1UECAwK U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQsw -CQYDVQQDDAJjYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALHzp+SR -T8EtrMBYNlgwIUGMZaXomQPwsjOpjt3RUsdE3LCQ15rLuyBwZ0SbLMDOazCDA7Xr -+AXHDYwg/PCJhe2N4NTzfgnWqbkIGwYLhys95Xxq+q+kV7Csk/27JSk1rsK3Nru/ -Iuj/NWgBkAQC8n10qfbJSXHcXm9wDVy/L8t7DXwxkX70LToGWsb6QW6VkiWe/QOW -QYTMVncqjrtLTCTnqGIk0s6ZIifgPLfaYJxtwK+LdIoJioSbAuVfHmVTe10vjonO -YqJ56KfqOn/G7dIf1l9gOLFBAe4jUf3RS1wkQMdk+oEGLbGnShgW+BQMfB48RkrQ -486h7slzwC29jBJLRARI2Oc9p8/zBDVph0pxkjVGka8dfIkZbmTD932h/1gfMgQl -F20G/H5FF1jk37bDcsczns0c24S1F2uJbzOlHjFLhqmH1IaVCWsYawfBs9khModW -VS6+WAv//cqWE3KmmJ2EdtAmzMCJzAQUEyrMZWmrFrBzpACyMq2zFEtyIXsCXpgq -eW4odxIIZlClibo1FGrflqN+hXnAhRyCj7WJBQ0ZPrEdRMRpBYhYdmygPJ+lWfsg -HOtNnshSuJTXGJTtsuJVr4Ioeq6tKfWGofRu4vvT6cILZjbp9VaxxfVLS7bTsjWR -c+5xHp+KbcjSfw2cJHQN85hlWTpMe9jPhgt/AgMBAAGjUzBRMB0GA1UdDgQWBBSJ -Kj1dSYN0jW+Mu0xFQrobZFmQpTAfBgNVHSMEGDAWgBSJKj1dSYN0jW+Mu0xFQrob -ZFmQpTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQCEm/epjNWe -w0A+1M3i0y8adrXWdTeRXuP3Klgtjh3NGFKNm9WGnOAufsqUIpVwKLw1uYNIhiEA -fj5KuD7YIVU9CrWx4Z/+lILUZhyn/br5mACtRiRTF/75/QwLcDz5z7K9MyMzdL99 -DLQ9bd3JCuEgVj6zacPrALwWVhM8u9fNGJxdQANnIC8yTY5+ZE59/fn7UI787JuR -4njOGWSVnDavbTyJCPMPiUkgwqL+QSWBcNbGAPzMaAblvc1SL2Lj/ikFDAETAZs2 -T/3ZqBqHEOuVhFQYTAvMAdMQX3w8bYv/CGL8++W+qHazY+uqPypd9CLnICbnkZmr -P+id9WleGl2F//u1CQ+YA2Q3EazSFhwRLA7IKIVCrYVaBsbe/bpxxZb6+AQVfM/i -+7+fCbr7A5HDe9Fi4dClv6xPI0GZZkarhQPsoLPaDQeqM4OE+K6oPSHJnqfAB8v3 -NgTt1QuVnKhwveX5DDEP4t/Qt4j2n7AFpeoZDEA8aM33K0noXNrwqHz3991O1RWz -t/gd+cFG/Z1jRP8kYtfAV8go2nzt8QvqBhfIcNnMwD8cwuKJ5G7SdqLvDFj3XCCO -YqQAALl4QFs046eVDEWLajSESmj4fCaTmO05sHHLy7U5asoAo/MWGbmGmL+8ExUX -sPO9r12lPJ7IThJ13PSqbJIJnnloL/XCxA== +CQYDVQQDDAJjYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAME6XxeV +Nl6RnBKhr1yygLIXYIVFmqgcEyAm/5hWhV2bERP6urmIYLWcokaFpBMCXkH5tumA +2B5PQQgVVTTVv51qGNgCHR7Vfmtf3s0BlAn1XI719TelWrtVP7XzLlDjDl5ubb3I +WB0ZhckU57of0kZrUIfrqJKaSyKqYp/Hgjdczuk2uIgv12eX7r47qhfq6B9rmhVp +a/WRB2LbBoyJyjsY6UdRLld7vO2wVSWdmy9wxVhssbkP9zNY3JbE81HUMhyw4F/W +j+ryTesILSYUGvx4DjxvP8zlMvL4N81/V7q8nVkwiu9A3owECqUz5pAHfVk6U4zF +vhU+7TTLP3eyFwJk9o1ctxjT0zKtxLRv+ss8eER/7loLOD+zvq3Y/5XCHGd5WILv +9cDfz44XS0UMRdE/M2RQnFmTDPDKy1IY2TxpQLEv4txtY9h/edRDwcPPaU17gY8Z +boXZR61d90zOIDGITAr8DD/xrsei57ly/W9cbIKfNj0QfH0T6ESP5lW3fS9GRwjY +djoeOZ8uK0d+pzHGUVTkE/9Hf9Zl4NfnwvvFvvF3X/U2VeBkxw+ZgcEiphI+uUjx +aXsn7leNuAemPB50LEnQD5bWKIT1qIeVIQWOHzP5g+tGxCsfg/YzoUb8MvZSe+n2 +FLDqSYKUPy6mZf0U38teh7eIfrIT6yux5IzXAgMBAAGjUzBRMB0GA1UdDgQWBBRG +jKB4cx0JVNfQH8o4feWwhu6NRDAfBgNVHSMEGDAWgBRGjKB4cx0JVNfQH8o4feWw +hu6NRDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBtD9HsFU2D +28aJozGf2sr2CL1wjlpuOSB5LYu3rcuMrU8R+6xR7rSSUsxuXGB5drnnur9UQ8gt +9Q/sxWUyErHTycfFSQy3mhsFIUy0bGwebSeEPCK3DJo5q3QsncxidcIpTJMgUHe3 +MfyEA2ow+qnlTKROHitiALA+OzEcDdTbU1+JSUM4w0JPfHSdHcwlZzTQu5BhJOjQ +msS3letdX5iqD8xdfIG7EWn/LRupTJjSMsMwspOq9Cfx9kcgI+n9rPJt9iL9eSE7 +gBhUc6Df3JDuX/qImH8lMzE/F7H/7XReCdWXinApW94SZttTbnEWhMhpsMw1psNo +vfw0HSQnu6G6eRdbZK2TCXgzc9BwGcowfQXbv1UvWAhoNBw75UO5nOzCR2T7SakE +R1fNHEQyjFBmq60r7AYitRjcX+nE7F+T/MatpBKFoEz1aDxWfcRx+kwuJZi0sw+B +US4PWVIGN2SFfShp9JpVa8Q3PQmUFyIWCPJlEbBohTTja/nvppr8/OSTg3A3a1s1 +J2DTscxDmQGCWRBbO2LlQE495ylNMfS7BdTDrp/Xv3h+pM+qi4my8+JOC7uX/9dL +X6b7Kz9d8HlxQGug/HvnaNN6k5sQpRJN7LBGPHUj1RG5/kw0WdEDs12SfXfMle6b +q+kwPJhMAGv7CfFXKWtCjt4M5sLCAagdxA== -----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/ca-cert.srl b/tests/integration/test_ssl_cert_authentication/certs/ca-cert.srl index cf47b0dc79c..71859de222d 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/ca-cert.srl +++ b/tests/integration/test_ssl_cert_authentication/certs/ca-cert.srl @@ -1 +1 @@ -05F10C67567FE30795D77AF2540F6AC8D4CF2470 +05F10C67567FE30795D77AF2540F6AC8D4CF247E diff --git a/tests/integration/test_ssl_cert_authentication/certs/ca-key.pem b/tests/integration/test_ssl_cert_authentication/certs/ca-key.pem index 2dea2ccd837..d4466037e77 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/ca-key.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/ca-key.pem @@ -1,52 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQCx86fkkU/BLazA -WDZYMCFBjGWl6JkD8LIzqY7d0VLHRNywkNeay7sgcGdEmyzAzmswgwO16/gFxw2M -IPzwiYXtjeDU834J1qm5CBsGC4crPeV8avqvpFewrJP9uyUpNa7Ctza7vyLo/zVo -AZAEAvJ9dKn2yUlx3F5vcA1cvy/Lew18MZF+9C06BlrG+kFulZIlnv0DlkGEzFZ3 -Ko67S0wk56hiJNLOmSIn4Dy32mCcbcCvi3SKCYqEmwLlXx5lU3tdL46JzmKieein -6jp/xu3SH9ZfYDixQQHuI1H90UtcJEDHZPqBBi2xp0oYFvgUDHwePEZK0OPOoe7J -c8AtvYwSS0QESNjnPafP8wQ1aYdKcZI1RpGvHXyJGW5kw/d9of9YHzIEJRdtBvx+ -RRdY5N+2w3LHM57NHNuEtRdriW8zpR4xS4aph9SGlQlrGGsHwbPZITKHVlUuvlgL -//3KlhNyppidhHbQJszAicwEFBMqzGVpqxawc6QAsjKtsxRLciF7Al6YKnluKHcS -CGZQpYm6NRRq35ajfoV5wIUcgo+1iQUNGT6xHUTEaQWIWHZsoDyfpVn7IBzrTZ7I -UriU1xiU7bLiVa+CKHqurSn1hqH0buL70+nCC2Y26fVWscX1S0u207I1kXPucR6f -im3I0n8NnCR0DfOYZVk6THvYz4YLfwIDAQABAoICABZRI14j5yen7cFVjsMyjgkl -bV4INKBs4DxgaF1jMglxlmfCUnfEUxx3XEwbVdp8SK8VzzJSfJFk7EsFnBMifBxV -rbunKchcFn7xCEqSyYnfwlb/J589cg3jJtAsVzW62MbsqT2Uc/FaiD0Z7RDDuduH -9QTRK5fO9jzthY97HqhbL07C/Kc6Qi3DvEC2A9y1f1WegcagYmkgIzvgp3PPtqXu -M4zTZ2inhcQQeCzqgzE7Bm49hAkHt0p4Ej3n1u0IMjF2lF6t9mq/9TCRzHJX5V1z -xrPBYnrAV1ihL1gwlk3g8grPnCbwOmzMORuaTdRd2HcGQh6B4f/5CPRUwuY5nkY7 -UMcX4rbRCcBDzG8s/NllTRaVC6yvEJPN4B/zOek6DI+tRy4tRj+BZO/1771bindD -nsYAklxbbpTiS7B073b+as3RFZW0xvmoqLyhzRQNW6GqSGj4C0KVigkWQ7Br69b7 -O2++oEurmLqI5Hm3RsJeJJ9obG4vKhPUPdtStF/W0W2TO2i0gpTOm+NeO5uYBRB1 -6OvhJH9dzMi+a0ekCpdQeD2xG4NLzwSe62/Ozz9Oi0rpAqHHhu+SvF+WEepLbkyO -2zx/OYpFK47idBRCAlHLC/9UyXpvw2yU9sleElVuHM41CzMe8Pwj3Qk0YdiHHDzS -Y19XEVHh/riXUufsAHVhAoIBAQDpUe+UJLgAIT6boz+MPLrPIGLeazCySEHEsdnp -jGuAx0M59PIDn5OeXQpMqCIRFDw6zhA/4gji0UegFRPIGA3Qduq+GsjVYRt6SHLC -N/rBRi2xg77yyKOMxv/+nwKFh/3TKIQbUc9EQj63WGBGCHu/EyMV7i9V0j8e7D2u -v/Z23nV/+XqawJXi4u2bHB3M/upOoclKfb3ewZBVouajzZc92kNC8XYfPn10Eofu -Pz7jxDX10IJNmzIYOozO9mlBsds7nFIBXS5wMz3iVg3GEUB05pPEKoEtZGrw474u -0M+gW9d7PV3qYdFgjSogiQf4JrSrOwXJQL/26nyfRX9QVplxAoIBAQDDP+fFT7Zl -eFLvouZu73lr++bV1+LogHgX+GHCSIujBELPyFEAyAeElFKFqnJ/NEOuPLG9X7tL -PDhh9NUijcBTPhVvwbH2/CRBdSX7Yf6RHh5fY+2Ik3hTF81L4bQcf0fgyX4roJY9 -YqpjQWFYGmSk4niCqWd+re/ZrYx/zpF+qgN21v37BAZNOCI+KidFScrq29p8kpLj -MdBWa0m7bhJcv4MPO46s2EJZVdczBU7iK86v5NVrGz7fPVm+tGxEDpwhyfYiu961 -U05XzT+trAaBa88KlSKdmUFq3qDdC4bFb6D+Ra4g+mhqNGtfBYXsavnENZgt0N99 -9M/dgaAEa/vvAoIBAQCm4YcFo9nDpgOc2H/Mc2d+XIC661oyOkJoSHk/dcMyodNw -scUkWFACdjO2ro9nPdzyho7S0n5elSew1UKH3MSMtXGjNY8uJ726kfUa+2UsNmDa -VgwOpPlt6KwTV3I7RhCDprgOvk4MWYF4LAr4LHsuKKbwuaM7tByXpotb4UuMrALI -3Q0XgOX0GTGvvsWF6VJ3mXpbAGL839+3kMN8p8EkaWewivpc0Jp0mgiFnNEDokSi -JFf+4CFNeRtbsJ2KcocHNQDmntpnQA9kQv6pC4/ZzU4lge1RJUDkOVC/NXU8ElSm -fjcdPIfAklduW/TKRgz1aEr0Lo7fMcqfNNsiAD7RAoIBAQCaeplZ13OsXMLhrrU6 -2GXtNeSxFJoG8n4SGQbfvJ4eYGSsGQVd5OVt1BxmfTERy7wwwvytpGx/XioN9rQb -HqQoOFqljU7M5zmYQKPIfQP4tSe6uUlaYbM1qwNXIkBqu5mXFFSrF+dGsiW1Wik2 -l8tBWZ2XY4jrBZtbUqBzDnC3ErSi9f8E92408lDFdnyTqYrOvxviq+VjtCnt9fzk -OnZ0w2FiT/DWeFQmcnBNgcmj0J07NYZVs7zOy6+R3xY50oVdhnkjihjuxfaaKV5U -fmK3SyEIcm5s2rCTaYlE2rXKyENMar0WgojSXp8FE02efBUZVH4O4c+xzFwaGVEN -rpIpAoIBAQDnAmhVy85n9oX6Bi+bXg9aZFa/teiCX33lCjNvNeJ5GljGqO0X6423 -6FVg91BYvnCbHQ0frqyKimVxNO/XYxCnid+Or48Cb9LwD31Wyip3UZABljg5sTb0 -fiNK0pxx6G8x1MsX0u97LogGwd5bMuff2lMi4xpinkz6ydB+fYsmM0UXGKsNkB/d -zR1izlqm87TMeQVi+pVZuOLmg/+hXVgISI2M7TlSoytODmpnSg5SZbaT7ut1IIwm -hJdWMTPHLv0X7NwsvhV4Knu27BJBw4470+okrOLfDuZ9+LZ6JHvYg22MywjCJx9s -MJuwAZJiZb+dQc0uwMkAEPMEfOQ1BI1+ +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDBOl8XlTZekZwS +oa9csoCyF2CFRZqoHBMgJv+YVoVdmxET+rq5iGC1nKJGhaQTAl5B+bbpgNgeT0EI +FVU01b+dahjYAh0e1X5rX97NAZQJ9VyO9fU3pVq7VT+18y5Q4w5ebm29yFgdGYXJ +FOe6H9JGa1CH66iSmksiqmKfx4I3XM7pNriIL9dnl+6+O6oX6ugfa5oVaWv1kQdi +2waMico7GOlHUS5Xe7ztsFUlnZsvcMVYbLG5D/czWNyWxPNR1DIcsOBf1o/q8k3r +CC0mFBr8eA48bz/M5TLy+DfNf1e6vJ1ZMIrvQN6MBAqlM+aQB31ZOlOMxb4VPu00 +yz93shcCZPaNXLcY09MyrcS0b/rLPHhEf+5aCzg/s76t2P+VwhxneViC7/XA38+O +F0tFDEXRPzNkUJxZkwzwystSGNk8aUCxL+LcbWPYf3nUQ8HDz2lNe4GPGW6F2Uet +XfdMziAxiEwK/Aw/8a7Houe5cv1vXGyCnzY9EHx9E+hEj+ZVt30vRkcI2HY6Hjmf +LitHfqcxxlFU5BP/R3/WZeDX58L7xb7xd1/1NlXgZMcPmYHBIqYSPrlI8Wl7J+5X +jbgHpjwedCxJ0A+W1iiE9aiHlSEFjh8z+YPrRsQrH4P2M6FG/DL2Unvp9hSw6kmC +lD8upmX9FN/LXoe3iH6yE+srseSM1wIDAQABAoICAESmzQdPc9A4srdp66inIRNl +O91N5AtrYh4vzOrefqbJw9ER6Yqmmhmca4lbzPYWBtAw/RxMMzzdkrDhB3NsqBYx +Dl+crKT6XF6g4sN6lpLNMpP2bifVn31kZezq2B9T9JR3bpUZm7jHCyHsxAH8cVSk +pymLgEqJ+Dz6RW0YULsDxd3VG84DFWiQcfnzr9SsmMklDUsjOAC5BONKBzMid4/0 +o1k7zhe19mOKnU3uh6bczbjDcYdbVUxQe5szoXGGz0EjVKyoSlzCtw1Uy9m2Ffpy +Wzh8nDHbkvUBXK4t2skVX2BDX3Fu3kLn4rJcVMQCbTYpA2tGDGl3AIcb2VJZG1QX +lWlrdnWIxsn69Yzqp9f4XE7PByJRcjLVRTROjzh8Zq2ygthAV3t9LOLUb34HehSz +56fENAIihwUZXHJoJVIcN4/1fn64TNxzeXGHc3CzFeeV/NhZb2ERBXT2SO5p3m65 +jX+vuqxsBphR5eHhXATIZnEOUwHGk4FcFu/Zv907ssxF5RqGNG2UZv8MNc6O8dmd +/uRHuCaNg0LpESS5Y6yu3lLfuVjxmZd5k364gf2U9bxedKMED8LLyzLWMAVLJ0wP +P4GuJlAKldoyhOK1R1QkIS84fWn+1YZbtorEvVtauN6fJeZzxjwEGE4TCTN89tjp +7Q6HvXzo+QUvYBzThiM9AoIBAQDtM8AkkLKEl4w/8pz1kkYXID8u9A7u9ltzfqEX +gn+StuhhOb50EkMP5LkguvHS9u4tHGl/pAfaaugdR2/m0658YZqSxFuMRQk9JUYl +fkL2zl5kQh1xC3dRhB7Wv/LGh701hFKnn54qpkzBbeMTQEkLNxkmeCEBVcJTwl83 +TnDnIchvC97CbHO5z7cnkF3glSVSVW4+EO4Qywly4n7A8yU9RFG/d20bnc/OwCah +Qxt2rHzesrJF2T5tQyqIpZNTMdyLQ7qwXNSCiLWjyF8jbah/czqE1E6vLsxi+JCA +MLc/5JEWERFJA+ck25m5EjWy7dkyAR5zBZbkVavgEHy3HR3DAoIBAQDQin5bjETG +HMC/N7hc7eijRs02qFvqHI/6nGszXPt+q4kTOMFnSFFcJEnshx4dlZ+MaPJoN6vD +6Vp6TA0F8UcgTTCxIpGrjfdTG+UwmOU8bZXjfJ0C4TDAaZhOGe2qSZRKM7ilZ8mH ++tZtEIj8qeFfOsWzeWYVdYFem4xLaJzHE8/O2s311gP5VSCeDhv9wCfoajz1h69G +AGYxCkHs7WEYdkuAHr+2/ln2CUgVIsI/Mq3zeNqkH2W6EcvoPaE33y81L+ccT+O2 +Xzm0E7gqyT5KMpS73MyKqMdUWIgdrXVolMRe5Uu8hY6RaqdD3JeEOriZdz2CklqI +bAtqQcaxb39dAoIBAQDNddkGEyv+e8KpFX5tnBEIMQ9pnU9eH5ihVts0oP3d2H86 +0CivI65hEu5kcmnLIca23nhnbm66+4Uo71r3Zv4pkOWyIVFnE9ln1GSJT2+zlpfe +YgJci+EDg9yuiT9CRgtXUtHBjhoTuU6c8ZESeMmMZGJtlvqHzH/xy1hscI6NNg8G +WRLP997Z4Rk1tPAL+0SDp6pBKqr7ctMxZw0CFtp30ji5Dzo8BJse7APKBXXBerEC +uWh/5igTvm0WRc1x76uoGbeY0h/lHYOwZXw7FN4x316d9cReK7wwMjwAPbJNLq+Z +nJkLVM0fePQTgblvzmQm0x9L5FiKWcbx43YTqq2/AoIBAGgyGopRXneCHEguZf6A +nHEgSh6aJjqmBMZZ6zIhrkMPIBErlW6ucoBQXWNB66kqfipgsWWjEz5y1NMwNn2W +nUwlZUPMGu6Vo1KyKGSMsFTuCCs+sxCiRYKOABxd1iL0WpRsnUB/x/EaQpMB5QAh +PUsgiXFwIbJtILONkp6G6XaKjpUzPB8tzJSClSaDtDaoEQiONGEVZ1zZjDXKg0AH +JzFawcSFGJMtKPPGlW6CDH4ZK1LwOoYwbdp3U+N5D5lj+SlhAt9hh0wy7hjslWND +Y80c0rW+z7AkPlwFVBVH+tbDJg6oEJxkOuSrXmGI0WevE1cSffhEedmZ6mYv8T/s +jr0CggEBAI9xrRumnEM2Xc5Ui58xnbI7/29DUx4bZRQgGsplXGzaS7puI7YKBcSI +sq3wW/jhFzDJZVZLa5RUOyLJJbLkSiFshD2gdFmlwbn3mEwDeUyInCOcTpWQjCTJ +4CS5Jq0aUOMwYxgnXKFgQOZoXAF5BfeKrBKwYYVz9BM+vi7ytKyZLjzuE4P0ov1Y +It2xCffK5EvpegjeTfv45UguDNOKn9yEnofv20v20q7MvPsLhPOEHJoLjlH+Vwu7 +ITmkZ+Cww5qLuRuHO2/y0VcRGxqcpk578QlNssO9dLkq0Bs8ohvHdFBna/PfCC2a +JLQFvYyizF5x7A50l4gv6PKcsyHFEyQ= -----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client1-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client1-cert.pem index 068855d96ea..c8fd22c9e40 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client1-cert.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client1-cert.pem @@ -1,30 +1,30 @@ -----BEGIN CERTIFICATE----- -MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyRtMA0GCSqGSIb3DQEBCwUAMFIx +MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyR5MA0GCSqGSIb3DQEBCwUAMFIx CzAJBgNVBAYTAlJVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl -cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTI0MDYyNjEwMjUw -NFoXDTM0MDYyNDEwMjUwNFowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt +cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTI0MDgxMjEwMzk0 +NFoXDTM0MDgxMDEwMzk0NFowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt U3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UE -AwwHY2xpZW50MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALaJgDgT -PPqKnBNHrxu/jVwMpfPa9ENvaHbtSi88jQrvu2O+nRV/EoOBb/ol3pQn7RveMyj0 -PV6Io3KzmkMQRcX2Z5H0BNPAM0WijXpzfxWSdS9yZ/BKbpyq3QfHPX9FTAcnnn0S -gBgKUDfSxnQt4tn32kNv09q8nk0H1JXzjxILwKAhllHweJlyVBt1a6AG/yNe9K+9 -atyhYgwuQHv7z0OdbF8NfBWhWODLPkZ4Lj4LktVakYSi5XS5sdTtYXwi8mEDkucZ -1CrlZswIzR44nlmkJVoZssMbGphLzxWw/XiDn2ZtO4QNU1cJYU/WbJiMY+SUjIyr -AsLf2GAbU5bUiluG+XZsYKUjvrzVV6gYHWncXPDVWvjz96PDRsc6M19rsrhMT9N1 -tMgydCPqpxJkweC8/IRt7GZhTlxQNL3ioFsjksPmvm112012XYHd+NiuVIZucY6P -c0dFRWi0VKZUjvLVRbtxWMlSawTk0S/C6sbL2r556GwxJTwkm+EIuK6nGDKg7Kmw -yLFlSyqtXkvUOnnAnIOzEH3VdjUyUniUbfFT4ODs6TLzIkFSSJDN7W4klP6p1Ot1 -ZUkB030FYpFt1r39AfWLPWLjwzKvMWenWaTSpZIRO3h8sXbh6gt7zVZKNMrf8AFJ -uyOnfYaQpUwrxvWvuJdWZETS7lFgoRrJxGDvAgMBAAEwDQYJKoZIhvcNAQELBQAD -ggIBAJ8XAILsGKVXyCBMazuq3k2rup8kkNU/hVg1RuYGmy4bNClNN6havzu/S1SO -/g00+tlXGBlSiPlRXq/p/oZgOI/efqIWSBfcmHuR9STdELfa7zugdFpscgaTOhTq -Ko5o1B81pZKw6wzVfn2JlGxnEy9e+lCC7ptMdFiBqc7SGrapkjCjLReszn1Jctk/ -9lsSvhWZ/7GhvRO/L93X3/ZM51K7VZxEwFnibULApObDZQBFipYdfKlrrVrqtdj2 -M7Plx2Hh+Ivt16Kj/DqRcRLcWVlaM8rp4QAtjn4bDYArFEGGi8ElWFRNjs5ztE12 -f0Iu+yqGmvDn0lHEocNf8fgxHIN1uJ2sYImS11Yn7xHp5FPb7efvYh8Ih6voCaTg -NojHi61q26YIU112A1ylStV4xMKgxt2rqRvmc6UTnWDtzNO9jp3NscQVHtUEJpv2 -Jd+JsDf1c/w42KTwTyOAz5j+D0acRmw1YRsv2BpO5tcly8nvdMX9k7JesdiQL9bx -ik863yOLG0AOMdWtZPVG1BAuiUUlbBS2RRUp3qsP4OuJ+eVKlXFieX+2NuzqyddV -CywaA+R05nutX5R34h3Cm2MmQOERAk9FUeHFX7cZMAXQRcmoBZKtUfKOGUKF0zOT -ZEs7xmHSqAOTx8ufDU26pnbyCxOBYwn1DVX9nYSskMGMSfGU +AwwHY2xpZW50MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALygXybg +ZQVWMyk3lYQS3DNZ+1Cs4qEgCHREtEKR2MgcE71ShJeWNzZEtPjjG18pCCpc2g3+ +TDM6fHlSmCx1mH0QsiEXEZsq6zbQVzA5a8ts1tHNOQKBK3BxDKDS38MScYTOo8nz +27zvVIJGSOPiuZKzr/aZ72yUlbhDEUEuf90vruAnZz3CbcD7COQ8E6XdlJfkCBMR +8MGXd4onHyWcfWU89dRvV/bfw4FU66MQkQlHa/z5DTQIH3Tst9Sdj8lp+w6sS8nP +p53yJbAOJ9hq4zUNyUjuNoeO+KFnJ2yETIHVGFPUuXqpMPzSk7AofEwxRHc1xtYa +tYM7KXKKZ2L50PMxRHEMbp/xgaSB+xOW/725uB6EqlUNQZZiuyhsk6c5G8alTc1k +nYhoNIleyfn+tmG3jLPgV+gIZnIorVtoZKh8bio3KAuqusONnWFHyhLhr4Y9tEMR +1lNK4vGCdoqVEPRB+Jm4oY7UQ75Dz1d3ivKktGzA4oSNwxfc3tBccKVV6j7qqPmq +IcqkWBZ4M8/6ulAvUB+9zEQ/HzajWZCW+PjDXHTSnSVybelzFv+ixBAFt+IGFGNW +SJRmu4zWOj8ZBMYJ834Ft7/kqfiTmGQzuZ4kQg+4qM5PQpkDnA+EwjrJ/OYeKbIc +9kB4WY02tKTiziW1hMr0Yk2gdWewmyo29bplAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggIBAK/dfx3lQt/ySYQ3POrAxZETvSk+Z7Fw2G8nJ3CNPtrA2nWJFvOS11iUjR3n +nsF6e7CfApHNE91CE8x+eCm7nukkuJTq8Vt0MszMQXyEIOKuVzTMYcIm7o3MtiHy +lGVkueOfYTV0d14aZyY9SKSVodtP3aJDzTTdcj4r/zVeSqWAD3qwfTxHTWoWEBLG +7/6yV5fWVN0ntzlf3EwADaV4qtazXmn1/a7HokJ0PlWRkB5LqqhU6c1dSAYvQIdG +fmZYZXAQKuhfBaOk4mLM8uNc3WzbdFeOMhVa4JpYdqVjboD5r+dWXyIrZPkHa+oQ +rZAe9GTBrZjZ7dfi7g/kdDCsLiRC4it3+cH2M1wwOcO/j8ZGvHil0bXDD4ZeRNzi +rgj7/U4mtaUn+LPZaAn5Re9fMTXE6VWADMim2aQnYz+OULlLu1O+tAcwGteUpuuJ +IESit3T2DTlC7ohHhDo9aq/LblsIZ1fPsITI83RIovYArdBqz52ULPYkkXf3Gsu7 +bhSU/XoEu7245AkOWr+dqcGcspU7bS7bBErXTI+G2uwNxY6oSBhhZNeSvIdvWPlQ +wIpUUZFd3rqTHQn8b7KHXoMbpZoTXBWrKxMgITSHWsxyK6htbGpqN/nlGf/U1tcF ++Ww7lXzJSA0HGJvgJcSwBSs7vaLBkAZpBW2rzoWGip64zrCL -----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client1-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client1-key.pem index 8d9b887b033..b4186a06ba4 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client1-key.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client1-key.pem @@ -1,52 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQC2iYA4Ezz6ipwT -R68bv41cDKXz2vRDb2h27UovPI0K77tjvp0VfxKDgW/6Jd6UJ+0b3jMo9D1eiKNy -s5pDEEXF9meR9ATTwDNFoo16c38VknUvcmfwSm6cqt0Hxz1/RUwHJ559EoAYClA3 -0sZ0LeLZ99pDb9PavJ5NB9SV848SC8CgIZZR8HiZclQbdWugBv8jXvSvvWrcoWIM -LkB7+89DnWxfDXwVoVjgyz5GeC4+C5LVWpGEouV0ubHU7WF8IvJhA5LnGdQq5WbM -CM0eOJ5ZpCVaGbLDGxqYS88VsP14g59mbTuEDVNXCWFP1myYjGPklIyMqwLC39hg -G1OW1Ipbhvl2bGClI7681VeoGB1p3Fzw1Vr48/ejw0bHOjNfa7K4TE/TdbTIMnQj -6qcSZMHgvPyEbexmYU5cUDS94qBbI5LD5r5tddtNdl2B3fjYrlSGbnGOj3NHRUVo -tFSmVI7y1UW7cVjJUmsE5NEvwurGy9q+eehsMSU8JJvhCLiupxgyoOypsMixZUsq -rV5L1Dp5wJyDsxB91XY1MlJ4lG3xU+Dg7Oky8yJBUkiQze1uJJT+qdTrdWVJAdN9 -BWKRbda9/QH1iz1i48MyrzFnp1mk0qWSETt4fLF24eoLe81WSjTK3/ABSbsjp32G -kKVMK8b1r7iXVmRE0u5RYKEaycRg7wIDAQABAoICACgRktW8U1xj5NLOn3+l0q/s -DtmyrH/JCtNgTzKDRiqqaSYCB5VaaYP4e84bVfqLsR627eAFjRsdP1PEXQ5vmgFU -j3OYbx7UR+z3O7svcywXFCYwJOS4UgON9iro73Tqjz/a0I1/7CJa0TUPzYRfNjbG -k2DOQWD4mn8qQt4Pss4xSj1cYhTmhnKYiCHm6pMcNhFbnLafC8AWpOErnfgZVGvx -OIK9AQn2ev4NX0Q0yWHRRJAU63CEGX4/7OtimE2Zlj75e9vC7bHk3WXYYL5Li2b+ -Azz9+yGc53+a1IBcc6dqrSjcvX3FNxAZ/QR7eycZWiwo95lBSL/iRysBlJ29VglW -YScc2Il/xWNp97PORwsJEDpeWq5UYdinARFK6PAGjnxmADZNAeZHP+r1C5CQaw72 -y31aIrhL2s9wRPZ2DytIlkSmvffIoNpZJW2AyVdJn8L37Aot0Hwr0SsU8/zibvZ7 -4d+7/6rnPnE1jTZlpnDgyH5e5Mtn3YUYDlPAEQudfYyvh0QrNfSOMnetWSYTh/Oi -40iQM2vaKDiK91deTR50g90A88eSgxWMGG6WUzqNoE5CwwiNQxHPhrmFi4H1V6y2 -uaF3s0Gx6aF6j+ws1ImbgrkpAbvgTCENoDtmS8MYbZjXgTzwnG4UtIwqdc5bK2B5 -i9mdb5w1v2v6XLUxVvKhAoIBAQDhVgmw/ssCscde91dWLMylm5Wf+Q7Ry32eYSr0 -UXqYUbChHkYNK5HpVY5b6Br7C13FrhboDk0hsz3agOFexcrua1N2huyJ8gGjlAzz -i+286WuwbhsX9rYgqTvAZmQYpklAfWLZH8nlwtt3iafNhgSVaa//A2m4hhZagElT -pctVakSyG3OYaNDTXBDOnZi9xagc3eWmxkS8PWFaYC0DJCw9yf+9ynH6+FRZg75x -t7nnDd/eSxtW9QUALUCheOO+yIp/uJUiIyWR69cfojQ2vNx5t8FzpK6EqHFCujhq -e+kJB81BAc2P59O8oGqw9fvc9pzCQXyFbx7dtl/Xu/JyMEqnAoIBAQDPYH0afED6 -qyvtQ1le6bjLW3jGCaymflgGwc0sm/pm/3XY4WXhrmqeSIF3tbhP2HnANGinW0wP -nFd0vAi8RU9UxB7iIUAZ6wXRS8/YQmv5zIouPzSCpmvW0l0IcnqjDUS0IZPRo+UA -FTzS2KIQ/yOsHSZoVNQe/Tsdk7Z8XVAJlq+YZ7o7pGR25yGPleUUbVwbIhoEiBPq -EFA+4BErok4CFQB9J8jLRdzmTEQFjQ/w4w066ZkplcIy009a4+LXIvL+MCPG3qMD -+2K/HlTYfMd+CyozaF0ZGTECtakrK+PWbbTj+VV30SD9Sckk8ZqIFUq18Fb574gF -K2KSq5SkYSh5AoIBAQDdenJ2HEkvcctzJQsbsVbII58yKFsPi8IBjKHql7c2xXwl -MJtL0JpOzH/rB7yVKXvWk6ECHyRizkkqXeil/STTqHvVkRInF83SmO8N5mgaeRcW -x3Ir4JrsiUoodrtFmxN+pn8kx+DqytZprMxY7rPMo5+PuCwOaQTJmTP5Woj7gELb -CK5ajBNM2z3Nxwrc48yz6soRXOksV+w7JzK21rQBW2zZf4T+V1yYyyvBnALF/lYe -qJXLp3Jt1QykaSz4VSYEGUnDzuXbggHknspRTtopbJpg7ul1jBYeruhKiVXoQVnV -3k7MdeEgkk+rdWtDqMU1Daa1hB3Db8DOS3YmFB8bAoIBAQDPDD476F0UKTzlWf3r -9pzLZNuTlmsrnC+VJ4ALjvwWQ+7MiFapWfQXbrrc47FO/wqoLWtj1JJ/b5Ad+/MY -znajYmCXU61lczLOwcuV1tNph59bBz4NR82ZoVTDr1DkZMX4tyGYCPQF/i5JMYO2 -Rpa+LCiBuFhFTH3uTOHBD4Vu3WUaXE4jaEHqOWBXtMgQehOg/45MgfSoGHuWGy7p -itYp3AAt9T/UPD+OLA0qIaoNzxQRgtOqIlzPVA0B6U89jyZfRX8i+nx16FKyEL2T -nBmtrcYHp6Zz/aPiWa+6a8rB96zIhNOhmko+uaG7YgHw5pk+R+T/C/mZd7SmTetN -p7e5AoIBAQDXqOVl33+eRw3CxNCJZsfglrD/Jz8VuZv5MZwRolEB4IQwm/whXdnT -34y0UUpUQHnVepzAP3QjsLKANPUY2rKO+f8NAX4Uakzn43QLI+hZcxx6hVkV6OkJ -Hi9fwSEBZzx5DWEbxmYMIGlaRL1yVff8wevQci7WA4rrztb9D5B0c49ItCrMkLNs -X6+9Bh4zafL/FxJSkTahQLe+KGNXSGGGrYB9M31oLSKKM955ZTRnICPxuyA2hffx -8lmHZ/5hmP+eMKoAJ9khilX4LmnkdXJEZ2w5lQTPUTNP8ggaXvFWpijjUsaXEdkR -NMnXQHpKE2RaT22UJ6z3W+biQqNlhlVW +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC8oF8m4GUFVjMp +N5WEEtwzWftQrOKhIAh0RLRCkdjIHBO9UoSXljc2RLT44xtfKQgqXNoN/kwzOnx5 +UpgsdZh9ELIhFxGbKus20FcwOWvLbNbRzTkCgStwcQyg0t/DEnGEzqPJ89u871SC +Rkjj4rmSs6/2me9slJW4QxFBLn/dL67gJ2c9wm3A+wjkPBOl3ZSX5AgTEfDBl3eK +Jx8lnH1lPPXUb1f238OBVOujEJEJR2v8+Q00CB907LfUnY/JafsOrEvJz6ed8iWw +DifYauM1DclI7jaHjvihZydshEyB1RhT1Ll6qTD80pOwKHxMMUR3NcbWGrWDOyly +imdi+dDzMURxDG6f8YGkgfsTlv+9ubgehKpVDUGWYrsobJOnORvGpU3NZJ2IaDSJ +Xsn5/rZht4yz4FfoCGZyKK1baGSofG4qNygLqrrDjZ1hR8oS4a+GPbRDEdZTSuLx +gnaKlRD0QfiZuKGO1EO+Q89Xd4rypLRswOKEjcMX3N7QXHClVeo+6qj5qiHKpFgW +eDPP+rpQL1AfvcxEPx82o1mQlvj4w1x00p0lcm3pcxb/osQQBbfiBhRjVkiUZruM +1jo/GQTGCfN+Bbe/5Kn4k5hkM7meJEIPuKjOT0KZA5wPhMI6yfzmHimyHPZAeFmN +NrSk4s4ltYTK9GJNoHVnsJsqNvW6ZQIDAQABAoICAB+qwt65fGw9cQBh8wmFyw8Z +862zIKcRRCoU4LhuWNkbkZrOsDDsyJ8lQMaqBg3fSx4lkXwY7b0J/By4J5ELOwsA +Ew+oGk4VEgEVUBXqBfVUc+osDH6lpd/OMuMLFALhZUp5EJlWhdT9zw/B8fnENghx +f0CkzkPJ3Crfo5VwU4oUN8UtCFGfgyKLYo5Cob/OZ+RqJYriD6vInIqGj92WHJ0y +80hzWu642U8srRSmstq+cVw6iaNrG7DbDu+0dg/H6Zyog17BrG/BV99rOPhF1Zgy +FNFYILJ4Z7hdI4q92URQvLFSPMNhhTZkueOwoFGAUfC7iAKo4EFEXOx8XOgKbbAk +gumbWjDMuLcl5NINpnh1yZz5vvDph7rkBjGBpja4FDMyxbCMjsLoTXtM0jCr/59r +hhriPavGZTQ5YhaF8dv/WaoOjXD0PDo8ujS925CoUJteuS/gNHe28chy9rI+dFVR +5XpO7kVQsoAb/54NdeLd0AKD/dQHfStMcxMucRRtZ9muYsXvIsXaSw9HnoG9RAyF ++VuHIUEs76+EAwYrKroUqzPwe6FD/LjmoQCiD6lVudpJQkY507eoCkWMrERXgGlF +y33YFVUmQ8+Uh1Tfq6XWGNGCxU3e1b36s557Lixbc4n45I6Zhso5lBTokch7dPHb +Gj1Tlx8fCSKmV8Gekqc/AoIBAQDlUEBcKtGK7l08jQyTMLfz8XMazbSuguVkIyU2 +aSW/qEl17F8BV2BgRnqCneIZER490RtooOpnAbTZ2AMw9Xx2ZCarCbYv/2ipp+fq +TwzCm96LipLXQCeu/Khwsez+04J3xwg3ziRGCOCwDT3aI1yvuTA4+DN5qa7gJc5I +qTyzQ7FkPcq0JOVhpqqNgDf2bHxteGUQz4Dnh6KVJ5WktVRS/x1fN7l/InUz4pWR +UmRGEmHcY4m5OS0mTDioRq6IoMDktjX7b/hicFC0lKBNGsYKW5FZgMZawseF6B9M +J4UBS1kuCacktumDROhxFvKym5n+qF8Yxe8D7usedg05pKrPAoIBAQDSk/cGkYoL +AYXG0h0Pu5GCw52MpgmQF4FKU82nUnWCFdHJQDKklBplD3e7IfjgjgVz51omPhq5 +jqcqvQJP+PaM1dPAP83Qg8IVJccrhKr8Uhm+yLsM/G6/zR/Jh+fvOXuOimYQptSB +LmMV9R/WS4GJAAWbrEkFRfXnV8U7hnjsQyb92fy3i32FZyB9wbhvA2830rEW/7X+ +bEFnqFliamxj3QfiKMH3pWKmBps1HiejgwScMzrAshD+SYdegNIsBU25M9fduBj0 +P4vHijfpikCW7KI8g/Ua7zomNwvPMAcrqf2K12D50g7sppPLeZVyiNxVupeU0iK9 +mwz7EllTckSLAoIBAQCrYXL4bzf0MXmVS4VebEbPFmEEY0DqPAHc+TdSlvsTZl6g +t2u5wOfSjFn7TKwyN8z2PlXRuJH3MhtLxFl+ObovcWJE6hkzfhtm0IpPX40V1RBa +EdL09Im0CjiJITESslkL08CclIAfp3rlmL/lTBB1dF5J9ZXjE6IS+65UN8eZ4rmt +RVE9mWWlub41yuIIVcQYvKYsDik04PXtq4v3cV34OZpmE2ubTNFzWaCW35D5nbtt +zyabO9Q2jzi46zp9+kavqE5oAKMOTwrCK2IyvhPMoerbjZaxNmmP9o+Nhuusr/eS +pLw6gEm9Rb7C5ne0bP6GiKeqCCQ+OE2YknoH3r2ZAoIBABMZSHZYDOUipjd6Jxie +MnxhsZBoHsXq29cZ8rUSTYYW5IgUQoOayyoPVRSr6B61wtjYStzdFR9vSMofim4m +bnYnjqUJJGmdCaVRX9camUd5Ssx6+DwdSQ4sVI26vdZ8qDAtbURlXTC6PFJpgRG2 +l3ENh3SvE5Be4AazdzgG4kDLdmXft4QOKrxGtjOxtkCbVyMBAP7NAnyo5OW55yF9 +9Ya5LVXdzHnhjijwwHL44S97WpewLcv8Wn41vQldNbK4aGw7v4Bkih7dBJh8eMW9 +vE3z+GL0tO5UdDmzFGcePAhByrUlA66k1qWwFgeGZ3+T2igMXGMrC+uWfH7pEyD2 +ZDECggEAT0o1NvWbsnms65w1oF8/2RFsvlTVeb0tpS9D3ijzCoeRv72fThiZ/diF +WtQPEW7WdYPWLG5UMOggake/GS4zF4+TBDNqZDcdVgu0esk738CUvLHzBl7n1Los +7CIOH0dT2Qook3XoOe2lLUD1arfqrpj/ygUW0MoHo9ihSPOact1jusNNyOlMAcma +OzSY6jJnRDE7vJr9h4+1fDJYd4PbSq83KNHJyVayaSk6RBxijPEjmkBZ6bWNdDt7 +/kihyxT3+ZBGvuFcvnBGFgRtUV6ZGsz/74MGDvKnmIyvt+PuvZueMJIqMKytUva5 +CG/KQT7wu3i0IjmX91IiDXTzH8oa4w== -----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client1-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client1-req.pem index d5cd522bc8f..37288c0a23f 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client1-req.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client1-req.pem @@ -1,27 +1,27 @@ -----BEGIN CERTIFICATE REQUEST----- MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp -ZW50MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALaJgDgTPPqKnBNH -rxu/jVwMpfPa9ENvaHbtSi88jQrvu2O+nRV/EoOBb/ol3pQn7RveMyj0PV6Io3Kz -mkMQRcX2Z5H0BNPAM0WijXpzfxWSdS9yZ/BKbpyq3QfHPX9FTAcnnn0SgBgKUDfS -xnQt4tn32kNv09q8nk0H1JXzjxILwKAhllHweJlyVBt1a6AG/yNe9K+9atyhYgwu -QHv7z0OdbF8NfBWhWODLPkZ4Lj4LktVakYSi5XS5sdTtYXwi8mEDkucZ1CrlZswI -zR44nlmkJVoZssMbGphLzxWw/XiDn2ZtO4QNU1cJYU/WbJiMY+SUjIyrAsLf2GAb -U5bUiluG+XZsYKUjvrzVV6gYHWncXPDVWvjz96PDRsc6M19rsrhMT9N1tMgydCPq -pxJkweC8/IRt7GZhTlxQNL3ioFsjksPmvm112012XYHd+NiuVIZucY6Pc0dFRWi0 -VKZUjvLVRbtxWMlSawTk0S/C6sbL2r556GwxJTwkm+EIuK6nGDKg7KmwyLFlSyqt -XkvUOnnAnIOzEH3VdjUyUniUbfFT4ODs6TLzIkFSSJDN7W4klP6p1Ot1ZUkB030F -YpFt1r39AfWLPWLjwzKvMWenWaTSpZIRO3h8sXbh6gt7zVZKNMrf8AFJuyOnfYaQ -pUwrxvWvuJdWZETS7lFgoRrJxGDvAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA -L2KVSFlGtuYjmV6sTYF0GlA4V0RvUTbLM7qnd321957ReDR8iRqj9ZnqKDEHeH9g -jfuW+TV/BeQFjTTAXM5Gy+LRz23xOwqcvxKh0WYjGLhDXIjWx1zzzGBaAcyWMllq -+o6LAUadST3yhfncP0A79hPLxOsPgMGd3OXGajKskDNmU3MTTsbtQ065HsBwo07P -leMx2jDkapesUaNaTmXERg6MT86hsknGsbO/tU3dGNy4hBuOX5O6bEkijF6eESLd -U4Xc54yScVvxpWqTSEAz9xHjIOpOZNfW+enbLdpApxq6IZkeVM8z7yy8DNjTJ2SD -aS/xKexqrWjWFxNa/CtKezkaZgmLs9jGGan+hmlNBeuixvJEekPliv6Syj3wvLp/ -L3/PmLgBzZj6iRdw5fky0swCn1qwpgwYRjBSN+SL0E8yG6BGKFWByQfwWbdOu9DS -lN/CPBe73yi8kYY5gBvBmPsrt3VMVRbXBLNM16jO6lkyYzyC48jTdicqpLsHazZn -Z4I6GZoUQKc9WPzSdu6tEXjM6e/2lkT8kaPmrae3JOKnP+lzjZjfplV1NylICNQY -whPWBVGaRg0dy8dZSTGtzygTNMoHS3zYsBGE4MuGZtm/4+x/XLkz32n1k58wAKxJ -JKafNaOReYFxJKd+ML5XnYOVICuw3nxQY+CeVZlz1Bc= +ZW50MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALygXybgZQVWMyk3 +lYQS3DNZ+1Cs4qEgCHREtEKR2MgcE71ShJeWNzZEtPjjG18pCCpc2g3+TDM6fHlS +mCx1mH0QsiEXEZsq6zbQVzA5a8ts1tHNOQKBK3BxDKDS38MScYTOo8nz27zvVIJG +SOPiuZKzr/aZ72yUlbhDEUEuf90vruAnZz3CbcD7COQ8E6XdlJfkCBMR8MGXd4on +HyWcfWU89dRvV/bfw4FU66MQkQlHa/z5DTQIH3Tst9Sdj8lp+w6sS8nPp53yJbAO +J9hq4zUNyUjuNoeO+KFnJ2yETIHVGFPUuXqpMPzSk7AofEwxRHc1xtYatYM7KXKK +Z2L50PMxRHEMbp/xgaSB+xOW/725uB6EqlUNQZZiuyhsk6c5G8alTc1knYhoNIle +yfn+tmG3jLPgV+gIZnIorVtoZKh8bio3KAuqusONnWFHyhLhr4Y9tEMR1lNK4vGC +doqVEPRB+Jm4oY7UQ75Dz1d3ivKktGzA4oSNwxfc3tBccKVV6j7qqPmqIcqkWBZ4 +M8/6ulAvUB+9zEQ/HzajWZCW+PjDXHTSnSVybelzFv+ixBAFt+IGFGNWSJRmu4zW +Oj8ZBMYJ834Ft7/kqfiTmGQzuZ4kQg+4qM5PQpkDnA+EwjrJ/OYeKbIc9kB4WY02 +tKTiziW1hMr0Yk2gdWewmyo29bplAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +BQWFyVKS6BQF+5jNHtyV1mozn39/+YVfHKN+QkNIF6EBBLE69swO4spVCyT0vkdg +5Dok6qomBf1bvqySyfvRNNNbFaUE3ySbUS0Egwtn9NonPZRS1oUNUM7O5f/hmRAL +wJI2nbsvc5XxeWPRpm7VKw9z3Rq9oFeZy6iMbQyRlYKDbUMcR+4x4o0p5GuFqXDD +NWJMAC1iilYiid3vE0Odjsy77yzLnTdzCuelxxOWNVQccF/Dv5vACpRkX7A6GHJS +Cc/hwuBzJxuz9K+oq4bsGDtuFOV4SyIbfW7OfYtVwfDNPjkPQmZbt4aSmpxtD8s4 +00qfYxCq7ZPU/yCnX3Qrz+VEj5vzVpzeeAKaHAeBFtJXzZdI1o+fUJVFmJ4pvNiP +Q4kJbkhTn+NuVQ2GycsGcg2rutcbNYVypP5wrujbWJll1DDB2O6AbCMgZvYR/uIP +SHB1kXvrY0rEGfbWoEQ66mhR+hrLQtcgkEwEJGQ0US1TcDi7T6imeSZQxvIqOuUW +Z4w72gaet20sSczKPKHIokJsEPgM+5dVCsWwZ92bJdADEVcf9jDMtUYRJw0wL8cm +3df2dY2mlioDm/Rj990jbdM4K453vJODZmuSfCX0yFh3Kxa6oAbe8GmuB5hiV9B6 +WxZvadyxvop6y1wUnHYv/q1ZfgufsJ3GaejPEm897bg= -----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client2-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client2-cert.pem index 9d317f07963..5eea9eedccd 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client2-cert.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client2-cert.pem @@ -1,30 +1,30 @@ -----BEGIN CERTIFICATE----- -MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyRuMA0GCSqGSIb3DQEBCwUAMFIx +MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyR6MA0GCSqGSIb3DQEBCwUAMFIx CzAJBgNVBAYTAlJVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl -cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTI0MDYyNjEwMjUw -NFoXDTM0MDYyNDEwMjUwNFowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt +cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTI0MDgxMjEwMzk0 +NFoXDTM0MDgxMDEwMzk0NFowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt U3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UE -AwwHY2xpZW50MjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANUgBLgZ -EjiW3LmjGlCu17h2ERYc75YX+7esbw/iMWMrcrXfvNNutJ4H7hOsV81ZpMEQouHR -fog8InraKCuwmb28DLhJZJlpE49hZxeUJws4yN8VCCTgr/tLtTsvsSS+TGA58B29 -HHaglJwY0w2mOlOVcUxOkjne4VHokLOomhzzqcqTLCjwUslZqRn+SDgXyRw9P5re -J/m6E36dFyBeVglZvealVp4uK/TTqVVFJYBljD22M9wrOeo9AIvrru5VlgDNHu+r -wqgSeE/bGcwhX2J9J++lwOxsFDR7OILY33yoD7/6NN61SJEoRh7xZTQkr/Dc3Hpv -jYyj4YagdWKq2xDZzIxH+QfEMQssuFvulq5L76TJpXB3ceyCu2NEvb/563p1EvXQ -bp5Zjz7Ojo1elobs9epvfbCmiANmTxG8GLKteLXfjcph492gdIw0nsV9/bIzE5C7 -lnff4nEU9E/uEJz0FTw61VjcZPtpqEzLE/8abBU48pTj5HqKo8Nsjx9hPl6trO4h -81yMaqwbQDmza1KsU+CPIFiycyv8Hn4w6JEjwnUc08rOoQ3e7HjqLNpn8X6RirVQ -UrwSU7L8RTKeOCOBLg6AMXfH/frPRnNQjUG/Z7tBTjTJhm38qucxyHI3J5jwX6vn -/jBfdFHfT6510V0Q9fzgzp3H3fyHpnLW1qxDAgMBAAEwDQYJKoZIhvcNAQELBQAD -ggIBAF9fs1tF/yL+eBTf7F/RMdDrb1q9/YZCZ6btJH6CnxCuj4M3o4EkTW2PPSY5 -AeTX0zYaNXvlHT54vjdO+9H3ocyY0HfjVSzttw7qiTkvsssRLLW0PMpc8QMRBpz4 -CmD8vfjY63hKzE2cF5GyP1RveCuFVf7//wM2dfPwrQkIOtKrctejYjn1tOAfgJtX -It+RWvJ8T9t4e3KxYgKSa6eyYxyNMZV67X91C3jIJLgTTLwXXGQF5G8hH3KsclSl -RDE3CAYoyDTtaMlI6A3qDtmvfFKzeltKZc8w7uIbjgHvF49p+n4oh1WwDc/C8SUy -1QAx6DSSW1f470Egtfp0hJKT9yJh7C+/EdeAq8Oh1vMxYKBrtjswCsrFQ+bayEcl -2SzMLez2S/bIFSF0WaDqqIOZDzcjpXjbFlm/px01qoPDk5lkTPGA18Zq8mVc0y2N -R3vYzvfpigjkjXgMcOIfP1Jnlrx1x/4+txR723hUkHQd38nKENepsoEoLrcpmbIl -VAKYTALTle6jJKGf6oZf1TIs09Bc1Qs8Oo4IymubOXD+FlUSmggVwMiST15O5vQu -zdvidRHhAE581DKK04GLmWn0UE0Ko4uaNHAgl2gzZsuJQ5oZynOxmh/z6t+mgA7L -l2qS1WOq29Cq2qWrrfvqbl21LWLrf2X75UyTd3GAlQ19aqLV +AwwHY2xpZW50MjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALU3+Z07 +LANG4lgyVe9pjWQEHyAZn4iadESZqpkcVKFpeHXmEmCvDkt4f+xs/tJle2YmzmVI +OgTrhOeY+3Et9ydfkNM8N1LUiem6i0LQq4yhg8/xM9jeKbOzol1CNM9TbRzZXHKy +PX1Ykics3iRGE3wqwK9quQqYIvBnUT367N7w9kvsgvMr+E825BaQ2hmmefi61vXb +CRO7dyRPIt4z9ERMeAHY1Y4ahM1OwWcGNfhqPnMoscWrWarwIE78SoiUMXd3mVUE +27m/opmLbvk59qh8o9bu5sje+4iKW7P5LVEXWhVnbUptht786f6zTFzWEeG8XUF6 +TFQIAtO5P+QxVmKhsliEt/vCJdTWVt4TKT7c6T+S6ivFl9Fn3/KWkscKLITsILLC ++DKxBracohoROMfzYSGgDmE2d6jnQZ8F4U2SxtGelk1BVYTOLP2y3Rg/xlPB7FgH +iq9OJitIuUIN8yAtjEedQN47OpUt8fMMZHukcslTjXCDgzFpZVh/J+oIULBoRBah +NWsroT45rrDhum/n1W8jc8R7VfLVXhb/YK84unxxGduElbNwW3T0mtDMqHBdvwj2 +pfNmGj/kgtSSuD9kREnIuywyiQCLMM8N3UacMwPZadjFxeugBuKCXX0U4j/llnas +w31ljfEL9EMSuk1a1PpLyPM3VM9guOzX6FKXAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggIBAC369EfL0SqOqGSleQfoQS88UDazzyysbTsNeqKqieSkK7Uf09dtXflg33mi +W+dEvEdcMbhouQT1nIi/nfeFktlhqtrFav5lUDQljVHbNteBQPMWgvMSGbuYlDoo +pxEmvCcof7C3IAzj9btEtMdo99V6Ecp6PfDAhDJWX9VQqvmKiwbpdVhl0Z32o9lV +XoQZvIYGMi15Ny/j6/8m9xeTql7fLq3o7xTg1fOVSJweT8MNYu8zYeohwDOIcLx1 +6mJLdx0DS7ok/7swMECdyd4zwCTgn7wDworS3MyPty5nVOIinlr51NXOKjbpNBTI +41z2NPfX793Qx43Er6zRBKhrRmF10DwiMgtrxJdF0khFjVuMczHZuOL3gVn573X/ +FTVHW/A5SyDmilMZRn+VYbnad1JAy/RqIb7e2EkkPzNMFMbUT8f++HQF6Uo5XALi +EsBiabf5KZ+uF9N9XN/7fOR+mPltc466etUqFCP+MjHRiLyisDYwcAay0V1Q350h +GEWWQj53akIVi3cEV73dW9co391xQ33EGRkqY4+kY6Uh7fKN+roRoyu62e8kKV60 +p3FjCtjDoOFSHknGITSofERfc7TMwBmhFBU1anooMZs2+zs352TAzk/qcnKfmSzV +PQoZSDuUzt+aaj+w4XisguJEq2Hdbt+41a8Sb4XA8+hzXUo8 -----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client2-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client2-key.pem index ed0d179712c..26cdf77769f 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client2-key.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client2-key.pem @@ -1,52 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQDVIAS4GRI4lty5 -oxpQrte4dhEWHO+WF/u3rG8P4jFjK3K137zTbrSeB+4TrFfNWaTBEKLh0X6IPCJ6 -2igrsJm9vAy4SWSZaROPYWcXlCcLOMjfFQgk4K/7S7U7L7EkvkxgOfAdvRx2oJSc -GNMNpjpTlXFMTpI53uFR6JCzqJoc86nKkywo8FLJWakZ/kg4F8kcPT+a3if5uhN+ -nRcgXlYJWb3mpVaeLiv006lVRSWAZYw9tjPcKznqPQCL667uVZYAzR7vq8KoEnhP -2xnMIV9ifSfvpcDsbBQ0eziC2N98qA+/+jTetUiRKEYe8WU0JK/w3Nx6b42Mo+GG -oHViqtsQ2cyMR/kHxDELLLhb7pauS++kyaVwd3HsgrtjRL2/+et6dRL10G6eWY8+ -zo6NXpaG7PXqb32wpogDZk8RvBiyrXi1343KYePdoHSMNJ7Fff2yMxOQu5Z33+Jx -FPRP7hCc9BU8OtVY3GT7aahMyxP/GmwVOPKU4+R6iqPDbI8fYT5erazuIfNcjGqs -G0A5s2tSrFPgjyBYsnMr/B5+MOiRI8J1HNPKzqEN3ux46izaZ/F+kYq1UFK8ElOy -/EUynjgjgS4OgDF3x/36z0ZzUI1Bv2e7QU40yYZt/KrnMchyNyeY8F+r5/4wX3RR -30+uddFdEPX84M6dx938h6Zy1tasQwIDAQABAoICAQDIuNYY+OvTRipt37IaCQF8 -Zh4jgG8ZIk9dJlaXVAYFi1cG+chiLSKIr5lHCArNiT8E4gE1wtNzxYcHw00QEMxL -CL/GFMFdRrw4TpkEePDovbtZdvprmP3FJAF00673lw5hlk+SApi7FPPBrBOiCEto -ixfgsSNAw6vcM7eMrR8wY0AnXMK7b9PYdMwxge5MfgJXyUuNNOvbY6eWmKa+Qnqv -ZcjXYCKa6YtWkr4pY+005u7U9DQViNSLypYoMXlYWFzlNkqLmW3EU1jihMzgFxI5 -tPwW1TpEsGm7H84SVeTuB26F9UUz9vJ4W8DmxZz2JhNaOvifi056BaKS466KlbWo -iZgt57ajj0VmYxB0ZL7QgQqb2xDZL12kU1AU09QAXJnpy/RqvV2HloKbqrOd5h4L -oME6j8vT6Q8o1vsh2zJuLXHAsMr30XK8x1HhLDDzln49gq/d3GrZkNrPDjcumiwI -o6PYR91Q4QI11kdqR/3005wV50g847uURFNF6J4ziDeDGsHqj2phmOIt6d8vWcFo -XBEovCZkXQUSx+4NgAAy1GRBjK6tLRRQnS9bGgkELS8+Jx84NlgKkH3m6+lKNJQ1 -o5SpUqmk1dYnpTv99U2+5qvA/o9SVy56wlfuo+u0GlbMjs3OmItXErg46UBPhd4d -HipFZrBItpw0DYAF+voLQQKCAQEA9ePjYyy53VGLq2+vRx02IZOMQOLaaBDfybtP -51ksHfCPg+XZ4jWsDH4EPn5DeUmjZ2nG8GYSuv8dAQ4v9kArToLDyy/qhXHzaING -uSd6KlTGrVrPK1Dyu2p69xYrnduD6Vm06sJ4olDq792rEj4/hdzVwrtgw+d1ZLXG -3ropWgrHQT8z7+B9CAIAOXhYlKrV7+UdbAod+j8OpCIHk5X3+NkT4Ht7biqzQvbo -pJJILFA1qHi230N9YR8ng3PHQYObYJ6NFBrxhpXIfXwbuPyrEApY3zaL3HbkYC52 -aAI3zy7WOqZSqRZ6aDzXdf2EMGusNSxj9/TAZhTAiJvwHdwBowKCAQEA3eNC/iMt -kmy4R3FQgti0Zq+CBUErMn46pQhBCcQreI/a5U4UT/iY5WGutKXp45d/BM2ztyQL -T/8p+85RkasVF/rJB2PwlzUZKAAq29nGXuV0I6N6EiMYa2LfFLzrrleNamPQ9Ubn -atp0kiyLiPZ6P0+Y5wZMirHlMyup+fzG6xsS7KVy0Z5Fy4YetP63r6xCVQ+Rdu3l -dvXqGb2Bdc9g4OxES1Zj7MKHg0b3ce2lYaL0cq0z3kJ52MAVbL9tQQOstJX41VYv -/QSVIjC5VACSa2qsqzquqxYOyT1U0l/8innHfD/uY/8907/q/JqoO1hU5LtvZ7OO -ZF/e/ycZCM2U4QKCAQAXUIJQ9v6wk3jQyoguEAD/8gOMa3YWA/OUJySOZRAfzp1s -/jBImJo1nQU9/67aIzdRKOBqDuObw3C2lufJS5BPo2p5K5PrD0DrGfdsuueEeAFW -kpOuIcDCMHh0US/ViejaCV10HPhfO5jrIXOFCU3wnV3PVwD30kx5PhsbJz+ggAEg -mKOODRUN21K2IEkV35TlaC3//n2VKsFyop9hSQj4GW0fDdZIPdg9czff0tbxDLHp -xXhhdv6+ZLvUZPfxqE7lPGNYEq3v+ufFrizav2pg3PpMP9nHD6bbz8v+VKeCB4jc -isSvr6fvlkU/tMgB51OuvwTDj/tmMnWG/nIoAqJNAoIBAQDWiLYsS8zzJwUhplDm -winiosz+0Zy3jE6dZBamH7K8NbK6RLzk+YKzPbgSV9yFPeQEu/KIH2SEqzxnh3tc -cWLKtaKK77keKaux/j9yI+RlukqJbrVHNgGVSppQTb096s8DT5Eopa54pNFSx5j+ -Cvn1nrtCm9eDvi7SQ+RrnVii1qF8hxc1z2bCOmIUM7dcNhxIa+4EZE2ZsHjw/EZg -puqPbkE16khhEVC+v+3djJ17gngBLK/atMFkrYvJgmhbFPd1/w8BDf0GENk0npGB -w6/OBez+/ZUGPCR9tDv/z+i35rjWzGVs78tSodvM8qe4AVbLdOJpDLWfHQbaAm51 -EXhhAoIBAQDmZVXAS4dTDAp/cGwPoXOyFxu+UNnGnAKO0S3aJW9qV6E5N7jsLqzI -4eD2Mk6chkBrO4Upmwgx4sLVnDMlHQGvoqpWUZES9k6oIgNZ6N7KXnvFm5GI7mlR -ySA2LftCeSb4BzQmwyX5wcVjOzfB6bkSgEkvuMFrRStSL6+79XZCoh54jBm+fW6g -up6oXa0+lJbyO4Qrx+oWoe2G9nrUJzsjV1Gj1njnxDECCMrmB+X5P4D02Ac2FgxP -rN+bxs0TpvO4fXsvBN4B+/dtF2Hjgo3rQm5FQ/NmpoO5lAs1VZPjVUiFCjhm3Fyk -Xe2nzT23gDTuPny4yivLAMHPZPfGLLg4 +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC1N/mdOywDRuJY +MlXvaY1kBB8gGZ+ImnREmaqZHFShaXh15hJgrw5LeH/sbP7SZXtmJs5lSDoE64Tn +mPtxLfcnX5DTPDdS1InpuotC0KuMoYPP8TPY3imzs6JdQjTPU20c2Vxysj19WJIn +LN4kRhN8KsCvarkKmCLwZ1E9+uze8PZL7ILzK/hPNuQWkNoZpnn4utb12wkTu3ck +TyLeM/RETHgB2NWOGoTNTsFnBjX4aj5zKLHFq1mq8CBO/EqIlDF3d5lVBNu5v6KZ +i275OfaofKPW7ubI3vuIiluz+S1RF1oVZ21KbYbe/On+s0xc1hHhvF1BekxUCALT +uT/kMVZiobJYhLf7wiXU1lbeEyk+3Ok/kuorxZfRZ9/ylpLHCiyE7CCywvgysQa2 +nKIaETjH82EhoA5hNneo50GfBeFNksbRnpZNQVWEziz9st0YP8ZTwexYB4qvTiYr +SLlCDfMgLYxHnUDeOzqVLfHzDGR7pHLJU41wg4MxaWVYfyfqCFCwaEQWoTVrK6E+ +Oa6w4bpv59VvI3PEe1Xy1V4W/2CvOLp8cRnbhJWzcFt09JrQzKhwXb8I9qXzZho/ +5ILUkrg/ZERJyLssMokAizDPDd1GnDMD2WnYxcXroAbigl19FOI/5ZZ2rMN9ZY3x +C/RDErpNWtT6S8jzN1TPYLjs1+hSlwIDAQABAoICACQBPBH2OLdp9Pyq+5H2ucXe +X8TD8oN0lJcwpMmI7HOpfeqGzK3y+fcauOfje/mXeh4Apc9Pu19Q3/YX2hVRmaJ0 +BVVCyIcnfUM38gVTalIloQfGl5OGCGkQzriSt+VdYPdzZD/RlA/8cgVgj75LQgHQ +iLoRjTRCp+Z10Jls6nUdEQJiul3QJaSvdj/ZhcvE3MUDckQkwbfu7iwDoRze27Ba +NRnA2CVEtZAJDroGbOCvUzsUIcXxVn50+SZYUAK01uMymv9eWL2eCpNRNbpEvvIY +2YCRtL7CNt2ZB8FJR6yt5BMOmpNIEZzqSR92xjxHuAPOpgSlZNbpRI6GZuBSwSO/ +04O/xGGCw7H3iOZdUurUd9+liVpNGjCk8vcgRDcacc7ibJTuXPnbNqcrpTU+Ohhw +PDgwOyxAhcpFREvofwHAVG0OlDngEb5AM6E/ocYB/nOPYvkgmVb39gOzoDHhEreG +KWzbs9w+gbt+UZHMnvjZe2LMOpovBZMFwReyuYjljeq8yt8MAi4uaB82I2XDGMhA +W3NcdF0BSNeVYQkxCF5Qrleogu84i+ldxwlPBr3Y2ve9kE59dditRZYY0ghlRi4b +p5ytq00cSog572tKiKOWTwR6HTDqKeS2xiqPtIcuRvI3F5ArautsyPJdZ/oF28UD +tY0iX6gg4Bxylkj3nmfxAoIBAQDljokxPggXH/4q1yAwVC7UHN7PsVjNUoU/oEzU +Kt6QiUog0TbFgY2km0xD+qSueK94IQrywdvBjpWU4GL2musPjGRx4g1BzIgBmLk1 +dQgCvaZNsZHL3K/c9CENb0U7iW5QSnf3l/Ua9sXHSFVXVxFaYZBJYB4JPjonZQZ4 +uAQUN7wgypdYFNzjmOEJ5lXesFyU5uGwr+Ek+3VDnQBqXD9OAVM2dS0hvPW71IG0 +kjoBdTvCKXBqVUQDcn0uv/K4wzP4mfd0RlFXKavfSrfXeijOyqccV7Niui7xHsOF +1ePN36Aaf35xy2PbD+7+oVBgeFLDQkewYFgkZAilTxPsFRE/AoIBAQDKF/uJgEHk +/2oyuhOaBhLHfekUrPZPfrENa24YiDP/fQXS+AwBM2a9kh6WneLT1Mj8wteyAqWq +qVwVmREvIG9iNvVnCAjJGkDESOZZmq1Ai0qVlUMB411z98raYltTLYCJ0Zab/gq3 +/F0c/kWk1kITF30w12rJEtmpiAxF6ptwMtJe1sonx0YEbwn1W/L/c0/B+wDRebgr +cABx/YN1+2LwoNCLEByX/Itra9s0PHnIX178RiWE0WFet9DlnMumCsMSr+sWOzHh +7SbabDDZyZCuE5cWpp83Yi/XcvoWDBzyqfwZTcIqCia4KAeM0OKT8fuLyQXZ7uqw +UYoGt/esFBCpAoIBAH8m1Y0mC/srKHD4WgcLCilJmI04cbTBKbeVhhjbag5rTVki +KZF68kckEjlxQvi1NpR20QaoJJ5w6R/erlJkeyFQpW6DRM6WiRtBSg13Nqm1/MD6 +Q4TFR9A+lXRIlvOkayskJJFLZWO7HtAEm+jV/HGwquhJyjlac4HQrqX9X50HRjhv +nycV30heLQB7ykKgM5fyzz48HZyLgGekNk+zqwj9KOd4Pjh63y5cRkAL4v9dvW0N +4Qu1EQhLyVU00zBOR6JDPlL0hCegmGgFjhFXw/TPQYMsfNuJv2ilnq2qTRWC8b7b +seE8RKJZkAsQtWHKJCWtt+HVa900X6mZRLK29KcCggEAS4jBNDhi6KNqXwFbeDMS +68ssxdSLz1SL+ncP1E18Gd7xwVOCsSQ0H2Cw3Byw/fnHEWoF0nvYzbu1NkFaweF+ +oBfoixTeq8OLN5IDvrPoU3p8awI55EfF0yVnhU5D0gfsxhY3E6DVAsVpS//hqeWH +FrmtygaJ3BOWlR3LcIpqhHoKVPY04r9BdUDHWRR/82h3BAq72I+E/X9EO1bEvuEx +rrx8la108n2q4oCSkqiT4Ad9ag8R0N2tZgiQJHIBT9Mv7BkiZdHEKCxKXFMZwxzq +XZXm38lxMvMY7I8F2B246B4OoqqEis/2ftg/Uit0tFU6bpm4/SamQzaGv1IdWQy6 +2QKCAQEA1Nr2xcoHH1XoSMI9gB3TNWXSEUvT4Rs1jFLabPNjJ2s4y3kEm+Ii1/Uv +KAyL12k7nb5uCpkWO//Seu3LcyUeiOIzUqlqcP1RtpqWsOz+5tJ9okN90x526FuH +vNsv7uzCQC3qXZ48u3P6bEZONc8QMx8a32pHNvvvPjjUHai07EoyMOH1WQc71m10 +3DTqHwTN1drOY/wkomzzCYojQ45KN84noLUJdLxUb8IBBwe3RipCJ/no25zBUdhV +tWMOVjg7f/7iOjygM8EAbR2MsgdmKiLjVUGCML1vtE78PC03OlFCKx6wOL9/pM9A +VXHl/tHRg27YjKQT1VZIyhcSLcxS2A== -----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client2-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client2-req.pem index f36eb94205b..225a6eb1212 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client2-req.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client2-req.pem @@ -1,27 +1,27 @@ -----BEGIN CERTIFICATE REQUEST----- MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp -ZW50MjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANUgBLgZEjiW3Lmj -GlCu17h2ERYc75YX+7esbw/iMWMrcrXfvNNutJ4H7hOsV81ZpMEQouHRfog8Inra -KCuwmb28DLhJZJlpE49hZxeUJws4yN8VCCTgr/tLtTsvsSS+TGA58B29HHaglJwY -0w2mOlOVcUxOkjne4VHokLOomhzzqcqTLCjwUslZqRn+SDgXyRw9P5reJ/m6E36d -FyBeVglZvealVp4uK/TTqVVFJYBljD22M9wrOeo9AIvrru5VlgDNHu+rwqgSeE/b -GcwhX2J9J++lwOxsFDR7OILY33yoD7/6NN61SJEoRh7xZTQkr/Dc3HpvjYyj4Yag -dWKq2xDZzIxH+QfEMQssuFvulq5L76TJpXB3ceyCu2NEvb/563p1EvXQbp5Zjz7O -jo1elobs9epvfbCmiANmTxG8GLKteLXfjcph492gdIw0nsV9/bIzE5C7lnff4nEU -9E/uEJz0FTw61VjcZPtpqEzLE/8abBU48pTj5HqKo8Nsjx9hPl6trO4h81yMaqwb -QDmza1KsU+CPIFiycyv8Hn4w6JEjwnUc08rOoQ3e7HjqLNpn8X6RirVQUrwSU7L8 -RTKeOCOBLg6AMXfH/frPRnNQjUG/Z7tBTjTJhm38qucxyHI3J5jwX6vn/jBfdFHf -T6510V0Q9fzgzp3H3fyHpnLW1qxDAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA -dr0LKtpOa+Xu9PKwnlsM48/ltph4q9+tsu4CeC8XGoLFNbVIALuZZsKZDehTf+/d -bgEtjW8vVnBGAvVodo1MgCHnhPPensDLfyggAULT2X400cly+suGbKeu3kIOlKCs -TQsFdNKOPm17NcpuM1wTik2UT2EWLdzZ25Wy3Coid+ILrf5YZ75djqtxZlYbRiw4 -4IndIjN0bYsn8l6Z8Pt5HdJ1nQnbDZhQrx6FXWZ3eSSmpklfl4O07z0KlXi1Nmaf -OiVcOMvZUnM8pYmNvul8Jus/XmP8x3jSbYzJDNOJ3YV8+OD8DVG3pLM8U1FmjCZ7 -KiR5DNSxZFpHGXhUqDpTrhLgoqGK9chOqPdzU7Mp4taEO9FV8Goc7BCeOKB3Znxb -XDIszs0oBIHO/tsqUwEcWBI0vjyC2pBYQAYK++qwwmvbfWg5lrb7eH1ZO42DU9QD -AVR/5luxImAA11AmSsGf8i+FJ3F63PzSr0uUG7BnTLC03xna7dPdKXS/pGojNVBT -Q5A5J0rB3+4L2mZLE3mjst3t1xHfLW/0RVRqGwz0QUIloZkO6wPN6Jz6l5Q+TgCY -uEks1YN/qlwjHwI3ycT+Hr/sY5igT0OAySo7qa7lN13qTiO2z7eAMDgafNnq34kJ -4OQDCE28Bni0fFRIaqVCqTU31Kei5jbORif2wK81Zmw= +ZW50MjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALU3+Z07LANG4lgy +Ve9pjWQEHyAZn4iadESZqpkcVKFpeHXmEmCvDkt4f+xs/tJle2YmzmVIOgTrhOeY ++3Et9ydfkNM8N1LUiem6i0LQq4yhg8/xM9jeKbOzol1CNM9TbRzZXHKyPX1Ykics +3iRGE3wqwK9quQqYIvBnUT367N7w9kvsgvMr+E825BaQ2hmmefi61vXbCRO7dyRP +It4z9ERMeAHY1Y4ahM1OwWcGNfhqPnMoscWrWarwIE78SoiUMXd3mVUE27m/opmL +bvk59qh8o9bu5sje+4iKW7P5LVEXWhVnbUptht786f6zTFzWEeG8XUF6TFQIAtO5 +P+QxVmKhsliEt/vCJdTWVt4TKT7c6T+S6ivFl9Fn3/KWkscKLITsILLC+DKxBrac +ohoROMfzYSGgDmE2d6jnQZ8F4U2SxtGelk1BVYTOLP2y3Rg/xlPB7FgHiq9OJitI +uUIN8yAtjEedQN47OpUt8fMMZHukcslTjXCDgzFpZVh/J+oIULBoRBahNWsroT45 +rrDhum/n1W8jc8R7VfLVXhb/YK84unxxGduElbNwW3T0mtDMqHBdvwj2pfNmGj/k +gtSSuD9kREnIuywyiQCLMM8N3UacMwPZadjFxeugBuKCXX0U4j/llnasw31ljfEL +9EMSuk1a1PpLyPM3VM9guOzX6FKXAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +b1qAUlTncymAKrwibT26XK8txk2ldN290bqpMAAGIYxQqn6ogjKAt6s2Tz3LOnZr +rKbLBVB6RJgevu8OEJdk+6HlqzCRLFwk10w4+aU+FIkLPMUsqyD0I4cfXoW2kSkP +LOMfjXjuxFE5wjiMgzKf9oIdeKcFaiMH7ThkxbWt/HXnI1VSbLH1Ff/vciA7tShu +Y8drVmps7+yGeNW2fzHkx//vIRLR6cufEZpAnvn0HyHiDHBmjCEPwihu7g+H2s57 +m2B17Pd4/kcuYEvv4pExjlslewBjqAfe5peN5/Lo2rful9oGB7OPwPCO3Ndb1kY9 +VQjjYkw8Ie3bQa/v6MQSfpBjuKDEkpG47mNQXy3n+pfl8Bo0E+FQQx0pE3cWrifZ +Rk+PmNqlaERuJvNpDmXhq22IWuUs5BwtttqKPJTox6e6wggIMXCukXxWE9ovSzu6 +VlNM9QbyfrACvLuP+4YPboXFiBHK4afZbpqWl1uT48XJxg6/caJvazKSx9KmV6PN +zaMFCtyGYhY/G14NWDfG3lW4VBRhwXHxHA3p8Qci3ldVgxCsakMtpnwCe/BLo1vP +aCdMrnUEwjs31wldiRFtOA5dP9TaWXtm57nsOSfL9tbE702wfT4oO8JehvbD1iLu +ozapJfOOc4nC0vUqVXQ+TZb7MWGdsV/t8Lly4qi8Ydg= -----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client3-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client3-cert.pem index 376c85ab8f7..b674ad4bb5b 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client3-cert.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client3-cert.pem @@ -1,30 +1,30 @@ -----BEGIN CERTIFICATE----- -MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyRvMA0GCSqGSIb3DQEBCwUAMFIx +MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyR7MA0GCSqGSIb3DQEBCwUAMFIx CzAJBgNVBAYTAlJVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl -cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTI0MDYyNjEwMjUw -NFoXDTM0MDYyNDEwMjUwNFowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt +cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTI0MDgxMjEwMzk0 +NFoXDTM0MDgxMDEwMzk0NFowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt U3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UE -AwwHY2xpZW50MzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAPrdk2YZ -HRhd4RPfa89z2Ay/TOby545N5n9R+kpQvyhnCbr41afzIU5DrBX7y8cKcvo7y9Dk -Cdd17Xqn4oYliSvVNI8B2nwkBz87BUYT8pNVD+QUc3Jf2P4Wj6XQM4pd9Ntaw7rO -yIf2Oo3Tq58SSjbXYrqAbCYuX4cs/VWLWyq9PapBwxEPeQ756GZS3Xvq1IfCNIKv -NLYRgExctHzkUWPf6WAS0lKydBcCobYvSMfEzPkbBlIhuDRdMmzuji4FefODY4lI -zvnaD7+IIiRC4+IY9xNhdH6s0UobpSIqLCSxOJNYwqhUQt6gNAO1mdJhUirV/XIl -xG5nCGbQS77yeoBLIBEL1t7tpo3/AdEzkR+/wS11dSpcllSj+7BJaKBhTKVZrX1i -gMqUSbiTF1Et9PnAkS1jtUy1w3Ja7FyPWfM8nt/K6vfNRudg/xwY0iY1RFdCXuMw -kPZSr4W+QryGaPqm0RlpCpLHZBOxBDpf0ZcA37ullh5hjXsn5CRw/0ZYpanqrkrq -2cVZLnY98IrJI1QfQhHlDqUP7prR4Omk8C7edXBqQqE/0mqL7AEhppOyLedLFC7W -wUBepmc1bNH+Ho11CZeSdTZfIgwAcD3v6MiMA5kMTRcW6HAHNS309zNJeDf3Eesz -TBXOSCqNBBbk+oW8bxkTLRdHRgdlLT7N6qzLAgMBAAEwDQYJKoZIhvcNAQELBQAD -ggIBAADJZ+I3CJs6E9U2RjIzi1lMo2sYgdcKJS5+yWW8CNjB+DibKfkWjgWvq2K0 -i3hT0Uc6y+ter4OOeIkGtofOiUPekaZsQkPpi73sabwhDVnlki9QL9Ayrd1qDX82 -fMM5roL7w/a+YdKzTQE9hiwPoQhrpj/2mhu7LeYhidSqwzH1anU5YtTKHq3ZrdGN -imhnklcmbqfcNQU0K2l2bu5vuJXFs/v5FCp72ux2p6QDPWwMbwvr413wibt8o7ZT -bBGsQ1MtfJynRVwLGLosn+2t3NPJTfjd4dMEsZhkDY0EX4vbE1/X+K09EN7jPOHe -aJ2AOt3cO3A2EHCR3Dbmt055C6Lb/YR6s05dX4lBT8zY0knsWSL3R77kQoa3+7oR -hU46ydU6K/Kt67nO938WBvFgI81IatRVKVRsXfTIP2oEa0TkwzuvS7nzj3czNU8o -EOa9ixawVYRlEkcuE4KE7x3TcLEGa1gYJDGbsXAfJct1Hur1SJ/rTDwZvlc+qp3o -wWOLtN0mVHEH1OaGlWmeeTuRG16CuTcku2DYiqeuRNy5eZddSuMOag/DKnIN5ZqV -s1GNrpnxPCxd/KFKtdGl+l++Bc9dBmkd+r1dJ/kRGvhul77Zm2xEnGdyybIs64iQ -gvXq8d8ohbZOPxswiFo3p8fbBjWjv0qm3UnlU3P4B3RDMrrC +AwwHY2xpZW50MzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOpzzJkJ +jTA4iBVvh4G+z4QNXoOyFsco6zFllwNNWyIGa6kemP057eSJLnacNNPboHEIK1zl +NinF9Sfj1R9PfdCrVRLnopX/6Vkj3L/pxPS/J+NpAZPWW2FosqkrFm8Ud8/p1AOm +dNNubgjmHaq4xAXPWeb/CQglTxYbsswHJtt3G+0ll5V6oJkGR89TYmko8Hqlgdw1 +3LDuJwlIfjxllLUsKB1cBQFxbeCPbG76F+tW3WatJXtEP7foKC+PZjeh4ZyLwSZr +AhbebrYdSyIwG8hCnAS9Dif1W4xPbyhZZuMe73t9APrdxZLHzmLaSzSOzEqFhEND +ab9fJ+goBWEt5p3qY+aL74jnJwwcneJAcsu8Jwsf5T8XOg6deNfpFTD1vax/A086 +8LzIeNTigoRo0Nt/Df0+s1qAr966TKcxkdBAVamNbbGR7Bj5OQO4GjKWPc3pNLEc +Xh/f441ZLoBoCRaWxIawEl5ZBJ5pwWuZdEVZFZj5oc4OPkKIfG6WkA6gy0kluuvm +3ArC8e2/6tQ7GytPIGot3HDz7QKIAvCWXS7uMb5/WBhw4R7pQ9rSYXLKdh3hPOg/ +GscwskBjQJTQk4XVtj3UQQFX3KgBXKOHg5XmNQRmDYNcNZs45PkgfwXM5copJEZN +TTwo1TDMF76CNA6V8akP2VsBM9hOFcHlHqlBAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggIBAA1n8oB//2uheGDoRmRWXM4o7uPLcjN1JkRoCkj6ysQ79X4qM0y9hkH8S6fF +k/sz3zSnpB4MpA+eNbZPMZQzcFMSVdM6iau0QN6gnDA6gy3f24J251AHLC0itTGb +2uh3f7ENgO/oMwqTR46ATh15yS9RnL5Fi+/coLvvUPpT/rChPbQUgzKOCpV2OoEg +2IDpp0UD7GmS3Xq/CxS/Q3+dKtXlnlSodr+pwx93ZD/ViHltv5p74pkJOXtadQ5Z +qF3+gMB/7qoPsbarkPz+UiLjefqhCD4WLWBkg3WEouejrvrpHL61Wl50nryx0IM1 +funVy0H+DEzLJD7i5l8IvAIZTIZC1Ewsh66bavYpyCySemapTsEnnVxWLG2LkTlo +iRchS5W3S8Bm/b3z6XY19uhOhguYpHBOnRhoykbR+jV8oyrt23r/RegqfMVSPqbz +1rmetb0qCYhhoFwJ2/kattLas1lHEbD86e1WwJ5Hc3P4Vd7JdU5zrNd/YGhV5hmF +1xDCoFzkS80yvnjHrsUiz6r1fgM1YcpsfYuDLhOGVHO8CQywrScGoWe+CUPD0f63 +azylUumnexFZVG9CMCxrePwVktbBOJvMN1IY47P5h+UyStHg4PJ3gHBbTIJJxVdP +W7LqqSw3gynzT3UMLWuYCc/KH/87Cf355hERFLetmTeIY6i3 -----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client3-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client3-key.pem index f88456215fe..a5552f19c0a 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client3-key.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client3-key.pem @@ -1,52 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQD63ZNmGR0YXeET -32vPc9gMv0zm8ueOTeZ/UfpKUL8oZwm6+NWn8yFOQ6wV+8vHCnL6O8vQ5AnXde16 -p+KGJYkr1TSPAdp8JAc/OwVGE/KTVQ/kFHNyX9j+Fo+l0DOKXfTbWsO6zsiH9jqN -06ufEko212K6gGwmLl+HLP1Vi1sqvT2qQcMRD3kO+ehmUt176tSHwjSCrzS2EYBM -XLR85FFj3+lgEtJSsnQXAqG2L0jHxMz5GwZSIbg0XTJs7o4uBXnzg2OJSM752g+/ -iCIkQuPiGPcTYXR+rNFKG6UiKiwksTiTWMKoVELeoDQDtZnSYVIq1f1yJcRuZwhm -0Eu+8nqASyARC9be7aaN/wHRM5Efv8EtdXUqXJZUo/uwSWigYUylWa19YoDKlEm4 -kxdRLfT5wJEtY7VMtcNyWuxcj1nzPJ7fyur3zUbnYP8cGNImNURXQl7jMJD2Uq+F -vkK8hmj6ptEZaQqSx2QTsQQ6X9GXAN+7pZYeYY17J+QkcP9GWKWp6q5K6tnFWS52 -PfCKySNUH0IR5Q6lD+6a0eDppPAu3nVwakKhP9Jqi+wBIaaTsi3nSxQu1sFAXqZn -NWzR/h6NdQmXknU2XyIMAHA97+jIjAOZDE0XFuhwBzUt9PczSXg39xHrM0wVzkgq -jQQW5PqFvG8ZEy0XR0YHZS0+zeqsywIDAQABAoICAQDAMTs48CqTPXEvyW6OS+EM -uw7OrO/r3RCnIIYRo1UgPfh9byA5AJLWpA/V88eF4SJ/RYp7qglEMcvTuYVZYq55 -j2kp2rCphOysa6o5qxSf/X4kLerYiEf1OhGpZh3mdt8doqbrmnqVd3YarD0CrH+B -DnhMDBFPGx4CsNwRSqd40ezJYIJyspj7eUisA/Y9doaGz6ltKY/HoRba6fc4667T -RntEKIdL5f38lv6PViB7M/IZMrQf/kdijrgQLp9s8LMiddmvFsHDN2XzRfdqMnjm -AlxgU7xtRDc/gHh9+TNClSeT81+GmK92YeQXp2yGehr6SGFYr0iTkIomQpSVYK2p -0haIIjQMHlc7E6WVkDELdpAxERgvV4uDN9iEkd4t9oNDPPRioPJQ4bhbMSxCO+CP -NdFHTxIbaDr39OdgqNNE14j7WJsFaCsYXH2NFF8jvwIkPQ3QVMQT/JPGStkyF+9P -5IjFfQ9aEF2i4mAVYiG0DE3NyD/OOI9/uF05POn15H9U+bA9hfBE0Rtm9nMqfVy+ -zgmajXkVb0jTHdL2t/UKv0YdgaglvDcWGFdEUskjJoB00NJwBGorSvcMZiSTxpLD -cGRqywRHOEqNIAbKv0Dt2AX5ZdBSQu7/z1/5Jcdmx8vp9lVhQKeMzYxsFKE4V7fr -ztDuPOlFGyffxpRenBIxUQKCAQEA/XVyoOW1cSFqeG46mjw+dbwjqRmLtEVhAMsG -TtW8pnMJHZ8u7lfM/UJyMN4NQEPJElrABns6I3dqPOwaKOy1a2leHMg5Vvd0/uqp -s5a2fduP6l9PXvhhWDN2sChbeKhl0jJDVnaTO7tiye8ZGMYOM/AlfQX/+PY4QNgd -O7UwcLKhoytxtPtHFZTOZp+cECdTvlmX9lZoNEzFp0nfzFaLVwDsy0B9e6KGt1xJ -fV3Drw7p7PeUyYBNKkyCRVee5S/pn5fT7pkIxMHvaL9BBnWVpwiH3Vi0hfTfFZk4 -8tLcVZgf3n0Y4dMVP2VQRF+kKBTL0coLne36HksQEJyk/4KZEwKCAQEA/WF4z6kc -YXwsU5847+ywq4ipq9efadkMDaGzI6Ez06TQjRYNsZGplCV9fiGxKX2YmZyFzjTf -4joqOmI6UANk+JZKW0Eyyak/TnxugrjMFq8WnK64cIz1TK054tAM/bHGkavaYb8K -bCfbKmaSkwkTbb/OasbQqsC7jbALdbM6Ae0PMrpPmI90YYIMYLRogIaBqCkB43vp -GEZN2VeNS7blhRMiq7YBDXn807aSMQ0+skNSQ7MA8F5i4BFvWyPb1nKZWux1RWLZ -O23IxGWmoGho1CAaEk55LXbqLygU5ZYlBSqkrP9N/elJykOp0LwpjoYBgjMPmanz -o6jy8XIUP78MaQKCAQEAi8+YjqaHosMTDyGG1AN9VMaWSTYdOTC4JI7ZiO0f5hU4 -pw1i/viRy/Y2NTyXxKZfqO9EU47v8BZ0FO0MNRz1qi1yS6Aq+Q0BjYh2Wek9+0j9 -JwSyLKoIUHX694sbggAqQnuVZ4F7EAz6nnd0uZSuyvmiREfl/jgbqbFM1t3IvbHb -tb1GONYPTRlLjZJnrQV0jWCwkaLyUj8zHGeEuxvWOwT4mdmWHnf1pfmTVEM/qTYp -1Zxwh4JtjnKrvYJq1PPMBEvlDQ1/p8FuxbISNXTxOzVadL/0vJvp3ukpX9Du14xV -sA4DhrZAVzsUvtKfI7jtAWlZZSGbwdAYKYGvBn7M3wKCAQAHZWX6YcxTSCWfF0G5 -NyZ9C1Mwke20UEKaz0KEYrs5jVENHTyvFzpk+actHFyogmMG8NuzBjYWy23aIG3l -UgQLgY+QFFogKtGPP/CV3kEO1HOLhUoa9vJeF5xd84a9jQfnzqVkPwhV2d/6392d -byFjDbs/wKfspA2VeDMNb3rc/Yd5CpkyMdXK1tn3pKx8O/Di8Ld+ZWqLa9nv4y9b -q24NsV5MttZXB12K7IRd7C4NVAu9sCbx3T9znO6sMWLEYrn5Pne528XNh0nZ+cGg -YwvUTU+VgzbkTdlOIRRjEzvnZ7RA3H7xT3L49XqqfiOUZnL60vS8nopfF5pn09Wl -erUpAoIBAQDWHJQT+Jvj+dXPC42oIRQKCiYtp1buM8YyL+dJNi7p73brF+2Oqx3k -XNT5eP9GthGqpVGJ732FWJDbPViuZB12zlx9tpGF3ghQTq3p/95KOhGEb2fG7mnl -bEcPqOoFEsAlc4DZYqsDDUvmsifimKm20ZWi4VjTqQJUHYCegJsjrA7D4obGbOxX -FujRMq7/idXRjEoWLloQTMPAQ0Uu4Omwnea25daRzrrrJ34uYrTO1sNgOKk9JAem -rGgrOzsRVG1aNwddcZT/t/icLKS25G7AszLnrNFxJB7DRAfgzpHkJBwNLpcPVtfR -KB6GTGRi7uqGHYScU6+wRMHjdVzdKGNM +MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDqc8yZCY0wOIgV +b4eBvs+EDV6DshbHKOsxZZcDTVsiBmupHpj9Oe3kiS52nDTT26BxCCtc5TYpxfUn +49UfT33Qq1US56KV/+lZI9y/6cT0vyfjaQGT1lthaLKpKxZvFHfP6dQDpnTTbm4I +5h2quMQFz1nm/wkIJU8WG7LMBybbdxvtJZeVeqCZBkfPU2JpKPB6pYHcNdyw7icJ +SH48ZZS1LCgdXAUBcW3gj2xu+hfrVt1mrSV7RD+36Cgvj2Y3oeGci8EmawIW3m62 +HUsiMBvIQpwEvQ4n9VuMT28oWWbjHu97fQD63cWSx85i2ks0jsxKhYRDQ2m/Xyfo +KAVhLead6mPmi++I5ycMHJ3iQHLLvCcLH+U/FzoOnXjX6RUw9b2sfwNPOvC8yHjU +4oKEaNDbfw39PrNagK/eukynMZHQQFWpjW2xkewY+TkDuBoylj3N6TSxHF4f3+ON +WS6AaAkWlsSGsBJeWQSeacFrmXRFWRWY+aHODj5CiHxulpAOoMtJJbrr5twKwvHt +v+rUOxsrTyBqLdxw8+0CiALwll0u7jG+f1gYcOEe6UPa0mFyynYd4TzoPxrHMLJA +Y0CU0JOF1bY91EEBV9yoAVyjh4OV5jUEZg2DXDWbOOT5IH8FzOXKKSRGTU08KNUw +zBe+gjQOlfGpD9lbATPYThXB5R6pQQIDAQABAoICAAMZGKEFEUqL1LCD0sSSIufZ +zV/sIITtM35pmz897HBcqVuIvfjKpSwZ6/VBRyltg4c61mfZ14dhyEWIqy5IvJ7f +RLaFPQ7CXPECmk4m5qVdSUemAZFUicyVt5aorRk2qgajTlvl/TE2ClovwECbRGvX +O5bj09i5tXvTTd+IUKkhv8q4bnJZNnoPLS++KFS/Z74XJcolJA9qdjWXMaPWq8ph +FP6eUqqcNxl6i7JDt8EyWqaarx4b3sOtW6qVOIKPrw2Egz7gtxxaQBhD9tQy0oso +5irh3KgGg/ksq4la9RMXO47kLfkiqROxdDa2L7w4DtcFQKQq5eDTfxGAReHs21lA +XzBgd2/D9X10HQc+t37DVej60kJ0I8G2YqCcWlmcmoWcRN67LSg0Brje+m4ohYEQ +rEz2tPWDIKQBRecsfegP1p07/kFd24PgEU60MSp9D6tOunIeiI66ix6UDxoFrSA9 +wzyBNBw/12IJESJX5ggFA2RrkAA/ZX88oWGlXJHl9SxSq0MFO6t7BtgeInuUigTr +YYexKAhrkz/M6PRSH1fyNd4PYdJjb6TYT7ITarfBU0YR7x6yjTgbLGQrnddbMc5K +THF5KuyImPw0/ewCw063zyC1Ze4XVNJ/i11Gx1knqYsN1XvE/OnbeAu0EyyNjpe+ +GSHXxfd6nF4B/tKFaL4pAoIBAQD9bqupNI5Yo+q4PrlGAjoqvhovmKdjlfbi2Rbr +Zqz9ooOnLp/5E8dwEQy4ofMlRUOe4yH0Cv/DVZ2tR5Z0GVmHbIBKn289cnJw80HC +ljeth7x6k9VUVX2m/2srkjAmPy+fY/iqxsR8Sei0uKAS3v8TTq9BXuM9Woh9W8WM +z4FJtRICCAqnhLewopvxgb3wHoMm28Tug8fW5+WQhAxD0fWNs9lUe628EFD1acqp +SJ4SVAa8pMXl8k9EmLEInDtImJcLFo3DEH+WF/IJF/WXYaNTDbzb6uvw7jYW5zYp +GRDDeReErcrO6nTaRdKUwv6Kjz9H2tZAmJ5JJ09QgpmQVT6TAoIBAQDs0+ZxRUGu +qI0csPBLz1F4YSl63tbYPBS7HWPhBNg7d1PExQRhOyV51OShgtFtB99cgw5uJ2ch +OWbqKi4vQehQEHTCEdkPCpRT5Sj64AKHHNUCgd1D/Z5ZdA+Aef+e3NiDptTWaCQ5 +Eaf53KCm/VqoGBQBctzOAkh9pGvsIIyXzgDLI03i5/aiePVJNSTFfRzOROtmcsyz +4Tup0ntbgu09qigylPJQ46P9Js7MCkuK5V+Bp+kYvUMv8Yz8oknaaHYmo01yn77L +XshN9Si1zY5T8CSAfJvGpgqRLpcsbHryKRYm9uUu9b5TUYi/pPiCPiTUCdV/fnIN +0mmMZ9hykMlbAoIBAGDdYP0+Wj3lAPzE9Jmelk7p9CZHCYuVsRSJdyooR2x2Ji3L +M1fHSI475gnX1JBlRfA/ziVx4sntOmQVnZroaYDKZsqe36yzxqwHPhY6xjMsU0zi +nkIqnukqbPLtYDvuIKyiUFQtvsHmmewhOQWdeA8QHwo0U1SK+uo3Hm2wjjD25Vgj +bLcUkgUQUdxgA5H4h5Zdn4qukb5BSkwPPITbhihQGnwg/YmJDviOI+jKGajVtvF5 +ZS57i/KjDd9Fn58iu3CAgVSSRMHAi0EQiE0BA6Tl9k50HxQqaEAexWO18eNUsDmF +F6Q4lssqrs8vLI0XLU+wg/2Sl8VMIhOap0k0W8MCggEAdyCn/DZAMeErClGeriOc +8Za+TMYnACJIs063XQsY0eDWTFZmO5qK4VvLncq2Gcgp/NkXuyUq5TWApS2Oicr3 +Vr7QXIapzr0dm43cLUXdP6WrKFW2vWqn6otM+O1lPb4NUzvqN+euwN42xqLKVPWA +Uqm59niWxTG00S6R8vb0ga/oCka5+PeBwnxhte97jbO/d6qHVsNMYPddEbGEx6V2 +PNyI19jAgxve84o/37cWBMoYXpnd0MnzL/yrVLE1wR1mwUzHum7MhHQrAbvePIUn +oOVdFxyXJzeCfUEYXgo7VfCA6hyrcBHBKRwMU4piTvF/iQFhWX57VKSQ30WlRQu7 +nQKCAQB0dBfxvhJVMIdWE5IkiXj8ui+86LjYiBrdb0QBGSesYjZ38eChi57X6UF5 +Z0h0zvb95P6D9ZOFwEJan3pd9nwS9ud4/kXfnx8k4KtimOpw/41bjjHja3oliY0t +wgrSvRvJtfAqaUf+2gbbFy+eoiNdtCmu9btnwPBV1h1uVLK7wi53mkrufP0X9Cx9 +wjz4TkDtckjxzxYp1vuGa8SKZsxS+uXK9vaut9cme0gpVqwNeTq9yQD+GvIcpGs3 +eOXqHRMuyClyt2/jInwVSAJPq3hgGL+Zn+DAa++VhWbHASh9m//ZdwEtUeIiyHhM +Tea8lz7hwFEIIR1F88FY6UjN1dIn -----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client3-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client3-req.pem index 7c679b4b367..0163a020c5d 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client3-req.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client3-req.pem @@ -1,27 +1,27 @@ -----BEGIN CERTIFICATE REQUEST----- MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp -ZW50MzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAPrdk2YZHRhd4RPf -a89z2Ay/TOby545N5n9R+kpQvyhnCbr41afzIU5DrBX7y8cKcvo7y9DkCdd17Xqn -4oYliSvVNI8B2nwkBz87BUYT8pNVD+QUc3Jf2P4Wj6XQM4pd9Ntaw7rOyIf2Oo3T -q58SSjbXYrqAbCYuX4cs/VWLWyq9PapBwxEPeQ756GZS3Xvq1IfCNIKvNLYRgExc -tHzkUWPf6WAS0lKydBcCobYvSMfEzPkbBlIhuDRdMmzuji4FefODY4lIzvnaD7+I -IiRC4+IY9xNhdH6s0UobpSIqLCSxOJNYwqhUQt6gNAO1mdJhUirV/XIlxG5nCGbQ -S77yeoBLIBEL1t7tpo3/AdEzkR+/wS11dSpcllSj+7BJaKBhTKVZrX1igMqUSbiT -F1Et9PnAkS1jtUy1w3Ja7FyPWfM8nt/K6vfNRudg/xwY0iY1RFdCXuMwkPZSr4W+ -QryGaPqm0RlpCpLHZBOxBDpf0ZcA37ullh5hjXsn5CRw/0ZYpanqrkrq2cVZLnY9 -8IrJI1QfQhHlDqUP7prR4Omk8C7edXBqQqE/0mqL7AEhppOyLedLFC7WwUBepmc1 -bNH+Ho11CZeSdTZfIgwAcD3v6MiMA5kMTRcW6HAHNS309zNJeDf3EeszTBXOSCqN -BBbk+oW8bxkTLRdHRgdlLT7N6qzLAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA -WLhSuFZ6pnoDe8LQx6eMPXzRkQb1qsJpyjpegUxFe71o2e23V/1yMnTfFiO+DsBQ -PP8RkLWUKAvkAvqPyttJBx9U5ZYspsSsTVhPsCjUFZ4IG+fc/dVP1ZRid5HQJz2+ -bFf4KPgErZkJZR02Q2q6ZpKq9clRzbDkho56OZXLYI/o2Z4xADbhzpa0xt8sx533 -bm0rKvz85WxH3cimRjKaGKzuKg38ZaXmmUbsigV3dzImT00KDWmMmaW9SB8lIm2R -JToms0Qs+mOr9qD2NiRoiUd1wmgG2QpFDViIqAZKJjjeesmeV2CAcPfLztOZBim4 -6bRIOIXDhYYOyDgs52XuijXUr4BR8aQmqBrjnccCMcGE8Ol5ZH/IDg4pCRSduCWe -T7ThhH7BpAWYdgF3ITcp5oEcpXK8IdAMAst1/6vk7Z1JHIOejxksbLsGDYkaLM6w -yTn4X3Ak0X6bVmLAY+xAL/WjAJhVtDPqGYAmpx4iQ6QjYG/8gRdOiUI8H7MCK8+h -P0auhyyMmO+kdhNnzwuX/eeLXZfNvnyK4n2uHWYgwV5I+Kv282zw94UIQgwVQ2DN -/IbXD7K57s7+ff9Eff8L/B8rt1i1cmv01mEgQ4kMsLOClGaceGcz/ivfzDCosmsk -Xg/zVmdunUY0lswYL4SQM3BhWB3xJ4likHikfQHklM4= +ZW50MzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOpzzJkJjTA4iBVv +h4G+z4QNXoOyFsco6zFllwNNWyIGa6kemP057eSJLnacNNPboHEIK1zlNinF9Sfj +1R9PfdCrVRLnopX/6Vkj3L/pxPS/J+NpAZPWW2FosqkrFm8Ud8/p1AOmdNNubgjm +Haq4xAXPWeb/CQglTxYbsswHJtt3G+0ll5V6oJkGR89TYmko8Hqlgdw13LDuJwlI +fjxllLUsKB1cBQFxbeCPbG76F+tW3WatJXtEP7foKC+PZjeh4ZyLwSZrAhbebrYd +SyIwG8hCnAS9Dif1W4xPbyhZZuMe73t9APrdxZLHzmLaSzSOzEqFhENDab9fJ+go +BWEt5p3qY+aL74jnJwwcneJAcsu8Jwsf5T8XOg6deNfpFTD1vax/A0868LzIeNTi +goRo0Nt/Df0+s1qAr966TKcxkdBAVamNbbGR7Bj5OQO4GjKWPc3pNLEcXh/f441Z +LoBoCRaWxIawEl5ZBJ5pwWuZdEVZFZj5oc4OPkKIfG6WkA6gy0kluuvm3ArC8e2/ +6tQ7GytPIGot3HDz7QKIAvCWXS7uMb5/WBhw4R7pQ9rSYXLKdh3hPOg/GscwskBj +QJTQk4XVtj3UQQFX3KgBXKOHg5XmNQRmDYNcNZs45PkgfwXM5copJEZNTTwo1TDM +F76CNA6V8akP2VsBM9hOFcHlHqlBAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +m+9LdrSuXxt1vtz+sIqotYTfWOlEO5yDvxYIV5jS9BaCGrexbj7ku/kaA7AvnosA +ztEb4ZOsiFFyETSYRR6+br+A3BHDBgyiKdilEhH7AzZYPxDmkUmmYWRXeia5o1tu +qQWWoB4pncEs303a6dGY4IFjbYTCu1UPP1A46da4xuDmsTVf1OE93T3KClEyyAZ9 +DCbAzEj0Mc0iiJtEumP71nl5xr8xeq7sQ0Qf2YQHG4svV8TKNwQSRd3taaFQ82H3 +gxqL7u2wS9lk2Xg5BOvcpzGVXjxw5qwNGgL9G0g8FiAT6fWi2Wxl707WOOUx4uKn +wLJJfO6q0ojSlsd8JXlPXfvoVKIUmDUKIwuz6/UgLpDdm43X62aFyO9htSVVK56L +OawMXFHLSkf1SDHirqhhTIDVlOCM4o6KjbnEQhjU3l+kFmKbAUQ+wfDOO16i/emS +tkuu1FRouZVwwTMXeEUciVjadUSrBJr+balX9fjJdqTDSTc1/9ytkQ66h1JlJfy9 +U4T2hZTzUlMKXm+x99qxQyI4fG97zgoL0vU2tKJf/5dkqTki5b66RLtiPbuduJtH +Sdz7GyuGQ/H8qzgAQx55qWA9j3AQVCZZUr7mA02Cuv71YnwpqW1uHM4re48+pKKJ +Uys9m1yS1AEloa9P+2qwLKylbsEzO+UxFXZYImfCLRA= -----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client4-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client4-cert.pem index 5eae58da627..175bca39128 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client4-cert.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client4-cert.pem @@ -1,31 +1,32 @@ -----BEGIN CERTIFICATE----- -MIIFWjCCA0KgAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJHAwDQYJKoZIhvcNAQEL +MIIFmjCCA4KgAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJHwwDQYJKoZIhvcNAQEL BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwNjI2 -MTAyNTA0WhcNMzQwNjI0MTAyNTA0WjBXMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwODEy +MTAzOTQ0WhcNMzQwODEwMTAzOTQ0WjBXMQswCQYDVQQGEwJSVTETMBEGA1UECAwK U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRAw DgYDVQQDDAdjbGllbnQ0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA -353z74lGXkwEc1n6r/M0FS1XTXhoVGMYIqK7HPCBOEGLeyyGwGfSQ7lRfT1xSkii -zBGG0Nod9cRT1CAewOSJ6BjVfkQcGEjlnVYm42nD6PMd9iFJj9Y5atPeFNvvr+wF -OFX+E8FRu8u9aEu7MIj+KCqoqBukFhFgJYX8sMbRROfLOPaCq0cSC+Vod4qR83+W -ITrQ5n8+/CC39uLY/oKgAKdVnmff595Uy76BVdYzuit1IRKwJxqIWMRrfNI+szmS -hdj0AHwgmwEGCaTNcOQyqvBLxW6qB5tc1FyV4LYv4iNftroqNQvlUbJ4UqVr55Fh -vZ38C1BQ4sWgo6FSS/B6u13clwpRzDh3H8tOMTTz1inUtg61Y49p2G8k3kNVH+QU -fRM4xvCkhFzIArgSiJ+/YUKboltSG5K28pegkk8RRMsaQK8g+NScKKu7/8ddRGE8 -454AqxPpzASij+djM0vxzgad6BB4e+iIVdj77NSjAxVAfg9GIjNHG1DZ87jLLgtk -SN2jaYsBRBRnmenslEGDtwO1SeWrzzicVfP9GRdiMLJwCkwv5P5hCzwrCB5eUPhm -tGHm4K8eXDAd+Ol9pKMySC79E5/W372wdbaO1fcAUKvpHhcRZnSusNAJkLGJYCkV -2gzTWlaeX4rGqjNVs4MSmNuMT+a0IafZeZivptxdgLkCAwEAAaMjMCEwHwYDVR0R -BBgwFoYUc3BpZmZlOi8vZm9vLmNvbS9iYXIwDQYJKoZIhvcNAQELBQADggIBAFox -3myVDr9yJkrF5+vB9gUlTv14JIPRd0OFCLcPOlHpvYKEwjRjTwT9oL3zU5PoRPX0 -AiD9sL5TOo0zraiFPUi1k5X6SoW/qU/kOJ/j5CgfChyyyit/V773LitM/cVXZGui -YX32V1zV9+RaCowC/16oHvfjMA8xNOYoYW83FgQ3GrKgRuqqVMT7JAHoDebVSqyb -w5W0G7RH3hHM1nCv51tnT1SZDn+qRBcX5faPUVARzdcRrZ/VSU2RoVIU/fPPiet8 -5TRioZFslZaFDWOLOuP0ZcOj5MsY3vQZtx2/NRgNc+iLF593YBUhRJYqfT5ePW3H -LwbZp/Rvd2kLucYd/W9WhKEzJKKvzm1V2hCDnh5dl32sZgdBzrdKzgyNB723cLR2 -cHFTIEj1Q/scay+iiSoV+VNfMSDQ71vkHqFHNhEqPFUpdF/SeooDFeQaDvYkomgr -Z9BJFtbp4kZRIEuPX+niTi0S/zwi7htiUn17wOIBcydcgG2GXBer5H3JyFnCXM1N -0jFQsuBFRj8xP71xzhN8YjA2Pe+MGYrMWiwaVMLTz8mdQ+Y2aEvOkfXFSaeNUqW3 -GYxAjEkhVCvzhOd6sD3QjLRX2qhwh8NJCJDbkTok66hno8QsHWASbaDiCMG9z7le -ci4dOHzu/buwqS4LVTmFWTn7mkd+FAlSY9Hj0WVI +2fVodgIdjzIVOxWDmdmdC56f3JOhzYXrYPP8JHyrSUQqL3puJM3/J1JBQvQF1Nbm +fVR/FmnI1FT2HlqWUh8M+G6d2a1iP1I3S08glSghPzuD6rb95o+e0f8ghWfpNt6I +wpf08wWVGWS8ONBm0BVIiLJ8rKb2ftI2H8VmCGIj+xGNXtNNdsOmkpIQiIHtzgw4 +b9yH0onBA4F13TWxOUExgcyXIeZICxR3CHHHJEyDKzGb2iKFaIzBeXxOrZs2aoYF ++A7F1J8O8CYaDU/p22g15hYPbvZ+K3idemzkkZVX35usiaa4Flo3rkHRMvFNGij0 +vvJbQsbMBAdRKsQwt5pP9LbGPywFDqxDCVx8FdFbtBlWpXfZLDSKNOUNp+Bojh+O +luJF+9qRq0ISj41nAf3tAB7djn6bvVpoLJ4Yxz90IDm4KoR3L2Lmd4xscghRffza +apcHKGqjy2G7nkGpxD5itqErzMZ/JgSN1Ue91afOtHO7dLm1nSHnriLCtW4v3NXD +FZmqz2APsbRWIazgOay+5Ai2FLRljN6ieZwMELcIAofzGY/BeaDv5hk6Rj8OBi1f +eIntvDJOXj9le/lOB8P4eW8WxiArLYktaD4BAJTIkEUvA0ckjPqkYN0AN8ICuPQ1 +Jhc0Q3bMSA5FEZs/Zj7Pw828q0/2PEABakLsgIS/A6sCAwEAAaNjMGEwHwYDVR0R +BBgwFoYUc3BpZmZlOi8vZm9vLmNvbS9iYXIwHQYDVR0OBBYEFITmpe10c7Is+D2L +GCRFfgf88duQMB8GA1UdIwQYMBaAFEaMoHhzHQlU19Afyjh95bCG7o1EMA0GCSqG +SIb3DQEBCwUAA4ICAQAiklhYl9ak9up3UBXyeayXVYsX5WJRP3xpYVBN2LFRYsTj +gWv/Q4N8ChegS1xSploVH8jkQorBep+zRj+s1u2k1NCxCAaIIakG1BNN2/W2qBBl +8JLnW4hJwy/XTLitmobqcFE9WTsi/d0V0GrsT9Ojh+Zmx3GCNGS2VWHJqM5S62d5 +iJycUIKEcWy9h2vjyKb2cAEQ9+OOtl8ODonfBIu7JwpUFAqWiBGG/r9fQCwS/9Yz +OCljZqaPi6w/w+04dA+0xRO86kd685QSvkD+hmtm8DzYLdI07gAiSxXD2sMszTF2 +C8JDKVxaIW0SV2mP2LcPdKEAKpZnW51rG4QiXjWRrDYluRAWjhlSHFVHau9vJOSM +6rCHahhGtUldaIGvruEN1TuEAuxQXupeBgQDMq/OBndrHSHwTRSEHzprrEvdFzuS +rn6MVx9f6VyEAsku23nhFuidXKRObF30b8T6t7T5JOE/KmeYcE0YGczob1YEbTR6 +rSDcOn41ymOQ4Jo0NRpe3N+Zz8USh2xzkFTgx4sxwKfyhh4peGya18klkc+fZqho +CMLtvkdoK4w6/bXxxGviakIe8OJJvKqKbSv43LoIDYgnYoiFVl1TRhvVe0uVothR +Yh3/laO/+p4WzPjFUHUtFo1yw+vOOsfA1++FvODgZa/p3DaTH7VKgRw6PEZQ6Q== -----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client4-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client4-key.pem index f1f17525a51..d3587f78e63 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client4-key.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client4-key.pem @@ -1,52 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQDfnfPviUZeTARz -Wfqv8zQVLVdNeGhUYxgiorsc8IE4QYt7LIbAZ9JDuVF9PXFKSKLMEYbQ2h31xFPU -IB7A5InoGNV+RBwYSOWdVibjacPo8x32IUmP1jlq094U2++v7AU4Vf4TwVG7y71o -S7swiP4oKqioG6QWEWAlhfywxtFE58s49oKrRxIL5Wh3ipHzf5YhOtDmfz78ILf2 -4tj+gqAAp1WeZ9/n3lTLvoFV1jO6K3UhErAnGohYxGt80j6zOZKF2PQAfCCbAQYJ -pM1w5DKq8EvFbqoHm1zUXJXgti/iI1+2uio1C+VRsnhSpWvnkWG9nfwLUFDixaCj -oVJL8Hq7XdyXClHMOHcfy04xNPPWKdS2DrVjj2nYbyTeQ1Uf5BR9EzjG8KSEXMgC -uBKIn79hQpuiW1Ibkrbyl6CSTxFEyxpAryD41Jwoq7v/x11EYTzjngCrE+nMBKKP -52MzS/HOBp3oEHh76IhV2Pvs1KMDFUB+D0YiM0cbUNnzuMsuC2RI3aNpiwFEFGeZ -6eyUQYO3A7VJ5avPOJxV8/0ZF2IwsnAKTC/k/mELPCsIHl5Q+Ga0Yebgrx5cMB34 -6X2kozJILv0Tn9bfvbB1to7V9wBQq+keFxFmdK6w0AmQsYlgKRXaDNNaVp5fisaq -M1WzgxKY24xP5rQhp9l5mK+m3F2AuQIDAQABAoICAAQfLTflF971F7/okK5dlUAu -rcVHyuSDTxaUWU6XQEqBKskCcRlq0H1fFRlx4Hy2Cgoo6ItA+fxlugXW8bosfD5C -9ux05O+tqE3WILFgabQJhyvaQTjdggFuFlHcG/bqKs53B0/l6FPF1Z/uhWzHmaez -4Zf3qnadq2AFsDqx73mNrDlIkfAGR1bgy6QocbhDSckjBGa7QbX0BHAQjl9imQBq -FTHuSDpF5to6kLe8UwfDdU0+wvB1lL3OIQ0T8wPqs8Cz1wuLPi6dPjc/SmoiSqzL -8RmaiJfLTVK8wiZ6NTe93y3HELAZoAh5ea5MTkjebSbJmrO6r0L+0Y8ykgnETP7O -Ug9PWeDDE15sNXIQCKtRe3QpHtJaoAJU1MGhNqwm9oKMcuSvBOV8XRuZORinTYRL -Q2ZD7czaT5VZXCQI4uHwE+KIlQF+658c9M9WETxClgUlhbzqig3ilUz3QUweaPvE -tqArjiYLsT2KtrgmsZ2DaDn2IlGSIRjXMZJ3kn6i49C0uhH2YkSZgU9/7kH2myse -3opxE1EbT4ARFWUbGgqXTOc/OSb9DAsxUK2u0eR/4yOMLQJkvxNJWQgwmi6N18iU -WdvTphNtMtmdsAhst9luwNaeJItzTDm7JeWx+MPs8f7PVOOkTz8HcBAvZnISH1Md -0i+0lBrBXbAcRK5X7tvhAoIBAQDwKPQA0uNk4Hemt5yke1jrg4B03OLimHeQ/1PY -I99hThh/RLncYaMxqsd5WkXXbjsWyGidKHYh3/cG9akmgU6D2Z16CFNMRhgBgRX2 -+LJkdS2QSuHPJlB9ERtOOiWFt7IDafB+tMKHE/VRQdxFRtvLe6pQMzP4veVXZsq8 -NNJGAQ8egUa6HDvkXzR2VDf2Kc61t4ZwT4JT6C12GnCfvXobaVkU1aWhcguoX8vI -o3UOkeracEKc/80ZRdFhA/nvPPXCobyjFjLi8WGp6PUySrVhN9eAdZaUBNeFHLdg -8urNvy5Q6mBAByEfHZeJNZbBeAEAw7S5YAxgL96blj2IPOerAoIBAQDuXazpCDXD -dG6XDZ9FS7MKBWriHUa54UgbX4JfQsOOFA8uxglIe5E4IKFSEbetlDORnKZjcmGa -SKTm0MLLi/kKrkhoDgi6HNbmbo9ZmKIhmEwws5L1TLeUdLrWj8GLgNphotOBKs1V -vQQkfh6rzovyFsMj44Xea8Kgx5ONVlB1L5pEepKdIyDRiQfxhwFox719HACSqCEa -06eFNGtUOLLqNMZhpur59bqgiVQtIZKis9juwzZID0svXBElpDrNvbWS1V5MCBOT -6AStW66YkmVWFCn7qQmNqMh4x19GveW8ajgrBSr/8GP/WXiACBDEsunWRORW57iS -KiPmC0uHlMUrAoIBAQCYTrCokRZLlJvtbIb4PY3gFw7xjmCJqn4xw+wNqHpzgI7C -r/hbjsRrrE5DZP/kJ3Fr+n92JAH/a8WDcWrsE5eSwQFBMmR5e/6ffZlLft/MHBBg -cU0SDc9/8chqbS/8xMotphMymDrCZeLvvKAQg2bDftM9d6ufNfdr3bH3eFxery9C -fmQ3hc5qAAMKhFDVWiBRWGn3ckVKJ3Ylb5E7jXQSTFaFgxU+9U/1YYOg5CFJszrJ -e+aTIRuWypOGPnpUwkluPRqgJ2TwTntMwYQ3d+/eDwcp3ek4SHXSYqrd3lERWQzr -niiakqrry92d1BGe8xdXv8Yuxn4yxkkcTUUK0O1vAoIBAQDLY0LW1BqL3B1A5m6w -QhdSxaydoz1l/cP5F1W20tDpulP6JSBmqIkQy0bbMCL6CSq3ZGLVGBQQAUwzZo3Q -AG9PncZKgy8PHux/UncejA5LfBgGtjL++6bpFXEXAzKyRhAQn065ODxcnBucx8CD -+ImQ17tKNClVz70SUzijsLKWSzfmlm/jhMXMBJCyle+t6EDXL72NZchZi5+1GTU7 -d+Wx0bY0PKji/7luob8hgzQLgEnp8MewVNxiXLyE0c0bIHR+BXGgjoOmAKN9CG3B -4ah1+l6YTXPJW+syo2u4gPA2BKxIiPBX0laA22bmV/t22vKL0dzECpSCo1JeR+T6 -mwZhAoIBAQDpLqRLxfZk2TK3wJ/bloXXsRg4TjSQ4m2Y3htVRiOQF83iERVRlAwg -9yKlyd99ux8tlYAK368Q+FFAwYTvUyghmfVTPARFTmeX0F5u+MX00WOa2FhPs3du -+ImYeQH3hg2O7qyVDMCwtqgIIuGLNwsVPqUUF5bx0He7wTwwzQmx3EVCOu6yZXG7 -Aw3qpOM2VhrtWgGP1mTONiUg5dh4sGbXX70gjG9cUpo/Owr69Q4Y8/OEyx9bzqSW -5BeVN0vONzQC+LHG5EvgNF6yOU7iCkuoDirZUrVchuAf+IDapK85TLIH8bm57LKN -Etg/x+MCoqlEQBVgnY7f3suMB89XerER +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDZ9Wh2Ah2PMhU7 +FYOZ2Z0Lnp/ck6HNhetg8/wkfKtJRCovem4kzf8nUkFC9AXU1uZ9VH8WacjUVPYe +WpZSHwz4bp3ZrWI/UjdLTyCVKCE/O4Pqtv3mj57R/yCFZ+k23ojCl/TzBZUZZLw4 +0GbQFUiIsnyspvZ+0jYfxWYIYiP7EY1e0012w6aSkhCIge3ODDhv3IfSicEDgXXd +NbE5QTGBzJch5kgLFHcIccckTIMrMZvaIoVojMF5fE6tmzZqhgX4DsXUnw7wJhoN +T+nbaDXmFg9u9n4reJ16bOSRlVffm6yJprgWWjeuQdEy8U0aKPS+8ltCxswEB1Eq +xDC3mk/0tsY/LAUOrEMJXHwV0Vu0GVald9ksNIo05Q2n4GiOH46W4kX72pGrQhKP +jWcB/e0AHt2Ofpu9WmgsnhjHP3QgObgqhHcvYuZ3jGxyCFF9/NpqlwcoaqPLYbue +QanEPmK2oSvMxn8mBI3VR73Vp860c7t0ubWdIeeuIsK1bi/c1cMVmarPYA+xtFYh +rOA5rL7kCLYUtGWM3qJ5nAwQtwgCh/MZj8F5oO/mGTpGPw4GLV94ie28Mk5eP2V7 ++U4Hw/h5bxbGICstiS1oPgEAlMiQRS8DRySM+qRg3QA3wgK49DUmFzRDdsxIDkUR +mz9mPs/DzbyrT/Y8QAFqQuyAhL8DqwIDAQABAoICABRNlMcACHKJLErI2hhFRKig +meyeq0utULvtzmGJuXJFkWvDIFUjrpkiQ5EHO9nT5jxA2DvQFn9CUNDgCFuq/SF7 +5blcD32th21norXc5fKcH9GH7UaCahLOt1/7Siase8SAtOEZy9tF3nX+Ym1ku3vR +qzpxV8nVhBQukQQb5up20UVG3VLrA8O91nYY1abuzX0MvRw4F8zfTN7n0alEhjuR +7CF3cDjpVZ3pl1mCxRSLdk9PYupWc5lWTS/vzZWxBAeo50sYlqLOhyO7+3akPPfA +u1GkpXkgLwSIoJFifivcC6pLkkoGv7lm886mhVOirWByIV0q4+CbhoyQzPxKuW16 +vHSjqwQDp0EhMjk1NZ1+emTOA37E8vtE2jVCLZITPwEiRtbwdl9MjgGjQKedS/oI +BJ5sGFupfwzmY1rAjkC5U014LQVOPhvXzPxwLAH6szlrXlUOJYtG/D3qlEEnW3oT +c/DfJYhEmyhaWDnHC72c74e9E4JjpMp8cbJt9uCQd+rxd/FPtzaC7zi9+b38aktH +gp9UazWOjNkBYdxk1ULn/iVf+5hxsGYuOGfdeu/62bJnNev3YTiV2IKeQRREjRgD +AG560k/IMN/aYwJ+M7hAYfUwHA1oW5CwuzdVl1bBLe6iXRdZTfqFvR1dHTJI3xRP +6Qbbo1pSZGpsGTT84Xk1AoIBAQD0C5c8A3Nnf5j/dqRS6OVUo+aTZvztyydwo/ae +AT+Ju+InmZGvBd8qpyd0pJlGPgM1MJ+74kYzLvmFrUdlCX91PfgG+1PEaK1Yh9r/ +uMTOp62K7x65/Cp6LFXfddIywB8RCDn1EfVmA6mw/Q8lAaI5B764WHGwcdncVzhH +1omYBPXz2QzOAsGh9Z86RvMvkQI7pb1oEpp5n98MxGXZq4wqvqhIYcXfFfFSAkGg +DyMm4DRKUpUMp+fE4hjD0DFAtcgOFCog/kcMsIR1W24+/R5mtwt4zet8m7EXh3yD +xtcggYT2IPqdvYYpNqRs6lewawH/jnni5oaJr9ZJWJ7rKF5/AoIBAQDkoq6V+0KF +r9uckkaSAQ2fnMDfx15icK7J6s87pTY+RoonhzsvM8YZFYgfmKtaR1y/A+8oMzBj +g/5C2nqUjBuIbwmT1cWpoODGI0yZ4tltDHf8LSehvjvws1RCKH6nBBr3VK+BFcSj +9ZX9QeKTCSYiMUvC2A08wXOipWCF6R2X1q3/VtjGYEsDcE+vbn9PVzMneHvZT7nX +u0kinRbg5el0ifclzMhT8SawSMf0UZF+Xzt89nwV8CgA2EGmkn8aVWD7AhYBz99X +6jPt/Ny89RZSpRtiWSrH7gO7U9c6Z1bWA6IfitKMqwuutg2lUUJT77KXPVynLhXu +9FwMYS+57ZzVAoIBAC2qKY9SOfro1lEe+xqEd70Bujhksfkw5V1fYNsp8qCsJmIw +iyEL4TlS88AQuJl7KxB927XFXktfg9MunEhcoZvnj4yG5KLAz4bcTO9CvJjKqV9w +5Pf6JlycBSl+G2nhpy6Bso6X4KpJDyyN/ZJ4Hsm7GEXYTtWsSA7Bx0cx7qcibn9F +Yu79dwv+MOAvvHtvD9LnbWyW5o7RimQSL9iuD9lDuatpmCPpMDGfOHZhvrscgsxR +bZBjrquZfvbM8xpfqWJxBX0M93B5ax+mjxd8N38SyzO8nE7Btz5m3Lavszti4OVJ +QHvZpcv6bIPwyZ26ODKAj3AUoKIM4/TvfzUTZbMCggEAeL35GTex78vpXIxbPGvV +9/7eVM+C5tW/WN7qRgN/2++AslNWTB05drHKS3ViuLCLKE05la9C1Ks2MbQTnDNJ +rwmEs0/nrSdKzRiDMZl9XXuCOejWzzIZkYoC61VbXXRrkUZl0PPf/v6JEVAkCaaB +Pvltmx5iDz3ShYh0qwNrH26+QORqYokj1FMRogHmtc3FNAkKzr6t1GIRw3fWRzrj +ySm7HXlCKiJmXh5xinMxNN2yGtJIwYUsexW5xfADs7J3HtgBKz/OIS2L8xjgseF0 +wnxzM8MHNsJZsioatbsxPqB4k9PKUYNbDg8SscVyE8W4OOqs7ZUG/ESL//WrMLNt +aQKCAQEAx9ZBfjVVqmMVPp1GV61+B3LySueyI7UTrFDxKEGPQAzcLxQr41iWnkCy +MNDn1E9fkvNND2pVqKC88NZ+GDk62W7j96pUP2AqxIE3lKjB3vFAWdH8izWQwUWp +P+/yKYscaqo5wAgMNV2W4lqZUmQ6QaK0rJU2SKaXtjlnUD6fbHhJaYUWdBc8VmTb +FkYGJnpOPuP0eo/ijThdDpCVTUnekSCEdoinuUmprBv04ypn2RCjx4dpHHXSK13M +X8Gnn2API9//cDuuv8TjV0UG+yOz1aHz6ciLpQX8AxBbgetkdf/jJTQFWMiFHMxU +LBnc/Bj54Ydh01rOez6w3PkOCAugFQ== -----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client4-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client4-req.pem index 224484f6611..5180a8697d7 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client4-req.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client4-req.pem @@ -1,27 +1,27 @@ -----BEGIN CERTIFICATE REQUEST----- MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp -ZW50NDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAN+d8++JRl5MBHNZ -+q/zNBUtV014aFRjGCKiuxzwgThBi3sshsBn0kO5UX09cUpIoswRhtDaHfXEU9Qg -HsDkiegY1X5EHBhI5Z1WJuNpw+jzHfYhSY/WOWrT3hTb76/sBThV/hPBUbvLvWhL -uzCI/igqqKgbpBYRYCWF/LDG0UTnyzj2gqtHEgvlaHeKkfN/liE60OZ/Pvwgt/bi -2P6CoACnVZ5n3+feVMu+gVXWM7ordSESsCcaiFjEa3zSPrM5koXY9AB8IJsBBgmk -zXDkMqrwS8VuqgebXNRcleC2L+IjX7a6KjUL5VGyeFKla+eRYb2d/AtQUOLFoKOh -Ukvwertd3JcKUcw4dx/LTjE089Yp1LYOtWOPadhvJN5DVR/kFH0TOMbwpIRcyAK4 -Eoifv2FCm6JbUhuStvKXoJJPEUTLGkCvIPjUnCiru//HXURhPOOeAKsT6cwEoo/n -YzNL8c4GnegQeHvoiFXY++zUowMVQH4PRiIzRxtQ2fO4yy4LZEjdo2mLAUQUZ5np -7JRBg7cDtUnlq884nFXz/RkXYjCycApML+T+YQs8KwgeXlD4ZrRh5uCvHlwwHfjp -faSjMkgu/ROf1t+9sHW2jtX3AFCr6R4XEWZ0rrDQCZCxiWApFdoM01pWnl+Kxqoz -VbODEpjbjE/mtCGn2XmYr6bcXYC5AgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA -2JVul/xcJ+YlepOHxJ9dczIcXEjjMOBxWuyK+G9/6wASgHg9e2SB+WS1VSeUARC6 -VkID3Jlwr1gEw4gR3lW2h5I21kdCaBfCHshUoOr9rV5uE76r9kxgyEsMUZMvSClC -eQd8VK4fUT9JEKigIJeCFT9IE9PyxrdH1xpp89jOLy40t3PkubDi8WR8dvPckg3N -juLU/6EtbrtgFMnCqB2TmH4mc6YSCeENUTvt+nSiBKZUblGDuIxu/edX3SscS0Yv -qPM5LPcNHEGeeMC5ZSfotaSzRP+x3OlV9VJNROG4brbaI+3kECtegBgFvKIiK+JY -m7dkt8oIpQc8CKZkM8Kk6e3JXHzKf4vAiWHf0Wyag3gqCukxdas/PMx/3ROi7iDm -XQN713lxhIjtqfXQZjZcRmYQwdnkaSY+H7hgAyhnavqkBmPLeMU5hffdBswrjH+0 -fD0FOIDOWNM9e2Q/qdtHxtglNUmox0ETvl/3gYRkN1I56zNan6FNzGMubilntt2z -xXQwxP4Jn+RoAwb5U9mIlaLJ73FDbl6KAvFSJHlZl34R/o1nKOOAiFSv4V+RcTMd -x49P7cyAcW+eSsIgDzqabhx1OcrFEFRtBy342w5m5Qdq62TpFmeALgRYhAarA9UZ -YY/XOglN88K/3iR+A5LO7Hdv0Q/wShokghcSdAE3JOo= +ZW50NDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANn1aHYCHY8yFTsV +g5nZnQuen9yToc2F62Dz/CR8q0lEKi96biTN/ydSQUL0BdTW5n1UfxZpyNRU9h5a +llIfDPhundmtYj9SN0tPIJUoIT87g+q2/eaPntH/IIVn6TbeiMKX9PMFlRlkvDjQ +ZtAVSIiyfKym9n7SNh/FZghiI/sRjV7TTXbDppKSEIiB7c4MOG/ch9KJwQOBdd01 +sTlBMYHMlyHmSAsUdwhxxyRMgysxm9oihWiMwXl8Tq2bNmqGBfgOxdSfDvAmGg1P +6dtoNeYWD272fit4nXps5JGVV9+brImmuBZaN65B0TLxTRoo9L7yW0LGzAQHUSrE +MLeaT/S2xj8sBQ6sQwlcfBXRW7QZVqV32Sw0ijTlDafgaI4fjpbiRfvakatCEo+N +ZwH97QAe3Y5+m71aaCyeGMc/dCA5uCqEdy9i5neMbHIIUX382mqXByhqo8thu55B +qcQ+YrahK8zGfyYEjdVHvdWnzrRzu3S5tZ0h564iwrVuL9zVwxWZqs9gD7G0ViGs +4DmsvuQIthS0ZYzeonmcDBC3CAKH8xmPwXmg7+YZOkY/DgYtX3iJ7bwyTl4/ZXv5 +TgfD+HlvFsYgKy2JLWg+AQCUyJBFLwNHJIz6pGDdADfCArj0NSYXNEN2zEgORRGb +P2Y+z8PNvKtP9jxAAWpC7ICEvwOrAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +liXJjRTg9O4AEadeTV7BMMFtRIKXdUvj7WBbUgim1UsbDYjCvURLnNR1dAhn3QrE +c8CHrqF58uegW540UWfqDXcvGPUYgLPPweK/j2eyQmhUL9jab0Fnwo1ALgB3VZe8 +vP6sleQLlswhl5kpJgyGFUsf14zzRA7WW0UggFZVTbvO81ymgz81uGQl760FfqtC +5dczp4Y5FeUFlfgFj/nK1y39ijYrxWM+otCvRvxSKbi0JT/FB/w9B+rHaIlwIU32 +Xg8KJKwHI2ykGw7tTTfcdiSYP24rO0Trc97fJLHggJqeC69htjwtNCgrMnkDJAsc +N5BKXB/zUXnbCrjE0NIhrbcDlYgRaD0+VAyxdbUvLtAy6pWhf4f0bkAg1b8+V7fE +Fam9iHxC8NcKmwtrt3D5/me4CX4x9j7YFvrjVY0iJefqemI0jxD/Nm3v28xiH+/E +ys7PaEyKdVZrTVvKzt3H0p9HJqwwp9ETBOdqE3Zeu24eqs59qbMnAoFaW2iG6h1T +XaAAkJyBH8ax0KVdmuc3igL+uf1Oy+F3eoPX8Ds2svre/KABfgeObAr93NW9e3t9 +u8rL0yKEzG8crx+W0KCLDKtghIumWgSrB8ezjpNrqhiOzYyE0Or65WfXpUpG1Qhj +CsUJlXESSOSAGotLPYClXYaJqZG7ttGJ26drRtlyWxg= -----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client5-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client5-cert.pem new file mode 100644 index 00000000000..890ff5b209d --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client5-cert.pem @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFojCCA4qgAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJH0wDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwODEy +MTAzOTQ0WhcNMzQwODEwMTAzOTQ0WjBXMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRAw +DgYDVQQDDAdjbGllbnQ1MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA +t7MEWyM5qCSWFfKIxMAL/Z6GvQtoBY2GmNBDwx9FSBHAXm38jRm1fxIm3xVoujse +zxxZwya1enE8Z/qKKVmElRQypnX0K/TqV4VM2qomfDsv6/f+ueMWgAnPrbK0+Ggb +O1Yw2VB6b3qsGYgETZ9TJjKdHbuqy/hLosdLvZu/+qd6Rl8iCqgIa7jnLu1HFiMc +ZiCN4qAJ39virYvog3t57jS3awsF072kO+HagDmOoFxNhLq+Q7m1DNs6jBiU3Nii +0bhKDEZHER2Fvz5oE9re+gGde/B9mZk2sktZY3Uiw4eT0JagruGgduInvqKAtTo4 +pU23C+zKzSVHXyuhzNjaK/RtgSCTmOvFgXN4uWthcjd3tZOTMY7iapRqRe3JM0Ck +gHEAomR79a3KkYSfuFBdIew3sR04+r3lm/kB+++xSL1ypDFznbI0l2ur0e6+2r9f +whR0d2zA4pNIyl6itUA0FuECVd8Hs0cV/6IdqbCmRz+qWgyDeBhMW3miqy2KXSQm +00HT2Yqf/XWq5bfmQ1sq0oY6+DvsYTW9haVirFb1lsvFKlcAPHke23jKY3pMSkqj +oz+mJd1XpIxqzprpLjd5ayNu9FtTWSt2U8Ys693IVTne9Axh0knmGnLdIwhYTfJC +8lM3lNCqqYQ3eLy12qTPUclObh6dZtjaiSP3ohdXL0ECAwEAAaNrMGkwJwYDVR0R +BCAwHoYcc3BpZmZlOi8vYmFyLmNvbS9mb28vYm9vL2ZhcjAdBgNVHQ4EFgQUwRwX +0H/tt7jtTS0knaDOucaxGKQwHwYDVR0jBBgwFoAURoygeHMdCVTX0B/KOH3lsIbu +jUQwDQYJKoZIhvcNAQELBQADggIBACettwt0CjSG4u1QpjZhgpVI0EIJ97QKld7I +ahSaM4G8SHmU1ROf8kzam1lHVMEZU3YOIkqhyPgZPtcMlg9/CFvgpgHsPTkQu15l +e01Inf7lb3OIj/+PdX3e+P8wu0dRGjH4u3891V+yjkUUQJ8EbUw1DMI5iRiNTGuY +oLi1Oyizpgzu+dsI66QM9ntQPVrpMY27gDHgOSGqW5Mx1pTFL3/Rr9AKtMeh9i01 +YtTJ6LlmjhTIvzVmbSLVDgnR39Oq5LOw0kwK56AErEsubIeds0V+9T4M6wBjrReK +5ubolfDo/Y8Q2ZZONmIPPCh0M7gNdn14tfzgIKXoGWPU0m62PXZv4Syt+McK5twr +V7kXAcMp6MVSZCMqP/FXsUO13NLQLdU/a6BRakzCnrL9ulkB+RWmPKWQYvA4wSEO +9U00a2HVTfNSogMFcABxoRAkJEb951X+rn8L1EKJ1b4HTASbVEE7iYyktl9wWH1j +JXjqRTSgCNRKeezbEkXjJ90PzvCVfQmNxTipKuJjxVzLZpklZzb5uEV0Nx99bOde +QSexYccwADvCcptm6K77BQNcK9jhqfDxpFi4y61dHurFNIQQOQotHg9cKM6obmIG +2WzBBy7e3zs4yI6/jyx1Zh+dZbIi2+m8c8xSnZZDgVTKszAdIDFs6kH+j8zkLwlv +qvounDzC +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client5-ext.cnf b/tests/integration/test_ssl_cert_authentication/certs/client5-ext.cnf new file mode 100644 index 00000000000..8cb20e70290 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client5-ext.cnf @@ -0,0 +1 @@ +subjectAltName=URI:spiffe://bar.com/foo/boo/far diff --git a/tests/integration/test_ssl_cert_authentication/certs/client5-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client5-key.pem new file mode 100644 index 00000000000..4abf9d1d980 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client5-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC3swRbIzmoJJYV +8ojEwAv9noa9C2gFjYaY0EPDH0VIEcBebfyNGbV/EibfFWi6Ox7PHFnDJrV6cTxn ++oopWYSVFDKmdfQr9OpXhUzaqiZ8Oy/r9/654xaACc+tsrT4aBs7VjDZUHpveqwZ +iARNn1MmMp0du6rL+Euix0u9m7/6p3pGXyIKqAhruOcu7UcWIxxmII3ioAnf2+Kt +i+iDe3nuNLdrCwXTvaQ74dqAOY6gXE2Eur5DubUM2zqMGJTc2KLRuEoMRkcRHYW/ +PmgT2t76AZ178H2ZmTayS1ljdSLDh5PQlqCu4aB24ie+ooC1OjilTbcL7MrNJUdf +K6HM2Nor9G2BIJOY68WBc3i5a2FyN3e1k5MxjuJqlGpF7ckzQKSAcQCiZHv1rcqR +hJ+4UF0h7DexHTj6veWb+QH777FIvXKkMXOdsjSXa6vR7r7av1/CFHR3bMDik0jK +XqK1QDQW4QJV3wezRxX/oh2psKZHP6paDIN4GExbeaKrLYpdJCbTQdPZip/9darl +t+ZDWyrShjr4O+xhNb2FpWKsVvWWy8UqVwA8eR7beMpjekxKSqOjP6Yl3VekjGrO +mukuN3lrI270W1NZK3ZTxizr3chVOd70DGHSSeYact0jCFhN8kLyUzeU0KqphDd4 +vLXapM9RyU5uHp1m2NqJI/eiF1cvQQIDAQABAoICABA9VC+WZUd3DImeKkxydIeU +QccT62XqRVwxtSaFF4PaheNoh8DAOr+kDtRT49vq1BDCfzD1tWoV+Ugb6aY5fV4M +g2Rx3bHGcge7XLhL/cunvoHMJ7RZM1OttQgJr16dHXiDvcfIXzqHDBB/FMT5RnQv +BJ1sn28ZYTO1Df4SIcj6kKBd/U1siL9f1kLbbm2yPEAkX/Fel+DhYyBX06ij/CXR +z+aM+pPlxPoXlPsW8Nk+WmAB05t7HwVRmtMTF+iPgj1WCb3HYgr3otZC9xfM6+oK +xdhUwgBtKgezCjufpFYddd96dslfCueCRg81SyQ6TxMAqXj1aFb1v7crIp+ltWEH +rpOrp7Z+VH1vWJHZQHRdlNKeM+JEnzwegsf40+L92jItTrNzQqtZwHjdwC+LvtNu +0iXPg5qGku/0rRJpIp6lBeinuFivnm1YFow2Z/fZT3kbai7aokQyavJixTwIH+Ze +hjkBOhMnv37Mn8EhtrlJYPO2m8VqaISFZOmbx+lGb4eQ2i5dVm/rZ0j2y37bMLEK +xe6NcXprMTlVTChVCBGSEJ1J7M0RFOi4I6YIw0A6mcZcEtS1MX7PonMLzhQjlyQt +WawoVhtYSIhpsa+0AqetkmBcjYHpu2vXyTGP32c6DiZDSUOz46jpP/XdcbghfZyy +SyFgjTCyrnrHCwo1odwNAoIBAQDhIlEqPsmN07/6IlwSxvEChEAe6ePsQ7qDC1xQ +Xh42e6E0GaiTeb7px75E0KiumDvqQ55m7QFcOWFJiVn3tBpekplN2T6aBIY1MwYw +2aCBO+SetdNtfeuaFjkO/RfNNgaKN43jfNfwpHeSZFIQf0v9l3xz55+pHkDDYTRv +Z1sAWTJniuyZdLtXgXpWAm31UMF+lNxje8gtg96hQfAQhfQohJ7d8m5IRzcyYbJZ +bguTNLWFH8d5RHEO6AfyOAzv//ExoewrqE8HcdydqJ0b6a2PqdOh/VOcmqDCsZvv +QTt2v81OK/TxTEaRCXNwh6ueTi/6lEr6+JIQWGm/ZzEH6WQTAoIBAQDQ4m+QuvY/ +uMQWlL82Lw+SuHJcjsVQML/PMvDR35+1G5IYliuHEK4URpRZBal9wXVzmqROR2gP +YK1KwUVcLCVjG3jxnCaCHJy634e5/8Hlj+HAPpeHrHldtvkLFjXNT9/zfo0mopjr +P58A5av8XEjjgTWLycD6ba049DntN1xpYCoT/QxdbxKcKDIKOBFfVQ5MnCNEU48P +hFkXxN+/j+im9KHOpz/qnen0WotvI4O6KHBOM48J8OIMpdgwi08hLqKeR+u3Fksm +mbIT77ivWsmmFMF9+Qr8pzW8Qr1ThUDfV3l/yTpXM0jLsp5L47ReueNn9St5muhg +y6STis3Z24HbAoIBABwagZFJdzWaKkvaQC4j4XAT/EEfp1EV/1FeeK6boR/h/PSZ +A18tgnSCd7D1bykkXc5XlI16poHRs+BPIgj21ZfaqwJ5zQPC7tHzRaGReSTIyjg5 +sGPppSB+B69kTrVSOizE1JSCQB6NGVQP6+KdUWodn+6ECn09Bo6dMsN5XnXDrLQa +NoDNfplk01WrRfiqlQztHjnY+NsdrAvrREF7x0fLjl1cZwHkyfIjGfVp9E3y9Xnh +xq/PqGFY+6zz7EGFbKFqeiXVA9CFGSohI0OZvw3BAGcnTWmuL8U+EJ4kQX0IYmOQ +vynkAKcYS+3PMdA4KGsaWeXMY8dgkgrPoeI/8zsCggEBAIXgCmb1HKv8bFiYnvBI +Oy/q3QjrTBS4GcbfPC8WptI5SGpPnLgk/ZzrG1ru4RFDM1aTTykIZO4uJQKbWivX +82JQkjQvwxJy6hRu/Rs9ivTpQWOn+hAvf12LzpPHeVYMElmJy0duwJlNnkfB63eD +YgfFoR7lgLRAJNwy0wlxCN5UjsNe1FTxwjipHmEkTCYQami5TXUWJMNqLHuqZ0js +H80ZTrcWxEVHTlGY0nK5jUx7bHcCXnOdmhNHG4cJUf7PV9Qf/jfKpwGnUdWrN2/T +vRefh9+IBP2m4S62i4cDCW/bLXiz7JtEuCwtX3z+BHUNqGzRWjrHCJY5DlKtjH9L +Sy8CggEAO7GVsRaSB8awfFBK0qI33WQvW84GGeMqhEIUeLsUplEIcw05rsV+TxFD +m8YC21yNEUUHICpEA3w5ZPXb4e7ZX7WRZwWpkF97sklX5tJBqwa9i/x/zhB3Jz5z +dQeAZmLotF1OFczw77lAwxqGTMEsuQ3iZdXMNug1tx/Uv2DehdB2b0kLl9/G2MXf +KhfsH2+08kdg1pkXmQqO27pUzX6CNNHe4g2qM/t9qpPj1cJEyIgL5Wk9GNrWhJLI +gwBOf97eEQ9qxTIJEWxLUVbGMz6ULo23hmZi63+wdlQXXQbrqQiUw2iUuWDvgEX0 +GmrseRP8qnzBQ6OdHEn48aOT5XMXGg== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client5-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client5-req.pem new file mode 100644 index 00000000000..a4a3c9d6531 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client5-req.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp +ZW50NTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALezBFsjOagklhXy +iMTAC/2ehr0LaAWNhpjQQ8MfRUgRwF5t/I0ZtX8SJt8VaLo7Hs8cWcMmtXpxPGf6 +iilZhJUUMqZ19Cv06leFTNqqJnw7L+v3/rnjFoAJz62ytPhoGztWMNlQem96rBmI +BE2fUyYynR27qsv4S6LHS72bv/qnekZfIgqoCGu45y7tRxYjHGYgjeKgCd/b4q2L +6IN7ee40t2sLBdO9pDvh2oA5jqBcTYS6vkO5tQzbOowYlNzYotG4SgxGRxEdhb8+ +aBPa3voBnXvwfZmZNrJLWWN1IsOHk9CWoK7hoHbiJ76igLU6OKVNtwvsys0lR18r +oczY2iv0bYEgk5jrxYFzeLlrYXI3d7WTkzGO4mqUakXtyTNApIBxAKJke/WtypGE +n7hQXSHsN7EdOPq95Zv5AfvvsUi9cqQxc52yNJdrq9Huvtq/X8IUdHdswOKTSMpe +orVANBbhAlXfB7NHFf+iHamwpkc/qloMg3gYTFt5oqstil0kJtNB09mKn/11quW3 +5kNbKtKGOvg77GE1vYWlYqxW9ZbLxSpXADx5Htt4ymN6TEpKo6M/piXdV6SMas6a +6S43eWsjbvRbU1krdlPGLOvdyFU53vQMYdJJ5hpy3SMIWE3yQvJTN5TQqqmEN3i8 +tdqkz1HJTm4enWbY2okj96IXVy9BAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +nMHx8YWaXuz3XKbouq27h4/2PWl07evoXfKMcG2F5hon6zhrk/scEbKIlLOlSZSE +L1zVZ0jQsWRFsezHVGo4e9ZEmNflWO1XSgHOquQW1NTkyHPL3D2duJTlVLus/4Yo +gqLGgyfhU3JWsFGD9hnlku9z95HdnT8BFXLCikb1KsaC8JOFwYAgxMITb9aKzP1h +eX2hVtXz+O5TZNYSTIXHs+9AtRjvX1Mhj0L4gCYQueAjS/f5jU1/RhqOf10ZLe+B +SbCirZgwvlErULM9KI6sT88OZlEgfoLaoxN2X/9YGGQTD8JvByFPryc7tCTpV6t4 +pKpUQ3vTS3LA6KUQYEHFAXg7u6/r11dk1k2Ni4q/NhPpjhSsooE9qTUDh0jUNW62 +BuOhkxyDFIQVgJqgkD2Uq6GswkgL+GX6fUBwYNYZbdC7tv7ssWX7yGcn0jYZEAIU +8SPhkW+IFqQrG4zhJKh9ohasAn7wV3AfSQBE2P+cgOCasQyTYQU2++1FBVe319lu +IoKDLRAU0shc2vuwnTbnWOlshBBD3gKfSsF4YxV+5zOe550HW+19v8Mlj4mdRaQ3 +Uh3SWGPBGVy+bJgBaSOjoOoUSDPPmzIPo9bGCZ8SOut+/jsZ3PJ8Mn1b+geBugt4 +wmJiTGQXbRunGYV0pft038caNzVJFp/uSd2O3PegwLs= +-----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client6-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client6-cert.pem new file mode 100644 index 00000000000..22e57c3f2b4 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client6-cert.pem @@ -0,0 +1,33 @@ +-----BEGIN CERTIFICATE----- +MIIFojCCA4qgAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJH4wDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwODEy +MTAzOTQ0WhcNMzQwODEwMTAzOTQ0WjBXMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRAw +DgYDVQQDDAdjbGllbnQ2MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA +ld8aBatdDBGEVARor//8lYQWW75VzNsTaNXzPFjitA0G4Dx4F1pcV2mBxoR/1aqv +4pdwku7FqWxqT7DfEzVMoeBFzMi6Vlrn8YFM7mT1fJ7r/jI+JzDc5yoOF7S963Sz +yuwWHFKrkyXftaxhorv8upz3qTNFvui+3NkCD2N6FNb++Qz2ircB/KQFq3dIluP8 +wL6zEMgyKcgYhr1MLPfe6JSqU4PtGUnKjEpFN6Fi3Ubvd6NjKqxIwQd46xJyFi9V +XLbMOhCtKXXcaO0Qi3rzzS+7seCINX06bAnUHMTMlWNmYw7b09+w9e+8yjICsKZA +hNyyNxA+CikhlcZpP3oMn3T/slIVekuRYszlxD0KLMbZ+lOC/LmEPIrAcfhCc45M +ys7s0ijaXO58NqsGstGHTSz8WW9YP6VWvfcd6OA2sXDAiLDkkgk4Sh6hGGOkRfc2 +RmMmKPgMdLQOlBC5HZ+m+EOsTTQRKDPs6HsxKksxe+mOkAnPExmpYgquSWEHXE0g +eo/BAfLMbYsob72XAe0fMXjRIGKQlJXv9hFIKhcqwnGaOs/n1d4uBkdvdof9RjUt +p+JQxkKIU72J960hSWj3YmVTzdN8HtHK9f78eenr7+HmB4sUFr1qzwA5h+Y9+6Ou +FUNM7+WfXWfqGHaMNbEVZA7saaEYjyR7FjM7+lQikMMCAwEAAaNrMGkwJwYDVR0R +BCAwHoYcc3BpZmZlOi8vYmFyLmNvbS9mb28vYmF6L2ZhcjAdBgNVHQ4EFgQUmRBH +PAQ50KDlkzSbJg9KYGQsbJUwHwYDVR0jBBgwFoAURoygeHMdCVTX0B/KOH3lsIbu +jUQwDQYJKoZIhvcNAQELBQADggIBABzYUj4S02LGetmZJbMs+g1BmD3mwerE44Ji +X6cqOlnIRtjnSjRZqDufAMh9qQvwn1g9d2GyFkkk3FDM0KRubW4w26CV1g4CtduV +CiIibpoVrUCkoHTbJ3WEygzWby1AQ4crgQPFbJwvyzsnjBnLO9EORyL+T9QAx5/j +vOOB/WSOwpu+LEjw3PHlCR5TuDfN8eYoO7thfqlCQ9A6aTVFizYLxfaQi5zluLyh +1UFgBJ5ZEzlkem4chvhNBDYSDnqmCw3X553MVGTa5pE5Q/vv0FzPyPUG/iml7DiT +xsGFLH3tumWw47HTKQit+9spMg29xiiakRcr+TlXy5DTmnR7ERYoV5jHg8EpJqid +dcacvjEVRYblmfb1lRrZaULqXE9+w7JqP4V0r6vQeiftGQ1Pn2MPqcxMHqehr730 +pp7o4L4c9IpyuGs7SQAf1LCQMhb3H+iQEeVI1kExeWcPZZJs6Bhi3fEb4/m61W6f +6vJd2WJBv0uxgMHKrv3k/5biPbVwz6yA1oCdwEGiCHyCFmyYNC76qrbU+A6F3zqs +fmTRlS8PLZ7iJJw6g6URh+Dvez/+PMq0KlyHKP8CFjyWr0i5rpeNBTTgnfCrKiJQ +TQdEC2+KATwYlzB8DXeC8jcj+L4K8Cp6XRBIQnALnOmwJcRsTSVggVHxF3zuxVf6 +SXfWywZU +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client6-ext.cnf b/tests/integration/test_ssl_cert_authentication/certs/client6-ext.cnf new file mode 100644 index 00000000000..d421f4198b4 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client6-ext.cnf @@ -0,0 +1 @@ +subjectAltName=URI:spiffe://bar.com/foo/baz/far diff --git a/tests/integration/test_ssl_cert_authentication/certs/client6-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client6-key.pem new file mode 100644 index 00000000000..9bf833a3a79 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client6-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCV3xoFq10MEYRU +BGiv//yVhBZbvlXM2xNo1fM8WOK0DQbgPHgXWlxXaYHGhH/Vqq/il3CS7sWpbGpP +sN8TNUyh4EXMyLpWWufxgUzuZPV8nuv+Mj4nMNznKg4XtL3rdLPK7BYcUquTJd+1 +rGGiu/y6nPepM0W+6L7c2QIPY3oU1v75DPaKtwH8pAWrd0iW4/zAvrMQyDIpyBiG +vUws997olKpTg+0ZScqMSkU3oWLdRu93o2MqrEjBB3jrEnIWL1Vctsw6EK0pddxo +7RCLevPNL7ux4Ig1fTpsCdQcxMyVY2ZjDtvT37D177zKMgKwpkCE3LI3ED4KKSGV +xmk/egyfdP+yUhV6S5FizOXEPQosxtn6U4L8uYQ8isBx+EJzjkzKzuzSKNpc7nw2 +qway0YdNLPxZb1g/pVa99x3o4DaxcMCIsOSSCThKHqEYY6RF9zZGYyYo+Ax0tA6U +ELkdn6b4Q6xNNBEoM+zoezEqSzF76Y6QCc8TGaliCq5JYQdcTSB6j8EB8sxtiyhv +vZcB7R8xeNEgYpCUle/2EUgqFyrCcZo6z+fV3i4GR292h/1GNS2n4lDGQohTvYn3 +rSFJaPdiZVPN03we0cr1/vx56evv4eYHixQWvWrPADmH5j37o64VQ0zv5Z9dZ+oY +dow1sRVkDuxpoRiPJHsWMzv6VCKQwwIDAQABAoICABApNaok12zEustAinVoGjmZ +O7v640+q1AX7BDaesxuNSA+IFb8W7Fw0sVzRzuMge8ToZahCfSgO28vqG7P7jF7s +SzqQ7p7/QIGnWr9ePaaJkJMhodgiCcAXpdtijxRDylt4Z834EC5W6C+z/fdJZDBP +WtlxAPVcLbCs6e704CY9JNOAVVmR8HHo0f+yZi53OTsjlTWbEOW2gQaOhRdYRz43 +QcKtuDSA44mLPLmJ+po32vzArqHcYCAIiT54B7tisUbFI3Kjd7i/z9u0rJMN2Bpp +Kkk9GBMEhPc/fBL6GcGGLXVDFetb9L4gcvAuwmGAG/b3wXP0mWwJEWkQMys4IwIo +oJ+FRCGtkUq7cdwtwX+mJyNKA1g1xO9YfLMhmKpUaxcLvsMBMb2ZO33/sIH0GNSL +csuIVAiVzwODNo9TbbgKylkKUjbVC7XnQwdGOUrl7wGQdI/jTjbfwHR67S7S6Zkw +Ctobr9+lxz9pUcR7U9QMAKwCheNcx+vVbmeU0QesL2pVWKjCgWJLI4hmK9epGrTQ +ssKmuIV4k8fDQ+4yOL6uk8SbGRLKE/tjG5IArloh+0TWlF537xhFu1U48CT9i60I +GN4FYes+P4LkJToJP4w1Dh83VuHLJBR6xoK+tb352SaQHONv0nKTnxm9PeRsADwV +Fu1YNitG7P86cpkvVgMxAoIBAQDHPgalQK/ld/aNbLj9Nx+M4Ok/eukNnQYjGstW +ScW4OPHhNDDS90X9vAA3sKK6qwpt2B0BTBoSb9zo34G2Zx4IJGOgzVlnRKb202aM +I1tSdt8ULWwjCXyN4jH078FAn9RKtA0T4V/Ps1R177oB3P5ohOCAWAFatK2ASd9y +bt7n5THi8xcjNikJiMqyHXzVMIB6owe9AXpoY89KRJm8c2J/tgypYSJ73JbMTkQ/ +sOQz7G6BpaZMc0IcjqsyhY7o4kcM+G2VYvoLvvstAJkLgWShZkHEFARmOfgVhNw6 +Fps+0VpmCCqoEIsMXA1D+Cr/XVDJ/O8xQLOQP6jhVeV2ySGpAoIBAQDAkKc5Nuzl +SQwnQNY68h4n8NPjpgtxwdxVwBvYWiBNo9wol+RjltgXb37ks3/SE9eoBJOLuzAM +agA8pR3hk8B0Jshpe+ucBwT2k5to2lCSWVcr+VdMjKcm4s1ts+zCacbaEZDsbBW1 +J5qXxy/78V2rLWcd4aWEu4U35mM48V4W6RuC7cqHWYargXM12U3aTQX/ZA71EONw +VjNYgOJ4YvBAoKaaPVBEjLftWQPCwuFua+dC/i+1QYJUhoxVUd5ySvz/P3FCVrP2 +vd/PHi02mim2lGkrl71bSeu3DLbJFGf60/LbsdK5/nOp66vqCJa+KBQ8UEqK9Pb2 +IzktLFWjBTqLAoIBAFY3+Rg5zeQ27XJ1wfeRiw2jpkvaHE/Py4AVcDh+5Bi8S69w +rlAcwSTz3gQ7Y6/zpCe0hZiyHfYsgiQj1DXimZRcauCC/FU72CSnhpDOHA6rcg2B +OnJPAJ9Faujbd31HrM5G7AHWXWe76qi+fHeh8lW4ao4fhzaBSTNixFb4s20WOWhE +WZbbMRb3Iv/A0uxOeMqZEhgwM5BI4ML6vKxYIh+x3/jMF+gRpQ/0LoSBP79YaNmh +nT0oA0voUR0jvpv4j1aFiqZCloHOu1LWF2RrMejGH1+CagTywPBO9h4M+lFMtxnA +HvVrc0B99R+cnL602ukNk1R08z9QzPv1975Xe0ECggEBAJ+A8lLtEQrXF+8HwKvw +01PIIuslJcOjjaulxirOdIV7HiitsCthzjqTSyuLF2xerTpqGAEdGy6dOvcA2iB4 +r2hGm6jsUXvbQJwyJf+THjwrLo6pKUuqEeae5QRijdF3ppQJCt8apFx6oo3oGvH4 +utrIb+qLdvvcC/wCpNuM6p/VlMk9yI2WdXtobZMEHX2eYUJrkgwiYhIyBLPhhjWr +1k/iAj4uXWd6m/tIyVmw7OP3Ewcl30SnL5puHJ2rg7NuM+QFm/4ULVtLabB40YCx +761hfz/xn5KXR11HdbxXX82fdEHQKLmRcCMmqC7h0GNQdXqEE0rIoCu/f2PQnlq7 +QQMCggEAZBCnrS1NtlP4KN+tbSfUssxvnU/Lriw+VzkGclsw5QMRjeTTvcEV0Bh+ +ylvf7xwX0nansg+hRbB0k9wYmkasqFPSOhQYTQ1H+N8oHgoOEDjZz7xkP5j7h9qb +K51uS6qoRE0qroI+BDWSom+9MPcnk1eyIzfNm2hk5ymbOz6lxSoSI3HdoxtAFoz8 +D3h/7o9dBMZIjBSawh1M/YLKYemZW/FyV3EDQqi5wbWV+C8vZkAiIohr4N55MM1A +BWZG9+PdP5kCiacwZqUzX5vKyLTG9d8Fv2GQ3eCzMb2aRMKX61Z224+XuLy5/rsF +jbMZsV6A1MRCzhDoVIaUcb/2RUtChA== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client6-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client6-req.pem new file mode 100644 index 00000000000..a83e6508dbf --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client6-req.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx +ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp +ZW50NjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAJXfGgWrXQwRhFQE +aK///JWEFlu+VczbE2jV8zxY4rQNBuA8eBdaXFdpgcaEf9Wqr+KXcJLuxalsak+w +3xM1TKHgRczIulZa5/GBTO5k9Xye6/4yPicw3OcqDhe0vet0s8rsFhxSq5Ml37Ws +YaK7/Lqc96kzRb7ovtzZAg9jehTW/vkM9oq3AfykBat3SJbj/MC+sxDIMinIGIa9 +TCz33uiUqlOD7RlJyoxKRTehYt1G73ejYyqsSMEHeOsSchYvVVy2zDoQrSl13Gjt +EIt6880vu7HgiDV9OmwJ1BzEzJVjZmMO29PfsPXvvMoyArCmQITcsjcQPgopIZXG +aT96DJ90/7JSFXpLkWLM5cQ9CizG2fpTgvy5hDyKwHH4QnOOTMrO7NIo2lzufDar +BrLRh00s/FlvWD+lVr33HejgNrFwwIiw5JIJOEoeoRhjpEX3NkZjJij4DHS0DpQQ +uR2fpvhDrE00ESgz7Oh7MSpLMXvpjpAJzxMZqWIKrklhB1xNIHqPwQHyzG2LKG+9 +lwHtHzF40SBikJSV7/YRSCoXKsJxmjrP59XeLgZHb3aH/UY1LafiUMZCiFO9ifet +IUlo92JlU83TfB7RyvX+/Hnp6+/h5geLFBa9as8AOYfmPfujrhVDTO/ln11n6hh2 +jDWxFWQO7GmhGI8kexYzO/pUIpDDAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +aQr207af+Rbh3xeTB6o7xA8okzQnya3E0sBHqXGTWGvDOYg+yMlkR7/06YWcM1uu +O//iyRcu2lfd/+NmduxG2RtBT+Nn91WSF9UZH37uIKZIBsj5zaYhNFMnCLjtdpgO +lL0HFPU0IVR41Fq+vSGuSiOLLjnNG42/irGJYcvfEQ2yNRYQ2/y6GV/puGjvH/oN +np+ucn7ae/sLlpQYMT5OZ8Nwy/2b5hcEC4ekuubGeYtnWx4is9hxz8IeJZAxRHJI +xxIDkhP82elJpTMv4dsZYTHYHv++M/WbWb73HiAlD9g0BKYuzolbCu1TYZzbEz+a +4/lnbkR6FkKxFlP0JrTJSceQW3d3iEuER+n71Si7thwk8Bh/np7GjbHSs0ZnT+NB +ua59bE2YBWpZEoA5IgfRK+4i8AtzyJqDPnQNM8DEBN/BLqppGLitw/nMcEC2nWT+ +b+PIB907kkikBUbyZmjF6f9Fhpm35WQuCubVz86f+BbGZLwfGyVnl7TLxXiVANI6 +DXqptFt3ppv/pPRzg473pUZutjzFsUbcJNal2FWtCH4tV5lTWaBeypC3PIzahdkA +PwF1IxmpBqmuWIHXIqId60g0LLah32sT9w4BaC0rpUNeuG2yTeXz3RRjG7a/x8xl +6m751kWb28j9L3ujZI1fr6E1RKVIhe0OC9wU3D/5J/Q= +-----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh b/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh index a09b7b2874e..e8d649fcff1 100755 --- a/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh +++ b/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh @@ -14,12 +14,16 @@ openssl req -newkey rsa:4096 -nodes -batch -keyout client1-key.pem -out client1- openssl req -newkey rsa:4096 -nodes -batch -keyout client2-key.pem -out client2-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client2" openssl req -newkey rsa:4096 -nodes -batch -keyout client3-key.pem -out client3-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client3" openssl req -newkey rsa:4096 -nodes -batch -keyout client4-key.pem -out client4-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client4" +openssl req -newkey rsa:4096 -nodes -batch -keyout client5-key.pem -out client5-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client5" +openssl req -newkey rsa:4096 -nodes -batch -keyout client6-key.pem -out client6-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client6" # 5. Use CA's private key to sign client's CSR and get back the signed certificate openssl x509 -req -days 3650 -in client1-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client1-cert.pem openssl x509 -req -days 3650 -in client2-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client2-cert.pem openssl x509 -req -days 3650 -in client3-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client3-cert.pem openssl x509 -req -days 3650 -in client4-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -extfile client4-ext.cnf -out client4-cert.pem +openssl x509 -req -days 3650 -in client5-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -extfile client5-ext.cnf -out client5-cert.pem +openssl x509 -req -days 3650 -in client6-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -extfile client6-ext.cnf -out client6-cert.pem # 6. Generate one more self-signed certificate and private key for using as wrong certificate (because it's not signed by CA) openssl req -newkey rsa:4096 -x509 -days 3650 -nodes -batch -keyout wrong-key.pem -out wrong-cert.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client" diff --git a/tests/integration/test_ssl_cert_authentication/certs/server-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/server-cert.pem index 073c6485bd2..1ad3712d547 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/server-cert.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/server-cert.pem @@ -1,31 +1,33 @@ -----BEGIN CERTIFICATE----- -MIIFZTCCA02gAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJGwwDQYJKoZIhvcNAQEL +MIIFpTCCA42gAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJHgwDQYJKoZIhvcNAQEL BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwNjI2 -MTAyNTAxWhcNMzQwNjI0MTAyNTAxWjBWMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwODEy +MTAzOTMyWhcNMzQwODEwMTAzOTMyWjBWMQswCQYDVQQGEwJSVTETMBEGA1UECAwK U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQ8w -DQYDVQQDDAZzZXJ2ZXIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDd -l4bW8vca9/iFDASgmGxyCQvEK5roWwPzE/DXdS+mMJM/fffyqB7Cldm1KW2ILDYU -6JGnlYdxvWLOEa7n4cIBrQ2za7vfDU4LzGJTBN/9C2TYGFP13FTfZi2gZOuipT+M -pkFCfJG1zMWR13yKNxiS6Ixf76c6ADUyt7GmRcuKSctKzM7hUARfeko5iBN8aHpD -nxYa6xSGlHXHDzHHEqEpsk+iQ/eK6o3EeElg7v74sZSzqleIkEfFLiRIPNvuAwEE -IRIQg+k/2S673kdANlyfszXXbUGCYU9lOOPYgAEFgb5ljSXZJBTW3uFlyIONK5I4 -EHgELy9Hs0qnqttXCx8d5I/eI/+Zx9GPF3x8gkMi5wQePiBpqde20kRYDvP2eGeZ -eFTnKYpqCshVAJ4+1Kg96f4S9+GZ1wfzsgS1lA7+9qr3qTvI+XnKhC2h3bqyWS+C -BQikgADbURgZT4EzhXvq7fSCFHFTA6xZzXNEVO7DkRrYbvkq06bIl1Ov9vcbCN4I -zOuJJMxlk2Dv3C3mUox3HmcOdO+vvtWnZt3HbFY7ZPCV68ObSf77YD3O5on5RFQu -hk+AinrsDIL6NiiVzALXBL+e8flkqZDzRk1mGphVXGcRP6nn4VtrfN+Bmbm8pu3m -6aYuqSX6vQXb7EHW1tAbvlfbxIlP/Hp5GoV8zqI/tQIDAQABoy8wLTArBgNVHREE -JDAigiBpbnRlZ3JhdGlvbi10ZXN0cy5jbGlja2hvdXNlLmNvbTANBgkqhkiG9w0B -AQsFAAOCAgEANvZ7QDkHIKRq/g4GPkuiU7DN44/TW4bOFe7rDC5S4z5sh/i/Tvur -JYW7m97vLui5PJf6Vbd7xyl5MFIiz2KzoLi26rlvYcI/BT8mIG8jMg7pJjp/wJGa -QdCxdO99a4SIwg7x8pvWChFAOij5e6RhrIvsEB0LN5kKRSnQ0sW5khfsdFbn+3Iy -VwyvvE+nsCqE+KK358EMHicn8FVD3Ze+YzckX0am9DbshL5+eVQ9nOhUV2B8PcbG -SGzqJF07wOBwCdcn3eY+V98SQqrpGC9tCXmv8qErfkq7pkUGWq15d+miF/gaUz+Y -yDPwgi1pephBJ34IhLUUk0IPZJ23uVv/zfB+SpZ9/5pjsmnapR3Zf725jWrhjeT8 -44i5kNeVCvZPzQO9cTOsLXJbWb0vqRzKsvuSvffDQZql4bMMvhPjMibqCiRuSHO/ -yPlWiJjhkZz52DPJX6+LOeP2pFfUe2TR6IqcFPfUs/bV6aD2L/s5UZfZXWS5i5FR -I8uvcKOWL7NBbdY+NVE5aT7DqfhaRurjp61Aym18FgXLHpDHYV9IpkU34+A1MBUi -bzHZRWhxZMRxYezC7jE4zsZ5CQtSq1miDPcDaeK5vMd/Vdys5MIekqCfKUh+Cd5Q -gfC2QgNgodcWRF2plNgA+3E0dUULR3+1s83gGWGC8/UFW+9dtYV4nv8= +DQYDVQQDDAZzZXJ2ZXIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC1 +l+TO19J5ZBVs+ZVzT62wM3JRh+CzO9/ogc5A0ERwVsfBFQYz1r6pGHHTxM2w3auu +ehyALbiW95eVvkpu0QyOd+CaH4K0DI/reotYyvAX5+Qv5tnHGAwFKNeDnNG4hpCR +MO/BlohZEh3Ppcm7jvMVpPhSM8ojZwSvHsEMBVb4QinvpvROLIDrA/zPQSlW0S0A +ZfvV+cjxIoDjTsg/M0Jn+6VNNRrAemx8aKv4TyOh2b+zYNa8wnlhdspTgjCseMeC +ORzDOKMP784p0xWFwQzaoS0HYrxLoijiwIw6/I72ADQxkupW4OY22W6qjheDe67M +33/KZVZ9lssc7QKGBuvkxgsMq63M+VI8umY3gsIRLJKEwXkqviIXNCqw+5zMjsRB +L3UUBjHibD1cHpc170xf4IOMa0Nk5svNy0tQ8OdTC90t2ni99O5E+MOHNTn3mr4G +Chv/+PVlWjf+2PWksmhNdTBkE64JXSrf3fb4OScbkhwD06YWVPtak5xv61QNgL9q +srARcm8f5jrAlzqU6rYHnCkjB2A011DLh3idwxZfOtxT1jfu+NMP2N8xFgIjtp8y +iVRQopXMA/HT/3J/INO6SLo3t5nBRjphDsaOcp3tOlAUYa++2QXM87+QKBlRBgC2 +Mr2iUonqPJFmAa62TudYGoBKqeVfz5yFmclUGZlb/QIDAQABo28wbTArBgNVHREE +JDAigiBpbnRlZ3JhdGlvbi10ZXN0cy5jbGlja2hvdXNlLmNvbTAdBgNVHQ4EFgQU +q1dxD9JZn+JeqZOIKHi85hq3J20wHwYDVR0jBBgwFoAURoygeHMdCVTX0B/KOH3l +sIbujUQwDQYJKoZIhvcNAQELBQADggIBALywV91PI6aClAL7LmZjVq+IF0AeSH64 +gqx4WJsOr8sRSu7SXea85JW5qHrZByjdH4oqxVrugji0hET1M5tqExsiQSBBH94W +kpPtr1UbJHWdKPKRFe8iqnFfSaTUZwA3gNJkmdv01gPGrySb5O8AhvGqCzhEQ5Iu +PH9Fu+xclKwF7UoZgfTWsQgw1GaNpSg7+86J3F9vfOzphsth2vKxGYVxnsTxbKtl +pwiy6jOwhUeix6z9+OgVOfmIxj6OvNRsGHRvlOumo3NyVMGrnABnIjBLj+k8rsn8 +wzKdck8K6ttj2fXQOdTywwipN6BrkMK9AJS5a+0ylKsyWG/scjbt0cMY1uvxLXJs +/KOw7FUy/FfdnD0BnYhmiIy65guWSjtyw+n5QK6z9OXTfcgeYZyKhgyK/HLxfMtX +PNs/DghZrI8vqX+oGW+HTBpitRThxCZ0gBra+OZRidF0Rqd4nBKwFJGaYk6B7uF8 +9oek3IHaPEBV5bqknHH8/DjzXnDpFZ/hphkBgmzU8wquGiX2sfYiQUQlcS5b1t05 +VYtcnOtIhbBe/kixcDtyqRCSCPRdrgg4vOXduOpUbeejmGpjSPe/KeVhMfP6JHWt +rxfwkTdN5xtlM6JMrbN/NBWtIrMsHyM5uwXqY9LnO19mHVYYzEZqSCaot5yOMvzO +BMSyrrc+OTy6 -----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/server-key.pem b/tests/integration/test_ssl_cert_authentication/certs/server-key.pem index 067adf4e1fc..8097d371e0a 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/server-key.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/server-key.pem @@ -1,52 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDdl4bW8vca9/iF -DASgmGxyCQvEK5roWwPzE/DXdS+mMJM/fffyqB7Cldm1KW2ILDYU6JGnlYdxvWLO -Ea7n4cIBrQ2za7vfDU4LzGJTBN/9C2TYGFP13FTfZi2gZOuipT+MpkFCfJG1zMWR -13yKNxiS6Ixf76c6ADUyt7GmRcuKSctKzM7hUARfeko5iBN8aHpDnxYa6xSGlHXH -DzHHEqEpsk+iQ/eK6o3EeElg7v74sZSzqleIkEfFLiRIPNvuAwEEIRIQg+k/2S67 -3kdANlyfszXXbUGCYU9lOOPYgAEFgb5ljSXZJBTW3uFlyIONK5I4EHgELy9Hs0qn -qttXCx8d5I/eI/+Zx9GPF3x8gkMi5wQePiBpqde20kRYDvP2eGeZeFTnKYpqCshV -AJ4+1Kg96f4S9+GZ1wfzsgS1lA7+9qr3qTvI+XnKhC2h3bqyWS+CBQikgADbURgZ -T4EzhXvq7fSCFHFTA6xZzXNEVO7DkRrYbvkq06bIl1Ov9vcbCN4IzOuJJMxlk2Dv -3C3mUox3HmcOdO+vvtWnZt3HbFY7ZPCV68ObSf77YD3O5on5RFQuhk+AinrsDIL6 -NiiVzALXBL+e8flkqZDzRk1mGphVXGcRP6nn4VtrfN+Bmbm8pu3m6aYuqSX6vQXb -7EHW1tAbvlfbxIlP/Hp5GoV8zqI/tQIDAQABAoICAQDaRKlTDRwN+ndXRlFAhyM6 -6GIopvL9MLmhM+EluY5n2q0P+1rCMIusC8LYSahUW4gh7DucoRM7G9s5M/3e9mcN -E5LNSq9RtF9OC9JGCCVBsXlxyfTZ1l/bdWA3/3CDUtZYCmN5xA4az0tErsdDtaWE -/39V+E/2N8Iu5PYd293zp2CRm0+kbBcCnQiDxt+6yYa1GPzDIw+iyJWCsBrOBjGt -SrBaGyy4LvXZsspEquWHvhPFLWLvZ37qYNroNNpFhbv4f0K19dlJRPpdn0L7oxB1 -VicQvdOrQ4LbJ8B2vw9Ch1wt12ySiJHmXMAUa//4jBSJGN++72NY8uf0Y72N7ayF -Bck5QE0we4i7hhuN0WL+IftYD/O5NgOnprjMWifLOhQ8OECZfhOKgbRU+i3aJl8D -+raQBW7/GM0uL7xIoMcEZSwMs/sQR4loNCJ0UsIeWTdWNrhXrEuzDQGXoWcB8K/r -VVayDO5Qqx8R77HB82/pRdqEWNaNQd7DhPgnWiixISpIGm6zMvT3S0hzEkxu7FNb -uciq9i82BrBkKhg1kiF70FqG13VeMFqTJUuqtoRs1QEQgumvWB47n6FiVwHdDbbH -sUeKZYwbrY22Cn4rrfXH+0KKM9TDR0CiCv+CkSGmG57l5tW7aSUWun46qP8vh7sc -ztzb4LzyUt6XEBIWIqBIQQKCAQEA9+f4TyGo88qzTKaQoko5OAYhvAGr1UGAS6qh -sJpd7rHd9g6DnXyOInpQNglToiJ94mNdIK8f/oOjUXZh2E4CWuxeK291BNiaqCxe -s3LS3XjkdHQIHvqJUw/r4YJ+zfoGznthNbDwDkBob9x3h9rknTbGdLcgaTGi/0PZ -cFnyWDPNPskbcZ3Dxr41oDHiVsOx8n4d4HtspXzh+EPiQiJz5HVfwGNEyzhmFWIa -EzQaxnHL+WF1Pqe1wrzOwZia0Jss8lxbHcTnJupaV5WDvbxY0E4ynofShicv1U76 -B41xDKP/8hFWck9LiMDXk9vrbQIHvGAcsYr5N/jzIrDiEXxvGQKCAQEA5NOegb6m -Ak0mXg+XKhZnSdR1KaWT4/hbVWqYjwxAeAJfMaxjD4MXA8qeEAJo3l3ETkdFCHp/ -lr/BZzNYXnmHU6uvDn2Xq8gPO04ruSV0WWthikXb5gszXdkUH+6fryHn6L0kYJQH -NARQzOgdEcmTP7sy/5GDKzydWbT5qRNOnESUWgnJi9ePjGB9zWxn4jx9AfOYtozh -UmEgofSDGbFlamQic8HGnSJFgOxIZ0AfurPIRSR51gvXx2D5TcsPjLlDrY07IcF3 -DjqfJl0gC1XN5BXdpPvjvNrns+ZK/SRoGlgb8Q4tZLQevox9W110amvMeZj4yMTK -9mgGOSYCzZ6U/QKCAQEA1mBZ4Qwpj1DNRk6PqlfnLSRYTb1gO9UdvdE7a33CFuTn -HZ2lgS2xt+zvqhrcoMuU8o2cfeQTFcP+Gjb2G9gxvzDBqmwC1IL/Esjzx9hWssCV -RoMEds2OrS6Ke4OeZj59XldhU83DeX+HEJylHO1UXwN8EHg/5dfPrVCeGsMdh9qb -9VxxiAm2wAnCU9pvcTpfimQ3L+VrqZvZyRfi8+/ZKkm52KO/XMFTvdAM3mhjcxH7 -Ipd9jQX4bwNZBB8UWaqm7pqhDJg2j/d+0lhwCUZzwwasTV0E14/RlHNsUdWlWhoD -/e+yQr2BgyvIAIvgBW8JA4RVq86S/y0gC/LMO/TQGQKCAQBB2rlaY7DJJsTs+xWp -EiuFrvRNGQ734+j9KyFewcrn/t7An/keZL7B45UbzGW74UZ2tMIkT4TasLMLbVZ4 -UgdlSBqoU/LLiFcB3Vxt+16BwYqfzb0cdorA7pGBIx6nu11PuOd4OAHesYNDhWWg -Ud/jzo89x/X1AovSXmgfhaPxCzeatghgC5iPcNGjxhgbnwbnAeEoYGEUYUmP8pus -UEZ8mPblU5ZCcLOKB/ZKaMT46Xawl2/M7zmZcsos3kzKViMpFmU3MMN/v9U/qDtp -p7cKdlSEf82p82INfzCDq++d7U+VT1w3CDN06V/GZJ31ZrLBKAopVaGHyqZH2i2i -WYpNAoIBACmfr9BoJh1/mbLdd/WpOyORKTbnwstlMgoUcARRJn86iPZuyI4QoSSb -TePZqrWRVmO/K5M65hFjhUpqTWKJGJy5LYIZ4yuIbonJAPNUhjA0bkar9cULBFzy -rb0xmW6sRlBnqhv4aDlOkhHkkR9lB9rTIUW+ankuvVBiGWo4eE8DvZYo30frltku -2K/kqd3NppTl7dN4EnGTo8ROZvr3EMwSu6nE+wUr4G7YuCLdPxwb8gAB8dbmaUsn -AXocUh96kYqTwRxo8FO9SqgQYMf81/ovPUfv+7mwO40oygzy/YkGmB1shFIbQuzU -lJvRfdXyyC9DbllQkxWfdvaanLS3r1w= +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQC1l+TO19J5ZBVs ++ZVzT62wM3JRh+CzO9/ogc5A0ERwVsfBFQYz1r6pGHHTxM2w3auuehyALbiW95eV +vkpu0QyOd+CaH4K0DI/reotYyvAX5+Qv5tnHGAwFKNeDnNG4hpCRMO/BlohZEh3P +pcm7jvMVpPhSM8ojZwSvHsEMBVb4QinvpvROLIDrA/zPQSlW0S0AZfvV+cjxIoDj +Tsg/M0Jn+6VNNRrAemx8aKv4TyOh2b+zYNa8wnlhdspTgjCseMeCORzDOKMP784p +0xWFwQzaoS0HYrxLoijiwIw6/I72ADQxkupW4OY22W6qjheDe67M33/KZVZ9lssc +7QKGBuvkxgsMq63M+VI8umY3gsIRLJKEwXkqviIXNCqw+5zMjsRBL3UUBjHibD1c +Hpc170xf4IOMa0Nk5svNy0tQ8OdTC90t2ni99O5E+MOHNTn3mr4GChv/+PVlWjf+ +2PWksmhNdTBkE64JXSrf3fb4OScbkhwD06YWVPtak5xv61QNgL9qsrARcm8f5jrA +lzqU6rYHnCkjB2A011DLh3idwxZfOtxT1jfu+NMP2N8xFgIjtp8yiVRQopXMA/HT +/3J/INO6SLo3t5nBRjphDsaOcp3tOlAUYa++2QXM87+QKBlRBgC2Mr2iUonqPJFm +Aa62TudYGoBKqeVfz5yFmclUGZlb/QIDAQABAoICAAE49rAsDa+JOAFZN/pg4lqw +4nzhxUWeZjazwj4/kBOETfFZG65gYPvBRBeg20WkmaCyLbTWsO8bqWa+CK1gT3mc +uslCh94kIanBH4XA0NJ2U8hNcN3WpeMySWxstnzVqaVj3eYPkARMkd9fW7cQH6NI +qle3OhsFhj54sj/17duGDoHBx6X87VPSYOwfc5cnV8Mc5hZv8mPHP0BFON3K02Wz +CzTYtBZMt8T4l7vkjLSUNinX1WNIZJpkKrkzTXV00IxCyJSBafO4NOHm8alarbFl +4AXbwd9GUzNe4QqY0xZHztcRkp73xvG0j4r2c+0zN64RqpwJapA8f9RGdYjWvMhD +8gB39KWCKjuVsFMnDrepmGjb2D5LgOlY8UTaG/9bA7U8QwJeCv7c8Z1GO5LxUIT4 +OcMoIkC+ooEOIrnty2AdWGVa8yyDXNaqejE1v76ZLKhM7lZxcmAJ4YU04gGhBp6k +wGxQHFPdfe4A4d+pfNsz8WnBKGihZJaC5EEef8loKuu84UezPVbup/YZqgfYWmQd +FoIxiY3aTre8jqEzSDiff8oKmjkSB+ab+ypCXxzzTDO4QV7BItdaV+T1ok5fdGqm ++k+M8pe9Fj35HTuBItwsvjPlvow5pt+qIe3YNjwgBOdrX+AeO5PDscZbOscAenr1 +ulCE/49ZIvT5kfNX9h0dAoIBAQDqI9hDGigVYChG4js7auJFMMouo6k19BbcUCBH +2jasHUI9MZrUiMZRY574YydPMA9jIzoNE8KuJUT5uvjIROYb3HbT2TxgOZhVCYr2 +Zr59AUtg4uIiXVqCTioLkoxc1PxnC+UR0kL162yS67w0xy4cEL/1RHSN3Q4NG1rZ +IC1Fzb5miSVReRbWLhdfxakcVdgYH3hwyRSgQo4H1Hjmi6oLAYGYDDgGwVPKmolC +hr0NBLkoKzXi015BTM5rcbBbm6vbrrcchWg8Q79fpc5tfASNdtoe1UHUFoypX3km +7dpyOfK225p/Ko4NqcpqsyKA1WcEswjwd4LL4muXLCmEu/6LAoIBAQDGjCLsErpG +XN5Ehl5iIuMVlGgMYoKG3qQcq3pJAsupQxic7p49JNPVqQHHhUbWow9p+4bfOGwp +WIg9NiX0917Jh8yiyNIR9w6LdD3j2EX826McRhTBCZb58Q4/wxYZS63XDnguX202 +WE5i9s3RGmPt7YlX6V1dyqrp6zpsU56tC0mnsuWjWwtXa3vdFpV4uMtCitWphvr7 +b+bEFtAf9fxzFfVwd7D3XhZXqB9ADS3UXNc68nufEd/arWaWHESTmuam/DL5FStq +nrMT4RYvlxGQ8TIjj3K99JKAOvBQtE5OBycP7FKgl+HKa6W4Caa2/2rqqTajM0/l +v/zuVZSXYaiXAoIBAQCX2CM6cU4pfrDgN3t+bzTK9ndIg+8LBH+G5ZfpwUTPnjIF +CZSF3SXwRpEkOPmXS23PN1tAe943ngf7hRwulTA+RV3yyeFz7iYyj/vFDKDg5OQD +s2BANRarhxGRa7sHwUDnezlVs2ylm8ZQCf2GpN3saZhz8a89CW1BZsomIc36hRHD +4ZGIIuWChWW3QLMJo0p/anlre5yayk7eGRdHPLBMbu+isdnr15kFve7ibAtS1AtY +V52cusNXyf3chGCBKJXt5ILwRjxxSOZlm1wieNli/P88G8WTTARxG2+wpXSTu60C +lay+Z9S0W7bgN8hakQs8aveTK45xtydbZNKQOHM9AoIBAAlJGoUB5zZNN5Pq2NUs +HoSu2SLjJImcNf6lQmXRvKag3bDRNBNV+pY+fjfPkt2M6LajLc6Yu+4/FrgOJT5M +p2Ezea2PSMvQSxzYheB3B+boertix6uzgB0WWf0/aXQrZujs40//5IKrJJdaRYvm ++Q9ykX7MCjLNvKqN9sCENKKRKZOvOTAnnOMswrE3UixAMDlfdtij2G8T763yBy/H +eFQFjeIBpwQaoV7eHQAI3cDVyrcWChQJaPe4LlkM32Qr1Wev2c5uYAZvf56JY1k8 +bnTh4t9o4QvjOUrH9t8/X34ktX34JnEeSVAHMsvln6dlUKHC4ixFxRHQpcqbtARU +a+cCggEBANs4GG0pXMXyxhqHANzvhukWQ6Q3shwGXf5iCInW4Ti5JcxjzkIzeLE4 +ou+EE/3c7YJ5xXCPPyhfIpxA8ff4xpiarlazm0qfnGu7F6oAdJVG0qdmN1S5tYXu +b4NIf6WCIWWI+qxhW5At3/91TC7ykxT/7qkI3uwIErzJK/1wttP+nxgw6xEOyW87 +AsPwVa/c3Ym41Mg5yqzoOXda/V90hcm0Xbi44hZBgVMP+pLhAOyfnGG56JF3ILit +VkEGVALXop6m7+/aGMr9z+xcCb1mp35r8aPP/GTKXvzGfN+yNx1di+kbXu1vIHd0 +0oKmfeNXirYgQwbe3yjrS9cCOW0+ERE= -----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/server-req.pem b/tests/integration/test_ssl_cert_authentication/certs/server-req.pem index bd8e2e1fb7f..53408abf254 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/server-req.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/server-req.pem @@ -1,27 +1,27 @@ -----BEGIN CERTIFICATE REQUEST----- MIIEmzCCAoMCAQAwVjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGc2Vy -dmVyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA3ZeG1vL3Gvf4hQwE -oJhscgkLxCua6FsD8xPw13UvpjCTP3338qgewpXZtSltiCw2FOiRp5WHcb1izhGu -5+HCAa0Ns2u73w1OC8xiUwTf/Qtk2BhT9dxU32YtoGTroqU/jKZBQnyRtczFkdd8 -ijcYkuiMX++nOgA1MrexpkXLiknLSszO4VAEX3pKOYgTfGh6Q58WGusUhpR1xw8x -xxKhKbJPokP3iuqNxHhJYO7++LGUs6pXiJBHxS4kSDzb7gMBBCESEIPpP9kuu95H -QDZcn7M1121BgmFPZTjj2IABBYG+ZY0l2SQU1t7hZciDjSuSOBB4BC8vR7NKp6rb -VwsfHeSP3iP/mcfRjxd8fIJDIucEHj4gaanXttJEWA7z9nhnmXhU5ymKagrIVQCe -PtSoPen+EvfhmdcH87IEtZQO/vaq96k7yPl5yoQtod26slkvggUIpIAA21EYGU+B -M4V76u30ghRxUwOsWc1zRFTuw5Ea2G75KtOmyJdTr/b3GwjeCMzriSTMZZNg79wt -5lKMdx5nDnTvr77Vp2bdx2xWO2TwlevDm0n++2A9zuaJ+URULoZPgIp67AyC+jYo -lcwC1wS/nvH5ZKmQ80ZNZhqYVVxnET+p5+Fba3zfgZm5vKbt5ummLqkl+r0F2+xB -1tbQG75X28SJT/x6eRqFfM6iP7UCAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4ICAQBc -TPShRdB7ZIL4xQNAxWLGoEbshbY/UpJQZdjojxn27roVwEhwP6B1/KgiVKV2X6bE -a36LUnaWYllIMAh4oOHJkIm2gZ3xitdEK1anCf5NJga7TnkwGfD4jTZA91fWCynt -a/64s0KQggKsUVY12TTJVQvOH/l9RMrXOq+jgIh4OURwCBtHTCS6oOp3eF04pEu7 -w54dMAsxp/N9AQaLh14IZUZAQ2v5kdipL99EEQH/G6llU08XrJ4LfGgwDkUjJSsA -TzrX2uk9MLHJVwjpZ99ktjNBs8Gyr4fGOmstT5TXEOO6bVhqZDC6kEL7vrmFSnDZ -g/9lrd4wLUT/STt+E4Qukedi0n/419IpkIAE5C1HOXRnQaOUdcnrLixUB21mBHt/ -n7gkwdY7Cu77dYvBIShzeCnxiJED0+XrBPD5yPokxEjE3MmiIK6meHHCuIwqLjX8 -I78ysv4COeH0svwSjvJInveS9QRCAWBpdvskxPJR8dpoysAff+jiyp3ZvkwcIiUO -Vbsusorpp8pFJXZxvPDBXCy5TOlFnIG/9itjPj98pFRIl5bxzNwDf4wrkwHEpR3i -jpM6y+/RWZn69BpTeAG0vZHhGk3SuXdU56cRzatys4X3biCImgDqeJMUcAoxlrIZ -vgbJVTorwqQmPz5kt28b8ddnUVobbZEPlRITcqjwFg== +dmVyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtZfkztfSeWQVbPmV +c0+tsDNyUYfgszvf6IHOQNBEcFbHwRUGM9a+qRhx08TNsN2rrnocgC24lveXlb5K +btEMjnfgmh+CtAyP63qLWMrwF+fkL+bZxxgMBSjXg5zRuIaQkTDvwZaIWRIdz6XJ +u47zFaT4UjPKI2cErx7BDAVW+EIp76b0TiyA6wP8z0EpVtEtAGX71fnI8SKA407I +PzNCZ/ulTTUawHpsfGir+E8jodm/s2DWvMJ5YXbKU4IwrHjHgjkcwzijD+/OKdMV +hcEM2qEtB2K8S6Io4sCMOvyO9gA0MZLqVuDmNtluqo4Xg3uuzN9/ymVWfZbLHO0C +hgbr5MYLDKutzPlSPLpmN4LCESyShMF5Kr4iFzQqsPuczI7EQS91FAYx4mw9XB6X +Ne9MX+CDjGtDZObLzctLUPDnUwvdLdp4vfTuRPjDhzU595q+Bgob//j1ZVo3/tj1 +pLJoTXUwZBOuCV0q3932+DknG5IcA9OmFlT7WpOcb+tUDYC/arKwEXJvH+Y6wJc6 +lOq2B5wpIwdgNNdQy4d4ncMWXzrcU9Y37vjTD9jfMRYCI7afMolUUKKVzAPx0/9y +fyDTuki6N7eZwUY6YQ7GjnKd7TpQFGGvvtkFzPO/kCgZUQYAtjK9olKJ6jyRZgGu +tk7nWBqASqnlX8+chZnJVBmZW/0CAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4ICAQBv +CpHPJBbcSRdDqO/iwJeySMSwhcM85CvyJmJNmqHLjXCl+cKQ1uJkjXFsz6HlPK1L +kXY5/ubRwQ3XuJIHi4PSshjEY4AXYTRJwTq8k2PGbDI4kBC36uEDQRI5eZBEH83E +KEvNggWXp2h1M0QBHJuG3qkrRnKjzLCUJ8Y2NjEq9+xNPFRR86WGy91uTMMct8xs +6bPPCLB6ySu9jsaenK5mTWEtYIAkxHJTK32STlFSOsost9vyYdvh7wd+mFDleD1K +vkZ4C0E8CWJISDQHHKIZEFaLFV0fEbOqvV9R18QTzAuV3uVzpP11pJxtciFHiuHg +/MCLJHWVcFNX6dSF4m/RmW8MKvdaI+fcuBgao1h8x6PMGPkPHYDQvupWwD0w5hrl +axqkXLpe73IXP5XMGeICVD8Nt77KYKW6LbtRNKIU28MfmhLn/zXqF0tf9EYtBn7s +rDwzzPN5ywhV+YXfQYSpCzheVMyfxjM3X060V5hyRWc9hMS7aC56aMNmEDI72IWl +ZAegSLnU5YF2iXaYRfftTPYiLL1n+TffK56qPI5QSn7fxuvG3N7cvVC3zeli8PGJ +gWWw4zPwRFvHk/NSepN2ZfiLZpMExZW4l5jvUrfWWjdQdh7B0a6JD6YfDS+Ot9nZ +eZkLUSJnrGqHq63BGjHDBNJo1A3GcegeaYrVkaGDzg== -----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/wrong-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/wrong-cert.pem index b56a10f8a92..6096794a1d8 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/wrong-cert.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/wrong-cert.pem @@ -1,32 +1,32 @@ -----BEGIN CERTIFICATE----- -MIIFjTCCA3WgAwIBAgIUMcX2R8I2H8vNtASHi0EoufIgWEUwDQYJKoZIhvcNAQEL +MIIFjTCCA3WgAwIBAgIUF+tVv6V0/vm8r0Gtvaro922XodswDQYJKoZIhvcNAQEL BQAwVjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xpZW50MB4XDTI0 -MDYyNjEwMjUwNVoXDTM0MDYyNDEwMjUwNVowVjELMAkGA1UEBhMCUlUxEzARBgNV +MDgxMjEwMzk0NVoXDTM0MDgxMDEwMzk0NVowVjELMAkGA1UEBhMCUlUxEzARBgNV BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0 ZDEPMA0GA1UEAwwGY2xpZW50MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC -AgEApABgxjCrAiDY529xEjboRE/PVwgVStD0dPE2vSOOqxfVYYOqP7VF7pG2PeA7 -Ek9Qqv2cqaDWbDWn5vT3AiN+aslfHtXJgGjPBrpOQ3Me6RSu2aUHPIguabUjz+kJ -VSkoj0pkbAqqIxKse1hZtUNHVwOmg/PthkpGPNxofNX1kCfBWoNJyuKFOoGeNgmY -6joY10zTaiHpq8hhA3b7WY35QdNGD7SwkYBvTjGGzBr/hu26fhX/DnceZ9kf9n6q -899gB2kZH5T1eTlShh12TI1sHa+BGz1YwR0HqM88zXDyAf7bWl7Hy5f8e9keZdx7 -Fx/ws93Bb3GusA5zwUm1iUe1OIwTLFlL+Kdkr0qzofDcQ0ZnNwrME4oAhCJFwSnJ -OWnrCmUou2XXj86Xl481uBana30zzJ9TniHM/hA54ttHsva/yB8tyoXcI4FASwk3 -GdihsOBbRS6KvmeKpEXQpsvBQ9GejSL/UUWuKg+O0ysHE9+QX/+OznFp66h/x7PP -Q7g6ipwAjgwuG5jm/Czz+dw4j3Qp5N5f7Dn3QhDzmXkKgzRzirKh9XVQqUFRwlLn -8VuzAhD5SjRN0JDE2jlt0Hin90zx/nkOV2b5hTYu9NVgmrfCye6uB/qsK7PQBh69 -9i4+8tBGXrcS2Nenm+Hm12fFhNum96A0ahj134H2ks4JcKcCAwEAAaNTMFEwHQYD -VR0OBBYEFIZYdI/00qzj+5JqEzEJfpj93AphMB8GA1UdIwQYMBaAFIZYdI/00qzj -+5JqEzEJfpj93AphMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB -AE+9y6Ea3pjIegOeffXzI725vXO0OSuv76vRoswUVQDKL7j7QcvqdcKCK/2hmeMW -MBNwKZneG5iumH2Yp5RQ14arihS9+SYEpgftvfNOZrzZ37RttJhSjBQ7GUorH7pD -uQ4s1unqKXv981JCtlKOQh5ZKthHAMsP/sfYuWg2ksVn1hvFTJZirewVUOgR8zPB -Djl6PdjZLDu1FlglQQ5YUNgsJbAqkBPcA0hEwGU0j5QEncvdspn4LiH2/mHhnzM9 -3QEvsXUxgJo989Am6+E77XNX0wcALj2zUmPPDWYgHgLKO/ZcSAOQ9JaauVhUre2D -7jPwnN47yLak5obVcyCpaDPKYk6sEUZkiWRaONvugoIbjYivmB/BJc0njfVA0kzT -FDwpUTtSddZgHzdTXe0p5C7IGmYkp/vgKlSpSYY6+aCiVApJSdJjL6FZKoOXqDnr -OgoQGSOJif4mDeipKOdrb2JtYwJkRl0c1S+tgOi8PU+ROvZxQGWI9/i20H58M7j0 -r/WhbudhcAqWglk5WOpCodbJhXffCrbUm5NjoFr7AKswxLJVz39WIe/duHPEGV7v -jLd/zj7eJRv5ycDyt91rbGxQ9NKzEx+by/5WIZTi+z+2PG75tdpQUwgEIh1c/XOt -6uXtS0sNnnjHVmXPBC+Myz+1NolYWjZMcBQ2xGIORvm8 +AgEAvKjjhLJvnrHpaxX/JgSqymisQ7IvMoiEapAf3fArNt1dg5b6E13w33ZuwtJ8 +Dem8BYceFWmu5YeOWQP038iJRDr5uofVTJAqyVzt4aM5YJVo7CaXVSlRRLhDN1zu +WsOnsC28uB9+0zn90729iJyU0DbQ2Tym/C6emzBS7OcD8cy84MvMdvyn6F5O8pjQ +A6V7l0dS2fD32GbAg8tpRABORUGnYDDHUIklwHQxQtZWQmJneYKGl0M3BdGaCA9g +AtRlRLwpNBQYwk5APh8TKNfAsEg6565IQ8sV12xYzphF9XMTFkRiLtyqMES+zpqu +rUu+xZ9gKFBL1rr+Jm36cA5zfDg0xdyr0u+j5zq2adM7lTctycaCw0pn4l1rGZHq +A33LV3Qgd4woCui2asgPvEiJU27qEcK4RM8x+Mf9vpm6jGhYjxLCAxoMHDaAoKKL +t3xd4PwqG8VBK3FbObU9qTXYnJW0tGvoMyo4u6onIGyQtqLul/Va8Op1QM9rOr2+ +3zUiNT5Zocg8EVb1gT+mJP/axSjCDAhk0+lh2HenEAR036Hicefp9O6SZkBdarTJ +ffqdIRNgbq5uIBqQ/Q6/A9lt5lR+lz1q+ZIJiwBD4ysE9dqwqUQboHoQg/yR80bu ++mlLow4E82rWW+qjD1FODLXukthibPh3vOLJyNl/UOKrj8UCAwEAAaNTMFEwHQYD +VR0OBBYEFGzWyPpUSrjEFc4yrS8QHu2wpVFVMB8GA1UdIwQYMBaAFGzWyPpUSrjE +Fc4yrS8QHu2wpVFVMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB +AGfn+XugP3raB31fEwqwN/wshe6jmcq9q2XYW+dHdrBq++ChE6Cw6BpjCFKLWWzr +udIo4opiDTkAauG3BOfQfUwJgyL8s81tAfOPxrB8O9NcvDNK92RugmT1FWGkvBV9 +lPRExxkjPbhaNzlcOIcqVX/O1eVuv+Oli+ZZBN2mwNySXR0VZiHjOV3eh6Kf7c49 +BWjgClWlGe93ZNTyaUJ4RXasBNQ4551Kvqw+pEMaaVLU3uVgJk2QQT+lSOJQrwqc +DHC7WqYS/uCm22CGOt9vP3dK5Xo8AMPYLvB3O8JcWrNdwJ/SoK6lTmKNLhqYer0r +UkP6noRRAJcXbTsH5iQvj9Daw6qjHEs2RuV2iRFxvCzAzf5bURAwXkjnCsZ24SJa +oc4yG4jfjzFUmO+xQ6+ykgMOHrLPHXcJOfgRhA0XW0BiTEKaZfI/M64W8kfU9w+M +Zo7n8CAWp75VDcAYXDfO8en6YLphpCkfhGrBqO4DfPpsHuNIETRv4wMexiaAc5xK +dud1EmuX46vreEMPtjlsO8BZTtVT5JF2P7y4wS8vd4s9vVHixvfEqVNcixVDNrQh +YZrUiTL14kPDrJy9UfzylWeEc4cQfTwzId2QKRfX7+u9Xe6LPfdsGVN5gGXdKtkY +z2aV9XUEX0VmDmKdKcvhMu+ReOsfM66vFheT1ZEAKNnG -----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/wrong-key.pem b/tests/integration/test_ssl_cert_authentication/certs/wrong-key.pem index 3924eac91c2..7a7bfe60326 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/wrong-key.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/wrong-key.pem @@ -1,52 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCkAGDGMKsCINjn -b3ESNuhET89XCBVK0PR08Ta9I46rF9Vhg6o/tUXukbY94DsST1Cq/ZypoNZsNafm -9PcCI35qyV8e1cmAaM8Guk5Dcx7pFK7ZpQc8iC5ptSPP6QlVKSiPSmRsCqojEqx7 -WFm1Q0dXA6aD8+2GSkY83Gh81fWQJ8Fag0nK4oU6gZ42CZjqOhjXTNNqIemryGED -dvtZjflB00YPtLCRgG9OMYbMGv+G7bp+Ff8Odx5n2R/2fqrz32AHaRkflPV5OVKG -HXZMjWwdr4EbPVjBHQeozzzNcPIB/ttaXsfLl/x72R5l3HsXH/Cz3cFvca6wDnPB -SbWJR7U4jBMsWUv4p2SvSrOh8NxDRmc3CswTigCEIkXBKck5aesKZSi7ZdePzpeX -jzW4FqdrfTPMn1OeIcz+EDni20ey9r/IHy3KhdwjgUBLCTcZ2KGw4FtFLoq+Z4qk -RdCmy8FD0Z6NIv9RRa4qD47TKwcT35Bf/47OcWnrqH/Hs89DuDqKnACODC4bmOb8 -LPP53DiPdCnk3l/sOfdCEPOZeQqDNHOKsqH1dVCpQVHCUufxW7MCEPlKNE3QkMTa -OW3QeKf3TPH+eQ5XZvmFNi701WCat8LJ7q4H+qwrs9AGHr32Lj7y0EZetxLY16eb -4ebXZ8WE26b3oDRqGPXfgfaSzglwpwIDAQABAoICACiTq1399qGtLN1+NjSyfx8/ -u+Ylqtb7AjDY6Zk8bfUpDXN2Fy5yFF5lkPiYPSVXmHbmDtftYoAdenBrVZ4i2All -z3IapSNvSyG4ANsxZYl3w5c3/KVecFVZKwYq+1MlvtJNLrGIpfXNjf1qq69seP8v -eQiW1sLuJ5ixU+znJz3GiFFzwFNBXoNORK3MDBiPzUufx4Mv5tfI2S/5RVEwDmbZ -9jC2nSUy6Nco69geKfDhas39dUDH+i7pir37MyLptqG+wCePPHkE1MU4Duf76a8i -mEf8ErSdESMUO0/9TPNvcihW4Qofjam624mKVq4vCegGyvBe6UDIIp3FNfREWLLC -ilit37ZVOHbq79qV+DobLADYXZGXrptzr6VqVZvvQUEwOjftD2B8GJzxbNxRl77F -BbeOhYA/IDr8YR7Qos0HjqDDekRavHKAa5kcf8yFVJaQycirHJZpjr3oNFktqg6H -9eb537SdPI5nHtgTSQSosxst+iBsjMCJ7rU7aowy9gKG75s9eME06hiQsukNcOI3 -hmBqQBeX+yLWh7Z6A2Y7MepHuYHWKereBGISR58bvTmNyI4mLWYwJZzjON2tot3a -MJM86gw83TuX1Qmp3+NjduQtdtMjDSXLN2yBbK4CufQYaTxK1xdHUoK/uvG9kIq3 -tP+/fiTHZHyuTSSgwOypAoIBAQDT2Vj2uahOypboLv33XtFr2kuDe1sogpLaAVu4 -Dv4vO835gto4fx79rK3u2fBxiVx0yv7kwitvpcwaoMdkUEoSBtOugYB9UezW+SX5 -91bpY0mBH0ToSmosPFVc6PEki6LRV+VGZ1gFXU7uZ4Wia9opENfT7d8cjBQ4NZ/M -sCyqHR2KYB82DHx6Lrtrs0eWn33N8BVgsy4xSbOi2YrgQCnJvfPWVYtcXjRbplj4 -jCVGnPlac0Z/bv3Kb7Q7EOS+d+RFi1ZpsFYPbW5KRhGshzOxGw5d/nCjkEXCV0ht -uK6KndjFOvCGfikZW7WVpw7bkCe0W2Ko/JSX99ccJBDyau1NAoIBAQDGLj/wVxss -dllwswBOAV3oSL9xbUPs+Xbr/FK4XKcL7wcY6cfdZBlsQLQCoAzyTZ8t+g756Hlt -a8qmW2/Wvdo+m63Z2ecnbI9pJsVyYeT8pVumx4biHuXbRBYO/0ZZPtB5kTT6Hzhv -ZHxoUj4jb7L/5kwEdEPFIZX4rVEXY20LJL5wtv2zEQylQk9kunjUgrP1L/GtcNh+ -QRzLXiJWAoC4HWcXmdxb/Hc0BU5geSwZL4bbl3YL3lwWvkd3aY17T90EjWy4uA6G -tsHCxbxauul1q8OkmCcLEfhYnDh95YoVddR97XhC33S0v4dYjX/iS46g8fJ0HhRo -9YGWsD+tRevDAoIBAFCp/5/iTV3C8fbyfa1FI0SH2Bz2SV2Bal0sCzpoKwzdHq6U -znaYoLpCl+/MeCsi/FtUN/3umQ9n9/FjqshdcfavNsbJdJ1DJoUsVPN65FL1hTVv -LJOuUgMJ7g70e21I5fQEHb7S9scEIlvQeye/HVBpo2SEvGFoTQKiGHid1EPp1ies -NfYkhvkW9jIqD2Yg0IwrkFhDoaEOySGG58Q/ainw8/l2lRvUmucSzenFoyPh/Wgd -YIiBQI1mPyAGbLLBf9+jEIIprHsvVcFeMLiaumoDPVM44LbG5mj7Rw7QNVV+iN2A -dbkgLJIFQ3z6IUQk/ZlE+qoRkprSuctzSCil4jkCggEAdiYWilNz6NL5yX193gNk -l9nfAGFS0JF8+31nV3AtSqkLAyhEtlE58ta0Oqhub3olPwTILucQlVJg80Kp700q -Mo8fWzRUYaWP7fFmXyXLnW97r3dei6o+ALWbrP81UnlnUkJmYgOA4q/2lz8Iupma -DoOeqD0kNf8q6KFzKc1lsfIK8ym1IC826cMZkAS3ioINhUw6+dq/xq1M3FVXhQ1i -7eDhmClrPQ/LhSDwtAUpbC5waLPodXTwU8LG2oL8DRr0ugUSXyGjz15fL54xB6pN -CpEHRzZKeIgTFci0ySGya87eiuCrBLsxWZyhtQJOznubIYp8sAtKwbQzuMGEhOmd -fwKCAQEAlZoi1SzHstg6PfpwrIHJV3imLa550k9hyAu610CKjMsg6IsFsgu9/J0b -9hkhlafeW+p9zhKSwjl3aCuWUMNE53R+zYmMIJJrBzejC+1H0SKW0Zix9+ghixOX -da1jRaUxUqApJYvvxUC8FbnATM/Eq0ofhGkG3o575SlO+54twJO+bXGAUf/C6xMY -AQUQh90pTbZ96Q3Wdm2Qmrhd/GUaC6k1vAHVHHU8WQHiLmo1fF4gL/TqRv5KEPUM -un6ld7h8BEWtMClhSIiL2h5nvSYGcB6Lai6rPO0UUbGkWBQFpGaeglUmoYi0ciC5 -lMRrRHGUiWHW9C4/siOKYrHBeH5oNQ== +MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQC8qOOEsm+eselr +Ff8mBKrKaKxDsi8yiIRqkB/d8Cs23V2DlvoTXfDfdm7C0nwN6bwFhx4Vaa7lh45Z +A/TfyIlEOvm6h9VMkCrJXO3hozlglWjsJpdVKVFEuEM3XO5aw6ewLby4H37TOf3T +vb2InJTQNtDZPKb8Lp6bMFLs5wPxzLzgy8x2/KfoXk7ymNADpXuXR1LZ8PfYZsCD +y2lEAE5FQadgMMdQiSXAdDFC1lZCYmd5goaXQzcF0ZoID2AC1GVEvCk0FBjCTkA+ +HxMo18CwSDrnrkhDyxXXbFjOmEX1cxMWRGIu3KowRL7Omq6tS77Fn2AoUEvWuv4m +bfpwDnN8ODTF3KvS76PnOrZp0zuVNy3JxoLDSmfiXWsZkeoDfctXdCB3jCgK6LZq +yA+8SIlTbuoRwrhEzzH4x/2+mbqMaFiPEsIDGgwcNoCgoou3fF3g/CobxUErcVs5 +tT2pNdiclbS0a+gzKji7qicgbJC2ou6X9Vrw6nVAz2s6vb7fNSI1PlmhyDwRVvWB +P6Yk/9rFKMIMCGTT6WHYd6cQBHTfoeJx5+n07pJmQF1qtMl9+p0hE2Burm4gGpD9 +Dr8D2W3mVH6XPWr5kgmLAEPjKwT12rCpRBugehCD/JHzRu76aUujDgTzatZb6qMP +UU4Mte6S2GJs+He84snI2X9Q4quPxQIDAQABAoICAApGgpncJCM3GnkIKiz2bRz4 +JYXMYzCz6c5qCipK5fenh+veYcGDSNbK+w5ma3ZQiDqe2N8esfVzdCfaBNDZecx7 +D9X+hvoUEhiElLpI6xudF6lhErYDOZduF88goyTTakM4woIeyQgVLQOG3pddu+c5 +TRe/63Jp0Z6vO50Gmhrl5VWzE/BZI4YO+OrSsuW38irTqioPq1gghJTJE/MttxWj +lUuybHCw/5rjWTmENg+Ij405ND5x3UHWYDbXK4oL6nYbb30UKSMQIwSfKap8UdTo +IjkzL2Ft4sModg/OkGTlfyEj2VsnDqfxXpkfKKtsqDfYTeL0OKU56xTJwa2vw89k +gtCP0t/UQzv2MRPvoCOKEIj8AiUj5X9uLPY6ijA4jD396OVnuZh9+KgpLwB7piT+ +WNrbm/Nrfe617kK41NGAVpVIMo1y8fSbplZqMOWSSqLO0eyHkEJK5yK1WtFfzVAE +UKH5jRjS52XcLrLa7VxAcsywv45UNiSiYRSxBY7MOg1mL94BlcghW/YHg3h3vKZC +NVEBoaDHNd8Q3V5sQgpENUjXmWFaE/1Su9i/oooy/GAwRBySHnUkKPvBP6Unus/S +Pvc2vQPMHpfMbPc+L9YxxbbgVHBembYJn7qIejoqfedtaZoqB240s4AowxDiK88f +QN1L8TiaDiGJBIuvfhtRAoIBAQD/2TAXiLZ4fkbcIbpxrU8nmQNrqxZPlqVLKLDj +4eq4wBLMywOsW1RGoaipooWmEWICgrh11YtoE3GjgPYh2x+oXPOpUn5P4Gbx88o7 +8rKMDu//yRKnj/e0GNVy+ugXqVZj3+o2Mdyv2nlAqLEeZsuzFi7nXoaNJjOsumyq +e3eP/iM9jV2CakhRPnZH9B5/6dgFjwR4Vw9DU2cCaQZtIEOGANCEfP3UWaP1GW0Y +eMtDYiGAxkYqpCDLwbQT2jbgCZUUBtIbkZhkVgXDrE6K2Ey9M7HroLo7Tl5NGBMA +V7lZxpVa/4T2R7RFlXZFChDNul9lngQEFHlrbITJG8iOLfpdAoIBAQC8xYIlIyLW +q/mzUErH7cqL76ORcm8ejymD+ii+ERQpqwTQJljBf5mnadfTQPN1kD7YLAXSgufq +oEnVNFYR/k/M2aztTWfsUNGW0MAhS4tTewFgcia8macfhOccP3mItZ3KOQgpXxnW ++h8DXLlq6FFvybK00msribpxNQybS7y4DK5VMRbg0aBZAn+6O4Lz5uLR+PU02pFG +FgaLVh1E0h2mp0ib42BaRyVJI0CY402XKrSm/BDnHhGfoLW5kK53gvk2g6RzIDwy +qF/DJ0HHlAQL9Z9OxoMy4haHXcBGP9rGJX59eyb6adinsSBl7wXStdjvTsmlNYD7 +KqltgoAdkaSJAoIBAHWZs01d/eGsyY1tw3F1JCkjYDshTQQQTrIZZhWZRnbdcsbH +mkyjPj2pGQnANoZ6/v4AcCFZotaX+WgaYwh03DxbXXS7AmxczXTxhke/6uoOA6sj +FXwH5OfXcmWDhyM5JwiJZ/K5QKNkXM+nuqIqxf7vd2fKPzaqFJ6UolZKok6Bllk4 +nX7Qs6UEfQHd6BcLucv0TS2zdsSPlY26EMYgSmlR/oannVT6Ty7eHRNekq/Kb3Pt +r1ryTlDaHJfzeb3JKckmyXT6m32jPMsQbJnNiph9Jo8UNgYEo7v0EOfbasslSImn +YcqCcw55AQAC/G5T+H2RAG+PqbADFZYLO0h/QdECggEAAJxS83PNpQwhXqcf1s26 +HgHEcbABFAQ6iibBAvxjKE9ZUKMPHnEfOh9ph1FqeDLjQSfDTQ8HWQbztjDZJm3A +LFV37byWXXlVdDtwo6ru5HI9auZzoaCNndh8NuctqXeM7x6IHDomhz9/4i7mmqXt +vYLVhSg5GIb1h3A4fjgcgCvqVHQ4Mrn63s7XQu72WXuhuDQp9uXOGn/vvXul1Jcp +aWSZI4f0w9X/FOF8UAJMOfT1aKTgGR9Lx0xpyhPhvJk73SVH3ud3ymIpDSSPXeno +qXE99q9FtWBt2jo/aPrD5mgwpI9FbQHypXg5NpszVZ8o+H00wcgOhsF4ktYdO/tA +oQKCAQB7TZVBjOsOY/S8eWbTZo0w5K0THL15k++gfZAOCcQamhGe+Cqar85kyJ50 +aAE+p04i7ZuNAtxAetPU8GvdYK/DNdDa+yeUTvBjl7Lpi6FjsPwdCSDQJNQk4qCW +rPHnBfCK7wHKpOKsQuwKXH4Ndo3En6y4EmICHMdfu9OWZlNhKc2o00SdkdXF9+Wt +7iKNb3JDCG1ZOEISMkBdzbRjNfzzxoleQFr+2C3nUogMkGdgwMtC4JmgO+K5dsSp +/vNHeNUOYxLj8D7H0y75iMPR+LqhlKvFj6P3BlslD0Lb/yg80U55v821mYBRs8pg +AVHPFI5a8A+VunzBvaL+SEP/rwrI -----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/configs/ssl_config.xml b/tests/integration/test_ssl_cert_authentication/configs/ssl_config.xml old mode 100644 new mode 100755 diff --git a/tests/integration/test_ssl_cert_authentication/configs/users_with_ssl_auth.xml b/tests/integration/test_ssl_cert_authentication/configs/users_with_ssl_auth.xml old mode 100644 new mode 100755 index 4bd30163ea6..b697c010195 --- a/tests/integration/test_ssl_cert_authentication/configs/users_with_ssl_auth.xml +++ b/tests/integration/test_ssl_cert_authentication/configs/users_with_ssl_auth.xml @@ -17,6 +17,11 @@ URI:spiffe://foo.com/baz + + + URI:spiffe://bar.com/foo/*/far + + diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 3af88759e82..1adfa4855f1 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -369,3 +369,33 @@ def test_x509_san_support(): instance.query("SHOW CREATE USER jemma") == "CREATE USER jemma IDENTIFIED WITH ssl_certificate SAN \\'URI:spiffe://foo.com/bar\\', \\'URI:spiffe://foo.com/baz\\'\n" ) + +def test_x509_san_wildcard_support(): + assert ( + execute_query_native(instance, "SELECT currentUser()", user="stewie", cert_name="client5") + == "stewie\n" + ) + + assert ( + instance.query("SELECT name, auth_type, auth_params FROM system.users WHERE name='stewie'") + == 'stewie\tssl_certificate\t{"subject_alt_names":["URI:spiffe:\\\\/\\\\/bar.com\\\\/foo\\\\/*\\\\/far"]}\n' + ) + + assert ( + instance.query("SHOW CREATE USER stewie") + == "CREATE USER stewie IDENTIFIED WITH ssl_certificate SAN \\'URI:spiffe://bar.com/foo/*/far\\'\n" + ) + + instance.query( + "CREATE USER brian IDENTIFIED WITH ssl_certificate SAN 'URI:spiffe://bar.com/foo/*/far'" + ) + + assert ( + execute_query_https("SELECT currentUser()", user="brian", cert_name="client6") + == "brian\n" + ) + + assert ( + instance.query("SHOW CREATE USER brian") + == "CREATE USER brian IDENTIFIED WITH ssl_certificate SAN \\'URI:spiffe://bar.com/foo/*/far\\'\n" + ) From 8d6e2e26a57764d2dfc133d8b063578848fd3e2e Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 13 Aug 2024 14:29:52 +0000 Subject: [PATCH 105/215] New test for asan issue with pr --- .../03222_pr_asan_index_granularity.reference | 100 ++++++++++++++++++ .../03222_pr_asan_index_granularity.sql | 14 +++ 2 files changed, 114 insertions(+) create mode 100644 tests/queries/0_stateless/03222_pr_asan_index_granularity.reference create mode 100644 tests/queries/0_stateless/03222_pr_asan_index_granularity.sql diff --git a/tests/queries/0_stateless/03222_pr_asan_index_granularity.reference b/tests/queries/0_stateless/03222_pr_asan_index_granularity.reference new file mode 100644 index 00000000000..662880055bd --- /dev/null +++ b/tests/queries/0_stateless/03222_pr_asan_index_granularity.reference @@ -0,0 +1,100 @@ +0 18 9899 +0 18 9898 +0 18 9897 +0 18 9896 +0 18 9895 +0 18 9894 +0 18 9893 +0 18 9892 +0 18 9891 +0 18 9890 +0 18 9889 +0 18 9888 +0 18 9887 +0 18 9886 +0 18 9885 +0 18 9884 +0 18 9883 +0 18 9882 +0 18 9881 +0 18 9880 +0 18 9879 +0 18 9878 +0 18 9877 +0 18 9876 +0 18 9875 +0 18 9874 +0 18 9873 +0 18 9872 +0 18 9871 +0 18 9870 +0 18 9869 +0 18 9868 +0 18 9867 +0 18 9866 +0 18 9865 +0 18 9864 +0 18 9863 +0 18 9862 +0 18 9861 +0 18 9860 +0 18 9859 +0 18 9858 +0 18 9857 +0 18 9856 +0 18 9855 +0 18 9854 +0 18 9853 +0 18 9852 +0 18 9851 +0 18 9850 +0 18 9849 +0 18 9848 +0 18 9847 +0 18 9846 +0 18 9845 +0 18 9844 +0 18 9843 +0 18 9842 +0 18 9841 +0 18 9840 +0 18 9839 +0 18 9838 +0 18 9837 +0 18 9836 +0 18 9835 +0 18 9834 +0 18 9833 +0 18 9832 +0 18 9831 +0 18 9830 +0 18 9829 +0 18 9828 +0 18 9827 +0 18 9826 +0 18 9825 +0 18 9824 +0 18 9823 +0 18 9822 +0 18 9821 +0 18 9820 +0 18 9819 +0 18 9818 +0 18 9817 +0 18 9816 +0 18 9815 +0 18 9814 +0 18 9813 +0 18 9812 +0 18 9811 +0 18 9810 +0 18 9809 +0 18 9808 +0 18 9807 +0 18 9806 +0 18 9805 +0 18 9804 +0 18 9803 +0 18 9802 +0 18 9801 +0 18 9800 diff --git a/tests/queries/0_stateless/03222_pr_asan_index_granularity.sql b/tests/queries/0_stateless/03222_pr_asan_index_granularity.sql new file mode 100644 index 00000000000..b7f37dd2856 --- /dev/null +++ b/tests/queries/0_stateless/03222_pr_asan_index_granularity.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS test; + +CREATE TABLE test (k UInt64, v String) +ENGINE = MergeTree +ORDER BY k +SETTINGS index_granularity=1; + +INSERT INTO test SELECT number, toString(number) FROM numbers(10_000); + +SET allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 3, parallel_replicas_for_non_replicated_merge_tree=1, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; + +SELECT 0, materialize(18), k FROM test PREWHERE toNullable(toNullable(11)) WHERE toNullable(11) ORDER BY k DESC NULLS LAST LIMIT 100, 100 SETTINGS optimize_read_in_order = 1, merge_tree_min_rows_for_concurrent_read = 9223372036854775806, max_threads = 1; + +-- DROP TABLE test; From 99dcf7e60ebf91858e4b16d9235bb1623a78b9b0 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 13 Aug 2024 15:57:09 +0000 Subject: [PATCH 106/215] Add 03222_pr_final_not_supported.sql --- .../03222_pr_final_not_supported.reference | 0 .../03222_pr_final_not_supported.sql | 22 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 tests/queries/0_stateless/03222_pr_final_not_supported.reference create mode 100644 tests/queries/0_stateless/03222_pr_final_not_supported.sql diff --git a/tests/queries/0_stateless/03222_pr_final_not_supported.reference b/tests/queries/0_stateless/03222_pr_final_not_supported.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03222_pr_final_not_supported.sql b/tests/queries/0_stateless/03222_pr_final_not_supported.sql new file mode 100644 index 00000000000..a018bfc53b4 --- /dev/null +++ b/tests/queries/0_stateless/03222_pr_final_not_supported.sql @@ -0,0 +1,22 @@ +DROP TABLE IF EXISTS test_00808; + +CREATE TABLE test_00808 +( + `date` Date, + `id` Int8, + `name` String, + `value` Int64, + `sign` Int8 +) +ENGINE = CollapsingMergeTree(sign) +ORDER BY (id, date) +SETTINGS index_granularity = 62918, min_bytes_for_wide_part = 0, ratio_of_defaults_for_sparse_serialization = 0., replace_long_file_name_to_hash = false, max_file_name_length = 79, min_bytes_for_full_part_storage = 536870912, compact_parts_max_bytes_to_buffer = 525685803, compact_parts_max_granules_to_buffer = 148, compact_parts_merge_max_bytes_to_prefetch_part = 23343456, merge_max_block_size = 22854, old_parts_lifetime = 93., prefer_fetch_merged_part_size_threshold = 3610162434, vertical_merge_algorithm_min_rows_to_activate = 1000000, vertical_merge_algorithm_min_columns_to_activate = 1, min_merge_bytes_to_use_direct_io = 1, index_granularity_bytes = 9550183, concurrent_part_removal_threshold = 34, allow_vertical_merges_from_compact_to_wide_parts = false, cache_populated_by_fetch = false, marks_compress_block_size = 97756, primary_key_compress_block_size = 80902; + +INSERT INTO test_00808 VALUES('2000-01-01', 1, 'test string 1', 1, 1); +INSERT INTO test_00808 VALUES('2000-01-01', 2, 'test string 2', 2, 1); + +SET enable_analyzer=1, allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 3, parallel_replicas_for_non_replicated_merge_tree=1, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; + +SELECT * FROM (SELECT * FROM test_00808 FINAL) WHERE id = 1; -- { serverError SUPPORT_IS_DISABLED } + +DROP TABLE test_00808; From 4448880bbe11d80047de2c3988165b4b54710f1e Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 13 Aug 2024 16:15:28 +0000 Subject: [PATCH 107/215] Remove settings 03222_pr_final_not_supported --- tests/queries/0_stateless/03222_pr_final_not_supported.sql | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/03222_pr_final_not_supported.sql b/tests/queries/0_stateless/03222_pr_final_not_supported.sql index a018bfc53b4..6c2e05c12a9 100644 --- a/tests/queries/0_stateless/03222_pr_final_not_supported.sql +++ b/tests/queries/0_stateless/03222_pr_final_not_supported.sql @@ -9,13 +9,12 @@ CREATE TABLE test_00808 `sign` Int8 ) ENGINE = CollapsingMergeTree(sign) -ORDER BY (id, date) -SETTINGS index_granularity = 62918, min_bytes_for_wide_part = 0, ratio_of_defaults_for_sparse_serialization = 0., replace_long_file_name_to_hash = false, max_file_name_length = 79, min_bytes_for_full_part_storage = 536870912, compact_parts_max_bytes_to_buffer = 525685803, compact_parts_max_granules_to_buffer = 148, compact_parts_merge_max_bytes_to_prefetch_part = 23343456, merge_max_block_size = 22854, old_parts_lifetime = 93., prefer_fetch_merged_part_size_threshold = 3610162434, vertical_merge_algorithm_min_rows_to_activate = 1000000, vertical_merge_algorithm_min_columns_to_activate = 1, min_merge_bytes_to_use_direct_io = 1, index_granularity_bytes = 9550183, concurrent_part_removal_threshold = 34, allow_vertical_merges_from_compact_to_wide_parts = false, cache_populated_by_fetch = false, marks_compress_block_size = 97756, primary_key_compress_block_size = 80902; +ORDER BY (id, date); INSERT INTO test_00808 VALUES('2000-01-01', 1, 'test string 1', 1, 1); INSERT INTO test_00808 VALUES('2000-01-01', 2, 'test string 2', 2, 1); -SET enable_analyzer=1, allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 3, parallel_replicas_for_non_replicated_merge_tree=1, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; +SET allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 3, parallel_replicas_for_non_replicated_merge_tree=1, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; SELECT * FROM (SELECT * FROM test_00808 FINAL) WHERE id = 1; -- { serverError SUPPORT_IS_DISABLED } From 62e45a4c8fefcd846d531c7c4d52e4277435d3c3 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 13 Aug 2024 20:55:20 +0000 Subject: [PATCH 108/215] Test 03222_parallel_replicas_min_marks_to_read_overflow --- ..._replicas_min_marks_to_read_overflow.reference | 10 ++++++++++ ...rallel_replicas_min_marks_to_read_overflow.sql | 15 +++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 tests/queries/0_stateless/03222_parallel_replicas_min_marks_to_read_overflow.reference create mode 100644 tests/queries/0_stateless/03222_parallel_replicas_min_marks_to_read_overflow.sql diff --git a/tests/queries/0_stateless/03222_parallel_replicas_min_marks_to_read_overflow.reference b/tests/queries/0_stateless/03222_parallel_replicas_min_marks_to_read_overflow.reference new file mode 100644 index 00000000000..7fafd4d13ea --- /dev/null +++ b/tests/queries/0_stateless/03222_parallel_replicas_min_marks_to_read_overflow.reference @@ -0,0 +1,10 @@ +100 100 +101 101 +102 102 +103 103 +104 104 +105 105 +106 106 +107 107 +108 108 +109 109 diff --git a/tests/queries/0_stateless/03222_parallel_replicas_min_marks_to_read_overflow.sql b/tests/queries/0_stateless/03222_parallel_replicas_min_marks_to_read_overflow.sql new file mode 100644 index 00000000000..112373e5db2 --- /dev/null +++ b/tests/queries/0_stateless/03222_parallel_replicas_min_marks_to_read_overflow.sql @@ -0,0 +1,15 @@ +DROP TABLE IF EXISTS test__fuzz_22 SYNC; + +CREATE TABLE test__fuzz_22 (k Float32, v String) ENGINE = ReplicatedMergeTree('/clickhouse/03222/{database}/test__fuzz_22', 'r1') ORDER BY k SETTINGS index_granularity = 1; + +INSERT INTO test__fuzz_22 SELECT number, toString(number) FROM numbers(10_000); + +SET allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 3, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'; + +SELECT k, v +FROM test__fuzz_22 +ORDER BY k +LIMIT 100, 10 +SETTINGS merge_tree_min_rows_for_concurrent_read = 9223372036854775806; + +DROP TABLE test__fuzz_22 SYNC; From 1a8f45464c1ef81dc8c2c16c8fbba2e696762962 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 13 Aug 2024 21:05:29 +0000 Subject: [PATCH 109/215] Fix min_marks_to_read overflow --- src/Storages/MergeTree/MergeTreeIndexGranularity.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp b/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp index 2a45ab1d927..2f9a4a47b11 100644 --- a/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp @@ -103,8 +103,16 @@ size_t MergeTreeIndexGranularity::countMarksForRows(size_t from_mark, size_t num /// This is a heuristic to respect min_marks_to_read which is ignored by MergeTreeReadPool in case of remote disk. /// See comment in IMergeTreeSelectAlgorithm. - if (min_marks_to_read && from_mark + 2 * min_marks_to_read <= to_mark) - to_mark = from_mark + min_marks_to_read; + if (min_marks_to_read) + { + // check that ... + bool overflow = ((1ULL << 63) & min_marks_to_read); // further multiplication by 2 will not overflow + if (!overflow) + overflow = (std::numeric_limits::max() - from_mark) < 2 * min_marks_to_read; // further addition will not overflow + + if (!overflow && from_mark + 2 * min_marks_to_read <= to_mark) + to_mark = from_mark + min_marks_to_read; + } return getRowsCountInRange(from_mark, std::max(1UL, to_mark)) - offset_in_rows; } From a3978b24d0ee8847e5b332a08014ced896edae2c Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 15 Aug 2024 11:11:49 +0000 Subject: [PATCH 110/215] Fix for subqueries with FINAL --- src/Planner/findParallelReplicasQuery.cpp | 6 ++++++ ... => 03222_parallel_replicas_final_in_subquery.reference} | 0 ...ed.sql => 03222_parallel_replicas_final_in_subquery.sql} | 0 3 files changed, 6 insertions(+) rename tests/queries/0_stateless/{03222_pr_final_not_supported.reference => 03222_parallel_replicas_final_in_subquery.reference} (100%) rename tests/queries/0_stateless/{03222_pr_final_not_supported.sql => 03222_parallel_replicas_final_in_subquery.sql} (100%) diff --git a/src/Planner/findParallelReplicasQuery.cpp b/src/Planner/findParallelReplicasQuery.cpp index 39edb1e6516..cd02e01c392 100644 --- a/src/Planner/findParallelReplicasQuery.cpp +++ b/src/Planner/findParallelReplicasQuery.cpp @@ -52,7 +52,13 @@ std::stack getSupportingParallelReplicasQuery(const IQueryTre const auto & storage = table_node.getStorage(); /// Here we check StorageDummy as well, to support a query tree with replaced storages. if (std::dynamic_pointer_cast(storage) || typeid_cast(storage.get())) + { + /// parallel replicas is not supported with FINAL + if (table_node.getTableExpressionModifiers()->hasFinal()) + return {}; + return res; + } return {}; } diff --git a/tests/queries/0_stateless/03222_pr_final_not_supported.reference b/tests/queries/0_stateless/03222_parallel_replicas_final_in_subquery.reference similarity index 100% rename from tests/queries/0_stateless/03222_pr_final_not_supported.reference rename to tests/queries/0_stateless/03222_parallel_replicas_final_in_subquery.reference diff --git a/tests/queries/0_stateless/03222_pr_final_not_supported.sql b/tests/queries/0_stateless/03222_parallel_replicas_final_in_subquery.sql similarity index 100% rename from tests/queries/0_stateless/03222_pr_final_not_supported.sql rename to tests/queries/0_stateless/03222_parallel_replicas_final_in_subquery.sql From 29fffd34ac5ab769153563f9c83b134cbaefb117 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 16 Aug 2024 11:06:20 +0000 Subject: [PATCH 111/215] Fix: check if table expression is present --- src/Planner/findParallelReplicasQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Planner/findParallelReplicasQuery.cpp b/src/Planner/findParallelReplicasQuery.cpp index cd02e01c392..25481d06670 100644 --- a/src/Planner/findParallelReplicasQuery.cpp +++ b/src/Planner/findParallelReplicasQuery.cpp @@ -54,7 +54,7 @@ std::stack getSupportingParallelReplicasQuery(const IQueryTre if (std::dynamic_pointer_cast(storage) || typeid_cast(storage.get())) { /// parallel replicas is not supported with FINAL - if (table_node.getTableExpressionModifiers()->hasFinal()) + if (table_node.getTableExpressionModifiers() && table_node.getTableExpressionModifiers()->hasFinal()) return {}; return res; From 37c3f4a8707598022d94ce5b20c10893882cf58c Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Fri, 10 May 2024 12:18:06 +0800 Subject: [PATCH 112/215] add threshold for table rows --- src/Core/Settings.h | 2 + src/Core/SettingsChangesHistory.cpp | 253 +++++++++++ src/Interpreters/HashJoin/AddedColumns.cpp | 63 +++ src/Interpreters/HashJoin/AddedColumns.h | 6 + src/Interpreters/HashJoin/HashJoin.cpp | 84 +++- src/Interpreters/HashJoin/HashJoin.h | 7 +- src/Interpreters/HashJoin/HashJoinMethods.h | 424 +++++++++++++++++- src/Interpreters/IJoin.h | 1 + src/Interpreters/RowRefs.h | 1 + .../Transforms/JoiningTransform.cpp | 6 +- 10 files changed, 841 insertions(+), 6 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index d8837d26e54..c1433ca7250 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -922,6 +922,8 @@ class IColumn; M(Bool, implicit_transaction, false, "If enabled and not already inside a transaction, wraps the query inside a full transaction (begin + commit or rollback)", 0) \ M(UInt64, grace_hash_join_initial_buckets, 1, "Initial number of grace hash join buckets", 0) \ M(UInt64, grace_hash_join_max_buckets, 1024, "Limit on the number of grace hash join buckets", 0) \ + M(Int32, join_to_sort_perkey_rows_threshold, 40, "The lower limit of per-key average rows in the right table to determine whether to sort it in hash join.", 0) \ + M(Int32, join_to_sort_table_rows_threshold, 10000, "The upper limit of rows in the right table to determine whether to sort it in hash join.", 0) \ M(Timezone, session_timezone, "", "This setting can be removed in the future due to potential caveats. It is experimental and is not suitable for production usage. The default timezone for current session or query. The server default timezone if empty.", 0) \ M(Bool, use_hive_partitioning, false, "Allows to use hive partitioning for File, URL, S3, AzureBlobStorage and HDFS engines.", 0)\ \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 2415323b4a0..b975c6b2fad 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -516,6 +516,259 @@ static std::initializer_list col >= '2023-01-01' AND col <= '2023-12-31')"}, + {"extract_key_value_pairs_max_pairs_per_row", 0, 0, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory."}, + {"default_view_definer", "CURRENT_USER", "CURRENT_USER", "Allows to set default `DEFINER` option while creating a view"}, + {"default_materialized_view_sql_security", "DEFINER", "DEFINER", "Allows to set a default value for SQL SECURITY option when creating a materialized view"}, + {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"}, + {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, + {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, + }}, + {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."}, + {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"}, + {"output_format_arrow_use_signed_indexes_for_dictionary", false, true, "Use signed indexes type for Arrow dictionaries by default as it's recommended"}, + {"allow_experimental_variant_type", false, false, "Add new experimental Variant type"}, + {"use_variant_as_common_type", false, false, "Allow to use Variant in if/multiIf if there is no common type"}, + {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"}, + {"parallel_replicas_mark_segment_size", 128, 128, "Add new setting to control segment size in new parallel replicas coordinator implementation"}, + {"ignore_materialized_views_with_dropped_target_table", false, false, "Add new setting to allow to ignore materialized views with dropped target table"}, + {"output_format_compression_level", 3, 3, "Allow to change compression level in the query output"}, + {"output_format_compression_zstd_window_log", 0, 0, "Allow to change zstd window log in the query output when zstd compression is used"}, + {"enable_zstd_qat_codec", false, false, "Add new ZSTD_QAT codec"}, + {"enable_vertical_final", false, true, "Use vertical final by default"}, + {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"}, + {"max_rows_in_set_to_optimize_join", 100000, 0, "Disable join optimization as it prevents from read in order optimization"}, + {"output_format_pretty_color", true, "auto", "Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"}, + {"function_visible_width_behavior", 0, 1, "We changed the default behavior of `visibleWidth` to be more precise"}, + {"max_estimated_execution_time", 0, 0, "Separate max_execution_time and max_estimated_execution_time"}, + {"iceberg_engine_ignore_schema_evolution", false, false, "Allow to ignore schema evolution in Iceberg table engine"}, + {"optimize_injective_functions_in_group_by", false, true, "Replace injective functions by it's arguments in GROUP BY section in analyzer"}, + {"update_insert_deduplication_token_in_dependent_materialized_views", false, false, "Allow to update insert deduplication token with table identifier during insert in dependent materialized views"}, + {"azure_max_unexpected_write_error_retries", 4, 4, "The maximum number of retries in case of unexpected errors during Azure blob storage write"}, + {"split_parts_ranges_into_intersecting_and_non_intersecting_final", false, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"}, + {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}}}, + {"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."}, + {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"}, + {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"}, + {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}}, + {"23.11", {{"parsedatetime_parse_without_leading_zeros", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}}, + {"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"}, + {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"}, + {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"}, + {"input_format_json_read_arrays_as_strings", false, true, "Allow to read arrays as strings in JSON formats by default"}, + {"input_format_json_infer_incomplete_types_as_strings", false, true, "Allow to infer incomplete types as Strings in JSON formats by default"}, + {"input_format_json_try_infer_numbers_from_strings", true, false, "Don't infer numbers from strings in JSON formats by default to prevent possible parsing errors"}, + {"http_write_exception_in_output_format", false, true, "Output valid JSON/XML on exception in HTTP streaming."}}}, + {"23.8", {{"rewrite_count_distinct_if_with_count_distinct_implementation", false, true, "Rewrite countDistinctIf with count_distinct_implementation configuration"}}}, + {"23.7", {{"function_sleep_max_microseconds_per_block", 0, 3000000, "In previous versions, the maximum sleep time of 3 seconds was applied only for `sleep`, but not for `sleepEachRow` function. In the new version, we introduce this setting. If you set compatibility with the previous versions, we will disable the limit altogether."}}}, + {"23.6", {{"http_send_timeout", 180, 30, "3 minutes seems crazy long. Note that this is timeout for a single network write call, not for the whole upload operation."}, + {"http_receive_timeout", 180, 30, "See http_send_timeout."}}}, + {"23.5", {{"input_format_parquet_preserve_order", true, false, "Allow Parquet reader to reorder rows for better parallelism."}, + {"parallelize_output_from_storages", false, true, "Allow parallelism when executing queries that read from file/url/s3/etc. This may reorder rows."}, + {"use_with_fill_by_sorting_prefix", false, true, "Columns preceding WITH FILL columns in ORDER BY clause form sorting prefix. Rows with different values in sorting prefix are filled independently"}, + {"output_format_parquet_compliant_nested_types", false, true, "Change an internal field name in output Parquet file schema."}}}, + {"23.4", {{"allow_suspicious_indices", true, false, "If true, index can defined with identical expressions"}, + {"allow_nonconst_timezone_arguments", true, false, "Allow non-const timezone arguments in certain time-related functions like toTimeZone(), fromUnixTimestamp*(), snowflakeToDateTime*()."}, + {"connect_timeout_with_failover_ms", 50, 1000, "Increase default connect timeout because of async connect"}, + {"connect_timeout_with_failover_secure_ms", 100, 1000, "Increase default secure connect timeout because of async connect"}, + {"hedged_connection_timeout_ms", 100, 50, "Start new connection in hedged requests after 50 ms instead of 100 to correspond with previous connect timeout"}, + {"formatdatetime_f_prints_single_zero", true, false, "Improved compatibility with MySQL DATE_FORMAT()/STR_TO_DATE()"}, + {"formatdatetime_parsedatetime_m_is_month_name", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}}, + {"23.3", {{"output_format_parquet_version", "1.0", "2.latest", "Use latest Parquet format version for output format"}, + {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"}, + {"input_format_native_allow_types_conversion", false, true, "Allow types conversion in Native input forma"}, + {"output_format_arrow_compression_method", "none", "lz4_frame", "Use lz4 compression in Arrow output format by default"}, + {"output_format_parquet_compression_method", "snappy", "lz4", "Use lz4 compression in Parquet output format by default"}, + {"output_format_orc_compression_method", "none", "lz4_frame", "Use lz4 compression in ORC output format by default"}, + {"async_query_sending_for_remote", false, true, "Create connections and send query async across shards"}}}, + {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"}, + {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"}, + {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"}, + {"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"}, + {"insert_keeper_max_retries", 0, 20, "Enable reconnections to Keeper on INSERT, improve reliability"}}}, + {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"}, + {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"}, + {"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"}, + {"input_format_tsv_detect_header", false, true, "Detect header in TSV format by default"}, + {"input_format_custom_detect_header", false, true, "Detect header in CustomSeparated format by default"}, + {"query_plan_remove_redundant_sorting", false, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries"}}}, + {"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"}, + {"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"}, + {"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}}, + {"22.11", {{"use_structure_from_insertion_table_in_table_functions", 0, 2, "Improve using structure from insertion table in table functions"}}}, + {"22.9", {{"force_grouping_standard_compatibility", false, true, "Make GROUPING function output the same as in SQL standard and other DBMS"}}}, + {"22.7", {{"cross_to_inner_join_rewrite", 1, 2, "Force rewrite comma join to inner"}, + {"enable_positional_arguments", false, true, "Enable positional arguments feature by default"}, + {"format_csv_allow_single_quotes", true, false, "Most tools don't treat single quote in CSV specially, don't do it by default too"}}}, + {"22.6", {{"output_format_json_named_tuples_as_objects", false, true, "Allow to serialize named tuples as JSON objects in JSON formats by default"}, + {"input_format_skip_unknown_fields", false, true, "Optimize reading subset of columns for some input formats"}}}, + {"22.5", {{"memory_overcommit_ratio_denominator", 0, 1073741824, "Enable memory overcommit feature by default"}, + {"memory_overcommit_ratio_denominator_for_user", 0, 1073741824, "Enable memory overcommit feature by default"}}}, + {"22.4", {{"allow_settings_after_format_in_insert", true, false, "Do not allow SETTINGS after FORMAT for INSERT queries because ClickHouse interpret SETTINGS as some values, which is misleading"}}}, + {"22.3", {{"cast_ipv4_ipv6_default_on_conversion_error", true, false, "Make functions cast(value, 'IPv4') and cast(value, 'IPv6') behave same as toIPv4 and toIPv6 functions"}}}, + {"21.12", {{"stream_like_engine_allow_direct_select", true, false, "Do not allow direct select for Kafka/RabbitMQ/FileLog by default"}}}, + {"21.9", {{"output_format_decimal_trailing_zeros", true, false, "Do not output trailing zeros in text representation of Decimal types by default for better looking output"}, + {"use_hedged_requests", false, true, "Enable Hedged Requests feature by default"}}}, + {"21.7", {{"legacy_column_name_of_tuple_literal", true, false, "Add this setting only for compatibility reasons. It makes sense to set to 'true', while doing rolling update of cluster from version lower than 21.7 to higher"}}}, + {"21.5", {{"async_socket_for_remote", false, true, "Fix all problems and turn on asynchronous reads from socket for remote queries by default again"}}}, + {"21.3", {{"async_socket_for_remote", true, false, "Turn off asynchronous reads from socket for remote queries because of some problems"}, + {"optimize_normalize_count_variants", false, true, "Rewrite aggregate functions that semantically equals to count() as count() by default"}, + {"normalize_function_names", false, true, "Normalize function names to their canonical names, this was needed for projection query routing"}}}, + {"21.2", {{"enable_global_with_statement", false, true, "Propagate WITH statements to UNION queries and all subqueries by default"}}}, + {"21.1", {{"insert_quorum_parallel", false, true, "Use parallel quorum inserts by default. It is significantly more convenient to use than sequential quorum inserts"}, + {"input_format_null_as_default", false, true, "Allow to insert NULL as default for input formats by default"}, + {"optimize_on_insert", false, true, "Enable data optimization on INSERT by default for better user experience"}, + {"use_compact_format_in_distributed_parts_names", false, true, "Use compact format for async INSERT into Distributed tables by default"}}}, + {"20.10", {{"format_regexp_escaping_rule", "Escaped", "Raw", "Use Raw as default escaping rule for Regexp format to male the behaviour more like to what users expect"}}}, + {"20.7", {{"show_table_uuid_in_table_create_query_if_not_nil", true, false, "Stop showing UID of the table in its CREATE query for Engine=Atomic"}}}, + {"20.5", {{"input_format_with_names_use_header", false, true, "Enable using header with names for formats with WithNames/WithNamesAndTypes suffixes"}, + {"allow_suspicious_codecs", true, false, "Don't allow to specify meaningless compression codecs"}}}, + {"20.4", {{"validate_polygons", false, true, "Throw exception if polygon is invalid in function pointInPolygon by default instead of returning possibly wrong results"}}}, + {"19.18", {{"enable_scalar_subquery_optimization", false, true, "Prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once"}}}, + {"19.14", {{"any_join_distinct_right_table_keys", true, false, "Disable ANY RIGHT and ANY FULL JOINs by default to avoid inconsistency"}}}, + {"19.12", {{"input_format_defaults_for_omitted_fields", false, true, "Enable calculation of complex default expressions for omitted fields for some input formats, because it should be the expected behaviour"}}}, + {"19.5", {{"max_partitions_per_insert_block", 0, 100, "Add a limit for the number of partitions in one block"}}}, + {"18.12.17", {{"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}}}, }; diff --git a/src/Interpreters/HashJoin/AddedColumns.cpp b/src/Interpreters/HashJoin/AddedColumns.cpp index 21cb6e401ed..d70781d2fb3 100644 --- a/src/Interpreters/HashJoin/AddedColumns.cpp +++ b/src/Interpreters/HashJoin/AddedColumns.cpp @@ -20,10 +20,13 @@ void AddedColumns::buildOutput() {} template<> void AddedColumns::buildJoinGetOutput() {} +<<<<<<< HEAD template<> template void AddedColumns::buildOutputFromBlocks() {} +======= +>>>>>>> add threshold for table rows template<> void AddedColumns::buildOutput() @@ -32,9 +35,15 @@ void AddedColumns::buildOutput() buildOutputFromBlocks(); else { +<<<<<<< HEAD if (join_data_avg_perkey_rows < output_by_row_list_threshold) buildOutputFromBlocks(); else +======= + if (join_data_avg_perkey_rows < sort_right_perkey_rows_threshold) + buildOutputFromBlocks(); + else if (join_data_sorted) +>>>>>>> add threshold for table rows { for (size_t i = 0; i < this->size(); ++i) { @@ -44,14 +53,19 @@ void AddedColumns::buildOutput() if (row_ref_i) { const RowRefList * row_ref_list = reinterpret_cast(row_ref_i); +<<<<<<< HEAD for (auto it = row_ref_list->begin(); it.ok(); ++it) col->insertFrom(*it->block->getByPosition(right_indexes[i]).column, it->row_num); +======= + col->insertRangeFrom(*row_ref_list->block->getByPosition(right_indexes[i]).column, row_ref_list->row_num, row_ref_list->rows); +>>>>>>> add threshold for table rows } else type_name[i].type->insertDefaultInto(*col); } } } +<<<<<<< HEAD } } @@ -74,6 +88,25 @@ void AddedColumns::buildJoinGetOutput() nullable_col->insertFromNotNullable(*column_from_block.column, row_ref->row_num); else col->insertFrom(*column_from_block.column, row_ref->row_num); +======= + else + { + for (size_t i = 0; i < this->size(); ++i) + { + auto & col = columns[i]; + for (auto row_ref_i : lazy_output.row_refs) + { + if (row_ref_i) + { + const RowRefList * row_ref_list = reinterpret_cast(row_ref_i); + for (auto it = row_ref_list->begin(); it.ok(); ++it) + col->insertFrom(*it->block->getByPosition(right_indexes[i]).column, it->row_num); + } + else + type_name[i].type->insertDefaultInto(*col); + } + } +>>>>>>> add threshold for table rows } } } @@ -82,7 +115,11 @@ template<> template void AddedColumns::buildOutputFromBlocks() { +<<<<<<< HEAD if (this->size() == 0) +======= + if (this->size() == 0) +>>>>>>> add threshold for table rows return; std::vector blocks; std::vector row_nums; @@ -123,6 +160,32 @@ void AddedColumns::buildOutputFromBlocks() col->insertFrom(*blocks[j]->getByPosition(right_indexes[i]).column, row_nums[j]); else type_name[i].type->insertDefaultInto(*col); +<<<<<<< HEAD +======= + } + } +} + +template<> +void AddedColumns::buildJoinGetOutput() +{ + for (size_t i = 0; i < this->size(); ++i) + { + auto & col = columns[i]; + for (auto row_ref_i : lazy_output.row_refs) + { + if (!row_ref_i) + { + type_name[i].type->insertDefaultInto(*col); + continue; + } + const auto * row_ref = reinterpret_cast(row_ref_i); + const auto & column_from_block = row_ref->block->getByPosition(right_indexes[i]); + if (auto * nullable_col = typeid_cast(col.get()); nullable_col && !column_from_block.column->isNullable()) + nullable_col->insertFromNotNullable(*column_from_block.column, row_ref->row_num); + else + col->insertFrom(*column_from_block.column, row_ref->row_num); +>>>>>>> add threshold for table rows } } } diff --git a/src/Interpreters/HashJoin/AddedColumns.h b/src/Interpreters/HashJoin/AddedColumns.h index f1b95a63be6..5ae69fbbf66 100644 --- a/src/Interpreters/HashJoin/AddedColumns.h +++ b/src/Interpreters/HashJoin/AddedColumns.h @@ -196,6 +196,12 @@ private: } } + /** Build output from the blocks that extract from `RowRef` or `RowRefList`, to avoid block cache miss which may cause performance slow down. + * And This problem would happen it we directly build output from `RowRef` or `RowRefList`. + */ + template + void buildOutputFromBlocks(); + MutableColumns columns; bool is_join_get; std::vector right_indexes; diff --git a/src/Interpreters/HashJoin/HashJoin.cpp b/src/Interpreters/HashJoin/HashJoin.cpp index 9c07a71e614..6f332118f8a 100644 --- a/src/Interpreters/HashJoin/HashJoin.cpp +++ b/src/Interpreters/HashJoin/HashJoin.cpp @@ -649,7 +649,6 @@ bool HashJoin::addBlockToJoin(const Block & source_block_, bool check_limits) } data->keys_to_join = total_rows; shrinkStoredBlocksToFit(total_bytes); - return table_join->sizeLimits().check(total_rows, total_bytes, "JOIN", ErrorCodes::SET_SIZE_LIMIT_EXCEEDED); } @@ -1361,4 +1360,87 @@ bool HashJoin::needUsedFlagsForPerRightTableRow(std::shared_ptr table return false; } +template +void HashJoin::tryRerangeRightTableDataImpl(Map & map [[maybe_unused]]) +{ + constexpr JoinFeatures join_features; + if constexpr (join_features.is_all_join && (join_features.left || join_features.inner)) + { + auto merge_rows_into_one_block = [&](BlocksList & blocks, RowRefList & rows_ref) + { + auto it = rows_ref.begin(); + if (it.ok()) + { + if (blocks.empty() || blocks.back().rows() > DEFAULT_BLOCK_SIZE) + blocks.emplace_back(it->block->cloneEmpty()); + } + else + { + return; + } + auto & block = blocks.back(); + size_t start_row = block.rows(); + for (; it.ok(); ++it) + { + for (size_t i = 0; i < block.columns(); ++i) + { + auto & col = *(block.getByPosition(i).column->assumeMutable()); + col.insertFrom(*it->block->getByPosition(i).column, it->row_num); + } + } + if (block.rows() > start_row) + { + RowRefList new_rows_ref(&block, start_row, block.rows() - start_row); + rows_ref = std::move(new_rows_ref); + } + }; + + auto visit_rows_map = [&](BlocksList & blocks, MapsAll & rows_map) + { + switch (data->type) + { + #define M(TYPE) \ + case Type::TYPE: \ + {\ + rows_map.TYPE->forEachMapped([&](RowRefList & rows_ref) { merge_rows_into_one_block(blocks, rows_ref); }); \ + break; \ + } + APPLY_FOR_JOIN_VARIANTS(M) + #undef M + default: + break; + } + }; + BlocksList sorted_blocks; + visit_rows_map(sorted_blocks, map); + data->blocks.swap(sorted_blocks); + } +} + +void HashJoin::tryRerangeRightTableData() +{ + if ((kind != JoinKind::Inner && kind != JoinKind::Left) || strictness != JoinStrictness::All || table_join->getMixedJoinExpression()) + return; + + if (!data || data->sorted || data->blocks.empty() || data->maps.size() > 1) + return; + + if (data->keys_to_join == 0) + data->keys_to_join = getTotalRowCount(); + if (sample_block_with_columns_to_add.columns() == 0 || data->rows_to_join > table_join->sortRightTableRowsThreshold() || data->avgPerKeyRows() < table_join->sortRightPerkeyRowsThreshold()) + { + LOG_DEBUG(log, "The joined right table total rows :{}, total keys :{}, columns added:{}", + data->rows_to_join, data->keys_to_join, sample_block_with_columns_to_add.columns()); + return; + } + std::cout << "sort right table rows" << std::endl; + joinDispatch( + kind, + strictness, + data->maps.front(), + [&](auto kind_, auto strictness_, auto & map_) { tryRerangeRightTableDataImpl(map_); }); + std::cout << "sort right finished" << std::endl; + data->sorted = true; +} + } diff --git a/src/Interpreters/HashJoin/HashJoin.h b/src/Interpreters/HashJoin/HashJoin.h index d645b8e9273..230343691ea 100644 --- a/src/Interpreters/HashJoin/HashJoin.h +++ b/src/Interpreters/HashJoin/HashJoin.h @@ -345,11 +345,12 @@ public: size_t blocks_allocated_size = 0; size_t blocks_nullmaps_allocated_size = 0; - /// Number of rows of right table to join size_t rows_to_join = 0; /// Number of keys of right table to join size_t keys_to_join = 0; + /// Whether the right table reranged by key + bool sorted = false; size_t avgPerKeyRows() const { @@ -465,6 +466,10 @@ private: void validateAdditionalFilterExpression(std::shared_ptr additional_filter_expression); bool needUsedFlagsForPerRightTableRow(std::shared_ptr table_join_) const; + + void tryRerangeRightTableData() override; + template + void tryRerangeRightTableDataImpl(Map & map); }; } diff --git a/src/Interpreters/HashJoin/HashJoinMethods.h b/src/Interpreters/HashJoin/HashJoinMethods.h index 97ad57d26ea..9d94c3f62c2 100644 --- a/src/Interpreters/HashJoin/HashJoinMethods.h +++ b/src/Interpreters/HashJoin/HashJoinMethods.h @@ -121,7 +121,142 @@ private: std::vector && key_getter_vector, const std::vector & mapv, AddedColumns & added_columns, - JoinStuff::JoinUsedFlags & used_flags); + JoinStuff::JoinUsedFlags & used_flags) + { + constexpr JoinFeatures join_features; + + size_t rows = added_columns.rows_to_add; + if constexpr (need_filter) + added_columns.filter = IColumn::Filter(rows, 0); + if constexpr (!flag_per_row && (STRICTNESS == JoinStrictness::All || (STRICTNESS == JoinStrictness::Semi && KIND == JoinKind::Right))) + added_columns.output_by_row_list = true; + + Arena pool; + + if constexpr (join_features.need_replication) + added_columns.offsets_to_replicate = std::make_unique(rows); + + IColumn::Offset current_offset = 0; + size_t max_joined_block_rows = added_columns.max_joined_block_rows; + size_t i = 0; + for (; i < rows; ++i) + { + if constexpr (join_features.need_replication) + { + if (unlikely(current_offset >= max_joined_block_rows)) + { + added_columns.offsets_to_replicate->resize_assume_reserved(i); + added_columns.filter.resize_assume_reserved(i); + break; + } + } + + bool right_row_found = false; + + KnownRowsHolder known_rows; + for (size_t onexpr_idx = 0; onexpr_idx < added_columns.join_on_keys.size(); ++onexpr_idx) + { + const auto & join_keys = added_columns.join_on_keys[onexpr_idx]; + if (join_keys.null_map && (*join_keys.null_map)[i]) + continue; + + bool row_acceptable = !join_keys.isRowFiltered(i); + using FindResult = typename KeyGetter::FindResult; + auto find_result = row_acceptable ? key_getter_vector[onexpr_idx].findKey(*(mapv[onexpr_idx]), i, pool) : FindResult(); + + if (find_result.isFound()) + { + right_row_found = true; + auto & mapped = find_result.getMapped(); + if constexpr (join_features.is_asof_join) + { + const IColumn & left_asof_key = added_columns.leftAsofKey(); + + auto row_ref = mapped->findAsof(left_asof_key, i); + if (row_ref && row_ref->block) + { + setUsed(added_columns.filter, i); + if constexpr (flag_per_row) + used_flags.template setUsed(row_ref->block, row_ref->row_num, 0); + else + used_flags.template setUsed(find_result); + + added_columns.appendFromBlock(row_ref, join_features.add_missing); + } + else + addNotFoundRow(added_columns, current_offset); + } + else if constexpr (join_features.is_all_join) + { + setUsed(added_columns.filter, i); + used_flags.template setUsed(find_result); + auto used_flags_opt = join_features.need_flags ? &used_flags : nullptr; + addFoundRowAll(mapped, added_columns, current_offset, known_rows, used_flags_opt); + } + else if constexpr ((join_features.is_any_join || join_features.is_semi_join) && join_features.right) + { + /// Use first appeared left key + it needs left columns replication + bool used_once = used_flags.template setUsedOnce(find_result); + if (used_once) + { + auto used_flags_opt = join_features.need_flags ? &used_flags : nullptr; + setUsed(added_columns.filter, i); + addFoundRowAll( + mapped, added_columns, current_offset, known_rows, used_flags_opt); + } + } + else if constexpr (join_features.is_any_join && KIND == JoinKind::Inner) + { + bool used_once = used_flags.template setUsedOnce(find_result); + + /// Use first appeared left key only + if (used_once) + { + setUsed(added_columns.filter, i); + added_columns.appendFromBlock(&mapped, join_features.add_missing); + } + + break; + } + else if constexpr (join_features.is_any_join && join_features.full) + { + /// TODO + } + else if constexpr (join_features.is_anti_join) + { + if constexpr (join_features.right && join_features.need_flags) + used_flags.template setUsed(find_result); + } + else /// ANY LEFT, SEMI LEFT, old ANY (RightAny) + { + setUsed(added_columns.filter, i); + used_flags.template setUsed(find_result); + added_columns.appendFromBlock(&mapped, join_features.add_missing); + + if (join_features.is_any_or_semi_join) + { + break; + } + } + } + } + + if (!right_row_found) + { + if constexpr (join_features.is_anti_join && join_features.left) + setUsed(added_columns.filter, i); + addNotFoundRow(added_columns, current_offset); + } + + if constexpr (join_features.need_replication) + { + (*added_columns.offsets_to_replicate)[i] = current_offset; + } + } + + added_columns.applyLazyDefaults(); + return i; + } template static void setUsed(IColumn::Filter & filter [[maybe_unused]], size_t pos [[maybe_unused]]); @@ -131,7 +266,120 @@ private: size_t left_start_row, const std::vector & selected_rows, const std::vector & row_replicate_offset, - AddedColumns & added_columns); + AddedColumns & added_columns) + { + ColumnPtr result_column; + do + { + if (selected_rows.empty()) + { + result_column = ColumnUInt8::create(); + break; + } + const Block & sample_right_block = *((*selected_rows.begin())->block); + if (!sample_right_block || !added_columns.additional_filter_expression) + { + auto filter = ColumnUInt8::create(); + filter->insertMany(1, selected_rows.size()); + result_column = std::move(filter); + break; + } + + auto required_cols = added_columns.additional_filter_expression->getRequiredColumnsWithTypes(); + if (required_cols.empty()) + { + Block block; + added_columns.additional_filter_expression->execute(block); + result_column = block.getByPosition(0).column->cloneResized(selected_rows.size()); + break; + } + NameSet required_column_names; + for (auto & col : required_cols) + required_column_names.insert(col.name); + + Block executed_block; + size_t right_col_pos = 0; + for (const auto & col : sample_right_block.getColumnsWithTypeAndName()) + { + if (required_column_names.contains(col.name)) + { + auto new_col = col.column->cloneEmpty(); + for (const auto & selected_row : selected_rows) + { + const auto & src_col = selected_row->block->getByPosition(right_col_pos); + new_col->insertFrom(*src_col.column, selected_row->row_num); + } + executed_block.insert({std::move(new_col), col.type, col.name}); + } + right_col_pos += 1; + } + if (!executed_block) + { + result_column = ColumnUInt8::create(); + break; + } + + for (const auto & col_name : required_column_names) + { + const auto * src_col = added_columns.left_block.findByName(col_name); + if (!src_col) + continue; + auto new_col = src_col->column->cloneEmpty(); + size_t prev_left_offset = 0; + for (size_t i = 1; i < row_replicate_offset.size(); ++i) + { + const size_t & left_offset = row_replicate_offset[i]; + size_t rows = left_offset - prev_left_offset; + if (rows) + new_col->insertManyFrom(*src_col->column, left_start_row + i - 1, rows); + prev_left_offset = left_offset; + } + executed_block.insert({std::move(new_col), src_col->type, col_name}); + } + if (!executed_block) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "required columns: [{}], but not found any in left/right table. right table: {}, left table: {}", + required_cols.toString(), + sample_right_block.dumpNames(), + added_columns.left_block.dumpNames()); + } + + for (const auto & col : executed_block.getColumnsWithTypeAndName()) + if (!col.column || !col.type) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Illegal nullptr column in input block: {}", executed_block.dumpStructure()); + + added_columns.additional_filter_expression->execute(executed_block); + result_column = executed_block.getByPosition(0).column->convertToFullColumnIfConst(); + executed_block.clear(); + } while (false); + + result_column = result_column->convertToFullIfNeeded(); + if (result_column->isNullable()) + { + /// Convert Nullable(UInt8) to UInt8 ensuring that nulls are zeros + /// Trying to avoid copying data, since we are the only owner of the column. + ColumnPtr mask_column = assert_cast(*result_column).getNullMapColumnPtr(); + + MutableColumnPtr mutable_column; + { + ColumnPtr nested_column = assert_cast(*result_column).getNestedColumnPtr(); + result_column.reset(); + mutable_column = IColumn::mutate(std::move(nested_column)); + } + + auto & column_data = assert_cast(*mutable_column).getData(); + const auto & mask_column_data = assert_cast(*mask_column).getData(); + for (size_t i = 0; i < column_data.size(); ++i) + { + if (mask_column_data[i]) + column_data[i] = 0; + } + return mutable_column; + } + return result_column; + } /// First to collect all matched rows refs by join keys, then filter out rows which are not true in additional filter expression. template @@ -141,7 +389,177 @@ private: AddedColumns & added_columns, JoinStuff::JoinUsedFlags & used_flags [[maybe_unused]], bool need_filter [[maybe_unused]], - bool flag_per_row [[maybe_unused]]); + bool need_flags [[maybe_unused]], + bool add_missing [[maybe_unused]], + bool flag_per_row [[maybe_unused]]) + { + size_t left_block_rows = added_columns.rows_to_add; + if (need_filter) + added_columns.filter = IColumn::Filter(left_block_rows, 0); + + std::unique_ptr pool; + + if constexpr (need_replication) + added_columns.offsets_to_replicate = std::make_unique(left_block_rows); + + std::vector row_replicate_offset; + row_replicate_offset.reserve(left_block_rows); + + using FindResult = typename KeyGetter::FindResult; + size_t max_joined_block_rows = added_columns.max_joined_block_rows; + size_t left_row_iter = 0; + PreSelectedRows selected_rows; + selected_rows.reserve(left_block_rows); + std::vector find_results; + find_results.reserve(left_block_rows); + bool exceeded_max_block_rows = false; + IColumn::Offset total_added_rows = 0; + IColumn::Offset current_added_rows = 0; + + auto collect_keys_matched_rows_refs = [&]() + { + pool = std::make_unique(); + find_results.clear(); + row_replicate_offset.clear(); + row_replicate_offset.push_back(0); + current_added_rows = 0; + selected_rows.clear(); + for (; left_row_iter < left_block_rows; ++left_row_iter) + { + if constexpr (need_replication) + { + if (unlikely(total_added_rows + current_added_rows >= max_joined_block_rows)) + { + break; + } + } + KnownRowsHolder all_flag_known_rows; + KnownRowsHolder single_flag_know_rows; + for (size_t join_clause_idx = 0; join_clause_idx < added_columns.join_on_keys.size(); ++join_clause_idx) + { + const auto & join_keys = added_columns.join_on_keys[join_clause_idx]; + if (join_keys.null_map && (*join_keys.null_map)[left_row_iter]) + continue; + + bool row_acceptable = !join_keys.isRowFiltered(left_row_iter); + auto find_result = row_acceptable + ? key_getter_vector[join_clause_idx].findKey(*(mapv[join_clause_idx]), left_row_iter, *pool) + : FindResult(); + + if (find_result.isFound()) + { + auto & mapped = find_result.getMapped(); + find_results.push_back(find_result); + if (flag_per_row) + addFoundRowAll(mapped, selected_rows, current_added_rows, all_flag_known_rows, nullptr); + else + addFoundRowAll(mapped, selected_rows, current_added_rows, single_flag_know_rows, nullptr); + } + } + row_replicate_offset.push_back(current_added_rows); + } + }; + + auto copy_final_matched_rows = [&](size_t left_start_row, ColumnPtr filter_col) + { + const PaddedPODArray & filter_flags = assert_cast(*filter_col).getData(); + + size_t prev_replicated_row = 0; + auto selected_right_row_it = selected_rows.begin(); + size_t find_result_index = 0; + for (size_t i = 1, n = row_replicate_offset.size(); i < n; ++i) + { + bool any_matched = false; + /// For all right join, flag_per_row is true, we need mark used flags for each row. + if (flag_per_row) + { + for (size_t replicated_row = prev_replicated_row; replicated_row < row_replicate_offset[i]; ++replicated_row) + { + if (filter_flags[replicated_row]) + { + any_matched = true; + added_columns.appendFromBlock(*selected_right_row_it, add_missing); + total_added_rows += 1; + if (need_flags) + used_flags.template setUsed((*selected_right_row_it)->block, (*selected_right_row_it)->row_num, 0); + } + ++selected_right_row_it; + } + } + else + { + for (size_t replicated_row = prev_replicated_row; replicated_row < row_replicate_offset[i]; ++replicated_row) + { + if (filter_flags[replicated_row]) + { + any_matched = true; + added_columns.appendFromBlock(*selected_right_row_it, add_missing); + total_added_rows += 1; + } + ++selected_right_row_it; + } + } + if (!any_matched) + { + if (add_missing) + addNotFoundRow(added_columns, total_added_rows); + else + addNotFoundRow(added_columns, total_added_rows); + } + else + { + if (!flag_per_row && need_flags) + used_flags.template setUsed(find_results[find_result_index]); + if (need_filter) + setUsed(added_columns.filter, left_start_row + i - 1); + if (add_missing) + added_columns.applyLazyDefaults(); + } + find_result_index += (prev_replicated_row != row_replicate_offset[i]); + + if constexpr (need_replication) + { + (*added_columns.offsets_to_replicate)[left_start_row + i - 1] = total_added_rows; + } + prev_replicated_row = row_replicate_offset[i]; + } + }; + + while (left_row_iter < left_block_rows && !exceeded_max_block_rows) + { + auto left_start_row = left_row_iter; + collect_keys_matched_rows_refs(); + if (selected_rows.size() != current_added_rows || row_replicate_offset.size() != left_row_iter - left_start_row + 1) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Sizes are mismatched. selected_rows.size:{}, current_added_rows:{}, row_replicate_offset.size:{}, left_row_iter: {}, " + "left_start_row: {}", + selected_rows.size(), + current_added_rows, + row_replicate_offset.size(), + left_row_iter, + left_start_row); + } + auto filter_col = buildAdditionalFilter(left_start_row, selected_rows, row_replicate_offset, added_columns); + copy_final_matched_rows(left_start_row, filter_col); + + if constexpr (need_replication) + { + // Add a check for current_added_rows to avoid run the filter expression on too small size batch. + if (total_added_rows >= max_joined_block_rows || current_added_rows < 1024) + exceeded_max_block_rows = true; + } + } + + if constexpr (need_replication) + { + added_columns.offsets_to_replicate->resize_assume_reserved(left_row_iter); + added_columns.filter.resize_assume_reserved(left_row_iter); + } + added_columns.applyLazyDefaults(); + return left_row_iter; + } /// Cut first num_rows rows from block in place and returns block with remaining rows static Block sliceBlock(Block & block, size_t num_rows); diff --git a/src/Interpreters/IJoin.h b/src/Interpreters/IJoin.h index 7374348da50..8f648de2538 100644 --- a/src/Interpreters/IJoin.h +++ b/src/Interpreters/IJoin.h @@ -115,6 +115,7 @@ public: /// Peek next stream of delayed joined blocks. virtual IBlocksStreamPtr getDelayedBlocks() { return nullptr; } virtual bool hasDelayedBlocks() const { return false; } + virtual void tryRerangeRightTableData() {} virtual IBlocksStreamPtr getNonJoinedBlocks(const Block & left_sample_block, const Block & result_sample_block, UInt64 max_block_size) const = 0; diff --git a/src/Interpreters/RowRefs.h b/src/Interpreters/RowRefs.h index 7c98c47dd11..f8ac68191d6 100644 --- a/src/Interpreters/RowRefs.h +++ b/src/Interpreters/RowRefs.h @@ -123,6 +123,7 @@ struct RowRefList : RowRef RowRefList() {} /// NOLINT RowRefList(const Block * block_, size_t row_num_) : RowRef(block_, row_num_), rows(1) {} + RowRefList(const Block * block_, size_t row_start_, size_t rows_) : RowRef(block_, row_start_), rows(static_cast(rows_)) {} ForwardIterator begin() const { return ForwardIterator(this); } diff --git a/src/Processors/Transforms/JoiningTransform.cpp b/src/Processors/Transforms/JoiningTransform.cpp index ca204bcb482..f2fb6327129 100644 --- a/src/Processors/Transforms/JoiningTransform.cpp +++ b/src/Processors/Transforms/JoiningTransform.cpp @@ -299,13 +299,17 @@ IProcessor::Status FillingRightJoinSideTransform::prepare() void FillingRightJoinSideTransform::work() { - auto block = inputs.front().getHeader().cloneWithColumns(chunk.detachColumns()); + auto & input = inputs.front(); + auto block = input.getHeader().cloneWithColumns(chunk.detachColumns()); if (for_totals) join->setTotals(block); else stop_reading = !join->addBlockToJoin(block); + if (input.isFinished()) + join->tryRerangeRightTableData(); + set_totals = for_totals; } From 29c94195e1830e098b3973b43ff6272012735dac Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Mon, 12 Aug 2024 18:01:40 +0800 Subject: [PATCH 113/215] add setting tests/performance/all_join_opt.xml --- src/Core/SettingsChangesHistory.cpp | 255 +----------- src/Interpreters/HashJoin/HashJoin.cpp | 5 +- src/Interpreters/HashJoin/HashJoinMethods.h | 426 +------------------- tests/performance/all_join_opt.xml | 8 +- 4 files changed, 11 insertions(+), 683 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index b975c6b2fad..9ddf40e87b1 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -95,6 +95,8 @@ static std::initializer_list col >= '2023-01-01' AND col <= '2023-12-31')"}, - {"extract_key_value_pairs_max_pairs_per_row", 0, 0, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory."}, - {"default_view_definer", "CURRENT_USER", "CURRENT_USER", "Allows to set default `DEFINER` option while creating a view"}, - {"default_materialized_view_sql_security", "DEFINER", "DEFINER", "Allows to set a default value for SQL SECURITY option when creating a materialized view"}, - {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"}, - {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, - {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."}, - }}, - {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."}, - {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"}, - {"output_format_arrow_use_signed_indexes_for_dictionary", false, true, "Use signed indexes type for Arrow dictionaries by default as it's recommended"}, - {"allow_experimental_variant_type", false, false, "Add new experimental Variant type"}, - {"use_variant_as_common_type", false, false, "Allow to use Variant in if/multiIf if there is no common type"}, - {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"}, - {"parallel_replicas_mark_segment_size", 128, 128, "Add new setting to control segment size in new parallel replicas coordinator implementation"}, - {"ignore_materialized_views_with_dropped_target_table", false, false, "Add new setting to allow to ignore materialized views with dropped target table"}, - {"output_format_compression_level", 3, 3, "Allow to change compression level in the query output"}, - {"output_format_compression_zstd_window_log", 0, 0, "Allow to change zstd window log in the query output when zstd compression is used"}, - {"enable_zstd_qat_codec", false, false, "Add new ZSTD_QAT codec"}, - {"enable_vertical_final", false, true, "Use vertical final by default"}, - {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"}, - {"max_rows_in_set_to_optimize_join", 100000, 0, "Disable join optimization as it prevents from read in order optimization"}, - {"output_format_pretty_color", true, "auto", "Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"}, - {"function_visible_width_behavior", 0, 1, "We changed the default behavior of `visibleWidth` to be more precise"}, - {"max_estimated_execution_time", 0, 0, "Separate max_execution_time and max_estimated_execution_time"}, - {"iceberg_engine_ignore_schema_evolution", false, false, "Allow to ignore schema evolution in Iceberg table engine"}, - {"optimize_injective_functions_in_group_by", false, true, "Replace injective functions by it's arguments in GROUP BY section in analyzer"}, - {"update_insert_deduplication_token_in_dependent_materialized_views", false, false, "Allow to update insert deduplication token with table identifier during insert in dependent materialized views"}, - {"azure_max_unexpected_write_error_retries", 4, 4, "The maximum number of retries in case of unexpected errors during Azure blob storage write"}, - {"split_parts_ranges_into_intersecting_and_non_intersecting_final", false, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"}, - {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}}}, - {"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."}, - {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"}, - {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"}, - {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}}, - {"23.11", {{"parsedatetime_parse_without_leading_zeros", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}}, - {"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"}, - {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"}, - {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"}, - {"input_format_json_read_arrays_as_strings", false, true, "Allow to read arrays as strings in JSON formats by default"}, - {"input_format_json_infer_incomplete_types_as_strings", false, true, "Allow to infer incomplete types as Strings in JSON formats by default"}, - {"input_format_json_try_infer_numbers_from_strings", true, false, "Don't infer numbers from strings in JSON formats by default to prevent possible parsing errors"}, - {"http_write_exception_in_output_format", false, true, "Output valid JSON/XML on exception in HTTP streaming."}}}, - {"23.8", {{"rewrite_count_distinct_if_with_count_distinct_implementation", false, true, "Rewrite countDistinctIf with count_distinct_implementation configuration"}}}, - {"23.7", {{"function_sleep_max_microseconds_per_block", 0, 3000000, "In previous versions, the maximum sleep time of 3 seconds was applied only for `sleep`, but not for `sleepEachRow` function. In the new version, we introduce this setting. If you set compatibility with the previous versions, we will disable the limit altogether."}}}, - {"23.6", {{"http_send_timeout", 180, 30, "3 minutes seems crazy long. Note that this is timeout for a single network write call, not for the whole upload operation."}, - {"http_receive_timeout", 180, 30, "See http_send_timeout."}}}, - {"23.5", {{"input_format_parquet_preserve_order", true, false, "Allow Parquet reader to reorder rows for better parallelism."}, - {"parallelize_output_from_storages", false, true, "Allow parallelism when executing queries that read from file/url/s3/etc. This may reorder rows."}, - {"use_with_fill_by_sorting_prefix", false, true, "Columns preceding WITH FILL columns in ORDER BY clause form sorting prefix. Rows with different values in sorting prefix are filled independently"}, - {"output_format_parquet_compliant_nested_types", false, true, "Change an internal field name in output Parquet file schema."}}}, - {"23.4", {{"allow_suspicious_indices", true, false, "If true, index can defined with identical expressions"}, - {"allow_nonconst_timezone_arguments", true, false, "Allow non-const timezone arguments in certain time-related functions like toTimeZone(), fromUnixTimestamp*(), snowflakeToDateTime*()."}, - {"connect_timeout_with_failover_ms", 50, 1000, "Increase default connect timeout because of async connect"}, - {"connect_timeout_with_failover_secure_ms", 100, 1000, "Increase default secure connect timeout because of async connect"}, - {"hedged_connection_timeout_ms", 100, 50, "Start new connection in hedged requests after 50 ms instead of 100 to correspond with previous connect timeout"}, - {"formatdatetime_f_prints_single_zero", true, false, "Improved compatibility with MySQL DATE_FORMAT()/STR_TO_DATE()"}, - {"formatdatetime_parsedatetime_m_is_month_name", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}}, - {"23.3", {{"output_format_parquet_version", "1.0", "2.latest", "Use latest Parquet format version for output format"}, - {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"}, - {"input_format_native_allow_types_conversion", false, true, "Allow types conversion in Native input forma"}, - {"output_format_arrow_compression_method", "none", "lz4_frame", "Use lz4 compression in Arrow output format by default"}, - {"output_format_parquet_compression_method", "snappy", "lz4", "Use lz4 compression in Parquet output format by default"}, - {"output_format_orc_compression_method", "none", "lz4_frame", "Use lz4 compression in ORC output format by default"}, - {"async_query_sending_for_remote", false, true, "Create connections and send query async across shards"}}}, - {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"}, - {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"}, - {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"}, - {"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"}, - {"insert_keeper_max_retries", 0, 20, "Enable reconnections to Keeper on INSERT, improve reliability"}}}, - {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"}, - {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"}, - {"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"}, - {"input_format_tsv_detect_header", false, true, "Detect header in TSV format by default"}, - {"input_format_custom_detect_header", false, true, "Detect header in CustomSeparated format by default"}, - {"query_plan_remove_redundant_sorting", false, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries"}}}, - {"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"}, - {"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"}, - {"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}}, - {"22.11", {{"use_structure_from_insertion_table_in_table_functions", 0, 2, "Improve using structure from insertion table in table functions"}}}, - {"22.9", {{"force_grouping_standard_compatibility", false, true, "Make GROUPING function output the same as in SQL standard and other DBMS"}}}, - {"22.7", {{"cross_to_inner_join_rewrite", 1, 2, "Force rewrite comma join to inner"}, - {"enable_positional_arguments", false, true, "Enable positional arguments feature by default"}, - {"format_csv_allow_single_quotes", true, false, "Most tools don't treat single quote in CSV specially, don't do it by default too"}}}, - {"22.6", {{"output_format_json_named_tuples_as_objects", false, true, "Allow to serialize named tuples as JSON objects in JSON formats by default"}, - {"input_format_skip_unknown_fields", false, true, "Optimize reading subset of columns for some input formats"}}}, - {"22.5", {{"memory_overcommit_ratio_denominator", 0, 1073741824, "Enable memory overcommit feature by default"}, - {"memory_overcommit_ratio_denominator_for_user", 0, 1073741824, "Enable memory overcommit feature by default"}}}, - {"22.4", {{"allow_settings_after_format_in_insert", true, false, "Do not allow SETTINGS after FORMAT for INSERT queries because ClickHouse interpret SETTINGS as some values, which is misleading"}}}, - {"22.3", {{"cast_ipv4_ipv6_default_on_conversion_error", true, false, "Make functions cast(value, 'IPv4') and cast(value, 'IPv6') behave same as toIPv4 and toIPv6 functions"}}}, - {"21.12", {{"stream_like_engine_allow_direct_select", true, false, "Do not allow direct select for Kafka/RabbitMQ/FileLog by default"}}}, - {"21.9", {{"output_format_decimal_trailing_zeros", true, false, "Do not output trailing zeros in text representation of Decimal types by default for better looking output"}, - {"use_hedged_requests", false, true, "Enable Hedged Requests feature by default"}}}, - {"21.7", {{"legacy_column_name_of_tuple_literal", true, false, "Add this setting only for compatibility reasons. It makes sense to set to 'true', while doing rolling update of cluster from version lower than 21.7 to higher"}}}, - {"21.5", {{"async_socket_for_remote", false, true, "Fix all problems and turn on asynchronous reads from socket for remote queries by default again"}}}, - {"21.3", {{"async_socket_for_remote", true, false, "Turn off asynchronous reads from socket for remote queries because of some problems"}, - {"optimize_normalize_count_variants", false, true, "Rewrite aggregate functions that semantically equals to count() as count() by default"}, - {"normalize_function_names", false, true, "Normalize function names to their canonical names, this was needed for projection query routing"}}}, - {"21.2", {{"enable_global_with_statement", false, true, "Propagate WITH statements to UNION queries and all subqueries by default"}}}, - {"21.1", {{"insert_quorum_parallel", false, true, "Use parallel quorum inserts by default. It is significantly more convenient to use than sequential quorum inserts"}, - {"input_format_null_as_default", false, true, "Allow to insert NULL as default for input formats by default"}, - {"optimize_on_insert", false, true, "Enable data optimization on INSERT by default for better user experience"}, - {"use_compact_format_in_distributed_parts_names", false, true, "Use compact format for async INSERT into Distributed tables by default"}}}, - {"20.10", {{"format_regexp_escaping_rule", "Escaped", "Raw", "Use Raw as default escaping rule for Regexp format to male the behaviour more like to what users expect"}}}, - {"20.7", {{"show_table_uuid_in_table_create_query_if_not_nil", true, false, "Stop showing UID of the table in its CREATE query for Engine=Atomic"}}}, - {"20.5", {{"input_format_with_names_use_header", false, true, "Enable using header with names for formats with WithNames/WithNamesAndTypes suffixes"}, - {"allow_suspicious_codecs", true, false, "Don't allow to specify meaningless compression codecs"}}}, - {"20.4", {{"validate_polygons", false, true, "Throw exception if polygon is invalid in function pointInPolygon by default instead of returning possibly wrong results"}}}, - {"19.18", {{"enable_scalar_subquery_optimization", false, true, "Prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once"}}}, - {"19.14", {{"any_join_distinct_right_table_keys", true, false, "Disable ANY RIGHT and ANY FULL JOINs by default to avoid inconsistency"}}}, - {"19.12", {{"input_format_defaults_for_omitted_fields", false, true, "Enable calculation of complex default expressions for omitted fields for some input formats, because it should be the expected behaviour"}}}, - {"19.5", {{"max_partitions_per_insert_block", 0, 100, "Add a limit for the number of partitions in one block"}}}, - {"18.12.17", {{"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}}}, }; diff --git a/src/Interpreters/HashJoin/HashJoin.cpp b/src/Interpreters/HashJoin/HashJoin.cpp index 6f332118f8a..e394b9913b5 100644 --- a/src/Interpreters/HashJoin/HashJoin.cpp +++ b/src/Interpreters/HashJoin/HashJoin.cpp @@ -1363,7 +1363,7 @@ bool HashJoin::needUsedFlagsForPerRightTableRow(std::shared_ptr table template void HashJoin::tryRerangeRightTableDataImpl(Map & map [[maybe_unused]]) { - constexpr JoinFeatures join_features; + constexpr JoinFeatures join_features; if constexpr (join_features.is_all_join && (join_features.left || join_features.inner)) { auto merge_rows_into_one_block = [&](BlocksList & blocks, RowRefList & rows_ref) @@ -1433,13 +1433,12 @@ void HashJoin::tryRerangeRightTableData() data->rows_to_join, data->keys_to_join, sample_block_with_columns_to_add.columns()); return; } - std::cout << "sort right table rows" << std::endl; joinDispatch( kind, strictness, data->maps.front(), + false, [&](auto kind_, auto strictness_, auto & map_) { tryRerangeRightTableDataImpl(map_); }); - std::cout << "sort right finished" << std::endl; data->sorted = true; } diff --git a/src/Interpreters/HashJoin/HashJoinMethods.h b/src/Interpreters/HashJoin/HashJoinMethods.h index 9d94c3f62c2..c5b54a62f36 100644 --- a/src/Interpreters/HashJoin/HashJoinMethods.h +++ b/src/Interpreters/HashJoin/HashJoinMethods.h @@ -83,7 +83,6 @@ public: const Block & block_with_columns_to_add, const MapsTemplateVector & maps_, bool is_join_get = false); - private: template static KeyGetter createKeyGetter(const ColumnRawPtrs & key_columns, const Sizes & key_sizes); @@ -121,142 +120,7 @@ private: std::vector && key_getter_vector, const std::vector & mapv, AddedColumns & added_columns, - JoinStuff::JoinUsedFlags & used_flags) - { - constexpr JoinFeatures join_features; - - size_t rows = added_columns.rows_to_add; - if constexpr (need_filter) - added_columns.filter = IColumn::Filter(rows, 0); - if constexpr (!flag_per_row && (STRICTNESS == JoinStrictness::All || (STRICTNESS == JoinStrictness::Semi && KIND == JoinKind::Right))) - added_columns.output_by_row_list = true; - - Arena pool; - - if constexpr (join_features.need_replication) - added_columns.offsets_to_replicate = std::make_unique(rows); - - IColumn::Offset current_offset = 0; - size_t max_joined_block_rows = added_columns.max_joined_block_rows; - size_t i = 0; - for (; i < rows; ++i) - { - if constexpr (join_features.need_replication) - { - if (unlikely(current_offset >= max_joined_block_rows)) - { - added_columns.offsets_to_replicate->resize_assume_reserved(i); - added_columns.filter.resize_assume_reserved(i); - break; - } - } - - bool right_row_found = false; - - KnownRowsHolder known_rows; - for (size_t onexpr_idx = 0; onexpr_idx < added_columns.join_on_keys.size(); ++onexpr_idx) - { - const auto & join_keys = added_columns.join_on_keys[onexpr_idx]; - if (join_keys.null_map && (*join_keys.null_map)[i]) - continue; - - bool row_acceptable = !join_keys.isRowFiltered(i); - using FindResult = typename KeyGetter::FindResult; - auto find_result = row_acceptable ? key_getter_vector[onexpr_idx].findKey(*(mapv[onexpr_idx]), i, pool) : FindResult(); - - if (find_result.isFound()) - { - right_row_found = true; - auto & mapped = find_result.getMapped(); - if constexpr (join_features.is_asof_join) - { - const IColumn & left_asof_key = added_columns.leftAsofKey(); - - auto row_ref = mapped->findAsof(left_asof_key, i); - if (row_ref && row_ref->block) - { - setUsed(added_columns.filter, i); - if constexpr (flag_per_row) - used_flags.template setUsed(row_ref->block, row_ref->row_num, 0); - else - used_flags.template setUsed(find_result); - - added_columns.appendFromBlock(row_ref, join_features.add_missing); - } - else - addNotFoundRow(added_columns, current_offset); - } - else if constexpr (join_features.is_all_join) - { - setUsed(added_columns.filter, i); - used_flags.template setUsed(find_result); - auto used_flags_opt = join_features.need_flags ? &used_flags : nullptr; - addFoundRowAll(mapped, added_columns, current_offset, known_rows, used_flags_opt); - } - else if constexpr ((join_features.is_any_join || join_features.is_semi_join) && join_features.right) - { - /// Use first appeared left key + it needs left columns replication - bool used_once = used_flags.template setUsedOnce(find_result); - if (used_once) - { - auto used_flags_opt = join_features.need_flags ? &used_flags : nullptr; - setUsed(added_columns.filter, i); - addFoundRowAll( - mapped, added_columns, current_offset, known_rows, used_flags_opt); - } - } - else if constexpr (join_features.is_any_join && KIND == JoinKind::Inner) - { - bool used_once = used_flags.template setUsedOnce(find_result); - - /// Use first appeared left key only - if (used_once) - { - setUsed(added_columns.filter, i); - added_columns.appendFromBlock(&mapped, join_features.add_missing); - } - - break; - } - else if constexpr (join_features.is_any_join && join_features.full) - { - /// TODO - } - else if constexpr (join_features.is_anti_join) - { - if constexpr (join_features.right && join_features.need_flags) - used_flags.template setUsed(find_result); - } - else /// ANY LEFT, SEMI LEFT, old ANY (RightAny) - { - setUsed(added_columns.filter, i); - used_flags.template setUsed(find_result); - added_columns.appendFromBlock(&mapped, join_features.add_missing); - - if (join_features.is_any_or_semi_join) - { - break; - } - } - } - } - - if (!right_row_found) - { - if constexpr (join_features.is_anti_join && join_features.left) - setUsed(added_columns.filter, i); - addNotFoundRow(added_columns, current_offset); - } - - if constexpr (join_features.need_replication) - { - (*added_columns.offsets_to_replicate)[i] = current_offset; - } - } - - added_columns.applyLazyDefaults(); - return i; - } + JoinStuff::JoinUsedFlags & used_flags); template static void setUsed(IColumn::Filter & filter [[maybe_unused]], size_t pos [[maybe_unused]]); @@ -266,120 +130,7 @@ private: size_t left_start_row, const std::vector & selected_rows, const std::vector & row_replicate_offset, - AddedColumns & added_columns) - { - ColumnPtr result_column; - do - { - if (selected_rows.empty()) - { - result_column = ColumnUInt8::create(); - break; - } - const Block & sample_right_block = *((*selected_rows.begin())->block); - if (!sample_right_block || !added_columns.additional_filter_expression) - { - auto filter = ColumnUInt8::create(); - filter->insertMany(1, selected_rows.size()); - result_column = std::move(filter); - break; - } - - auto required_cols = added_columns.additional_filter_expression->getRequiredColumnsWithTypes(); - if (required_cols.empty()) - { - Block block; - added_columns.additional_filter_expression->execute(block); - result_column = block.getByPosition(0).column->cloneResized(selected_rows.size()); - break; - } - NameSet required_column_names; - for (auto & col : required_cols) - required_column_names.insert(col.name); - - Block executed_block; - size_t right_col_pos = 0; - for (const auto & col : sample_right_block.getColumnsWithTypeAndName()) - { - if (required_column_names.contains(col.name)) - { - auto new_col = col.column->cloneEmpty(); - for (const auto & selected_row : selected_rows) - { - const auto & src_col = selected_row->block->getByPosition(right_col_pos); - new_col->insertFrom(*src_col.column, selected_row->row_num); - } - executed_block.insert({std::move(new_col), col.type, col.name}); - } - right_col_pos += 1; - } - if (!executed_block) - { - result_column = ColumnUInt8::create(); - break; - } - - for (const auto & col_name : required_column_names) - { - const auto * src_col = added_columns.left_block.findByName(col_name); - if (!src_col) - continue; - auto new_col = src_col->column->cloneEmpty(); - size_t prev_left_offset = 0; - for (size_t i = 1; i < row_replicate_offset.size(); ++i) - { - const size_t & left_offset = row_replicate_offset[i]; - size_t rows = left_offset - prev_left_offset; - if (rows) - new_col->insertManyFrom(*src_col->column, left_start_row + i - 1, rows); - prev_left_offset = left_offset; - } - executed_block.insert({std::move(new_col), src_col->type, col_name}); - } - if (!executed_block) - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "required columns: [{}], but not found any in left/right table. right table: {}, left table: {}", - required_cols.toString(), - sample_right_block.dumpNames(), - added_columns.left_block.dumpNames()); - } - - for (const auto & col : executed_block.getColumnsWithTypeAndName()) - if (!col.column || !col.type) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Illegal nullptr column in input block: {}", executed_block.dumpStructure()); - - added_columns.additional_filter_expression->execute(executed_block); - result_column = executed_block.getByPosition(0).column->convertToFullColumnIfConst(); - executed_block.clear(); - } while (false); - - result_column = result_column->convertToFullIfNeeded(); - if (result_column->isNullable()) - { - /// Convert Nullable(UInt8) to UInt8 ensuring that nulls are zeros - /// Trying to avoid copying data, since we are the only owner of the column. - ColumnPtr mask_column = assert_cast(*result_column).getNullMapColumnPtr(); - - MutableColumnPtr mutable_column; - { - ColumnPtr nested_column = assert_cast(*result_column).getNestedColumnPtr(); - result_column.reset(); - mutable_column = IColumn::mutate(std::move(nested_column)); - } - - auto & column_data = assert_cast(*mutable_column).getData(); - const auto & mask_column_data = assert_cast(*mask_column).getData(); - for (size_t i = 0; i < column_data.size(); ++i) - { - if (mask_column_data[i]) - column_data[i] = 0; - } - return mutable_column; - } - return result_column; - } + AddedColumns & added_columns); /// First to collect all matched rows refs by join keys, then filter out rows which are not true in additional filter expression. template @@ -389,177 +140,7 @@ private: AddedColumns & added_columns, JoinStuff::JoinUsedFlags & used_flags [[maybe_unused]], bool need_filter [[maybe_unused]], - bool need_flags [[maybe_unused]], - bool add_missing [[maybe_unused]], - bool flag_per_row [[maybe_unused]]) - { - size_t left_block_rows = added_columns.rows_to_add; - if (need_filter) - added_columns.filter = IColumn::Filter(left_block_rows, 0); - - std::unique_ptr pool; - - if constexpr (need_replication) - added_columns.offsets_to_replicate = std::make_unique(left_block_rows); - - std::vector row_replicate_offset; - row_replicate_offset.reserve(left_block_rows); - - using FindResult = typename KeyGetter::FindResult; - size_t max_joined_block_rows = added_columns.max_joined_block_rows; - size_t left_row_iter = 0; - PreSelectedRows selected_rows; - selected_rows.reserve(left_block_rows); - std::vector find_results; - find_results.reserve(left_block_rows); - bool exceeded_max_block_rows = false; - IColumn::Offset total_added_rows = 0; - IColumn::Offset current_added_rows = 0; - - auto collect_keys_matched_rows_refs = [&]() - { - pool = std::make_unique(); - find_results.clear(); - row_replicate_offset.clear(); - row_replicate_offset.push_back(0); - current_added_rows = 0; - selected_rows.clear(); - for (; left_row_iter < left_block_rows; ++left_row_iter) - { - if constexpr (need_replication) - { - if (unlikely(total_added_rows + current_added_rows >= max_joined_block_rows)) - { - break; - } - } - KnownRowsHolder all_flag_known_rows; - KnownRowsHolder single_flag_know_rows; - for (size_t join_clause_idx = 0; join_clause_idx < added_columns.join_on_keys.size(); ++join_clause_idx) - { - const auto & join_keys = added_columns.join_on_keys[join_clause_idx]; - if (join_keys.null_map && (*join_keys.null_map)[left_row_iter]) - continue; - - bool row_acceptable = !join_keys.isRowFiltered(left_row_iter); - auto find_result = row_acceptable - ? key_getter_vector[join_clause_idx].findKey(*(mapv[join_clause_idx]), left_row_iter, *pool) - : FindResult(); - - if (find_result.isFound()) - { - auto & mapped = find_result.getMapped(); - find_results.push_back(find_result); - if (flag_per_row) - addFoundRowAll(mapped, selected_rows, current_added_rows, all_flag_known_rows, nullptr); - else - addFoundRowAll(mapped, selected_rows, current_added_rows, single_flag_know_rows, nullptr); - } - } - row_replicate_offset.push_back(current_added_rows); - } - }; - - auto copy_final_matched_rows = [&](size_t left_start_row, ColumnPtr filter_col) - { - const PaddedPODArray & filter_flags = assert_cast(*filter_col).getData(); - - size_t prev_replicated_row = 0; - auto selected_right_row_it = selected_rows.begin(); - size_t find_result_index = 0; - for (size_t i = 1, n = row_replicate_offset.size(); i < n; ++i) - { - bool any_matched = false; - /// For all right join, flag_per_row is true, we need mark used flags for each row. - if (flag_per_row) - { - for (size_t replicated_row = prev_replicated_row; replicated_row < row_replicate_offset[i]; ++replicated_row) - { - if (filter_flags[replicated_row]) - { - any_matched = true; - added_columns.appendFromBlock(*selected_right_row_it, add_missing); - total_added_rows += 1; - if (need_flags) - used_flags.template setUsed((*selected_right_row_it)->block, (*selected_right_row_it)->row_num, 0); - } - ++selected_right_row_it; - } - } - else - { - for (size_t replicated_row = prev_replicated_row; replicated_row < row_replicate_offset[i]; ++replicated_row) - { - if (filter_flags[replicated_row]) - { - any_matched = true; - added_columns.appendFromBlock(*selected_right_row_it, add_missing); - total_added_rows += 1; - } - ++selected_right_row_it; - } - } - if (!any_matched) - { - if (add_missing) - addNotFoundRow(added_columns, total_added_rows); - else - addNotFoundRow(added_columns, total_added_rows); - } - else - { - if (!flag_per_row && need_flags) - used_flags.template setUsed(find_results[find_result_index]); - if (need_filter) - setUsed(added_columns.filter, left_start_row + i - 1); - if (add_missing) - added_columns.applyLazyDefaults(); - } - find_result_index += (prev_replicated_row != row_replicate_offset[i]); - - if constexpr (need_replication) - { - (*added_columns.offsets_to_replicate)[left_start_row + i - 1] = total_added_rows; - } - prev_replicated_row = row_replicate_offset[i]; - } - }; - - while (left_row_iter < left_block_rows && !exceeded_max_block_rows) - { - auto left_start_row = left_row_iter; - collect_keys_matched_rows_refs(); - if (selected_rows.size() != current_added_rows || row_replicate_offset.size() != left_row_iter - left_start_row + 1) - { - throw Exception( - ErrorCodes::LOGICAL_ERROR, - "Sizes are mismatched. selected_rows.size:{}, current_added_rows:{}, row_replicate_offset.size:{}, left_row_iter: {}, " - "left_start_row: {}", - selected_rows.size(), - current_added_rows, - row_replicate_offset.size(), - left_row_iter, - left_start_row); - } - auto filter_col = buildAdditionalFilter(left_start_row, selected_rows, row_replicate_offset, added_columns); - copy_final_matched_rows(left_start_row, filter_col); - - if constexpr (need_replication) - { - // Add a check for current_added_rows to avoid run the filter expression on too small size batch. - if (total_added_rows >= max_joined_block_rows || current_added_rows < 1024) - exceeded_max_block_rows = true; - } - } - - if constexpr (need_replication) - { - added_columns.offsets_to_replicate->resize_assume_reserved(left_row_iter); - added_columns.filter.resize_assume_reserved(left_row_iter); - } - added_columns.applyLazyDefaults(); - return left_row_iter; - } + bool flag_per_row [[maybe_unused]]); /// Cut first num_rows rows from block in place and returns block with remaining rows static Block sliceBlock(Block & block, size_t num_rows); @@ -617,4 +198,3 @@ extern template class HashJoinMethods; extern template class HashJoinMethods; } - diff --git a/tests/performance/all_join_opt.xml b/tests/performance/all_join_opt.xml index 0ab9c39f67c..2ecd76ee976 100644 --- a/tests/performance/all_join_opt.xml +++ b/tests/performance/all_join_opt.xml @@ -5,10 +5,10 @@ INSERT INTO test SELECT number % 10000, number % 10000, number % 10000 FROM numbers(10000000) INSERT INTO test1 SELECT number % 1000 , number % 1000, number % 1000 FROM numbers(100000) - SELECT MAX(test1.a) FROM test INNER JOIN test1 on test.b = test1.b - SELECT MAX(test1.a) FROM test LEFT JOIN test1 on test.b = test1.b - SELECT MAX(test1.a) FROM test RIGHT JOIN test1 on test.b = test1.b - SELECT MAX(test1.a) FROM test FULL JOIN test1 on test.b = test1.b + SELECT MAX(test1.a) FROM test INNER JOIN test1 on test.b = test1.b SETTINGS join_to_sort_table_rows_threshold=100000 + SELECT MAX(test1.a) FROM test LEFT JOIN test1 on test.b = test1.b SETTINGS join_to_sort_table_rows_threshold=100000 + SELECT MAX(test1.a) FROM test RIGHT JOIN test1 on test.b = test1.b SETTINGS join_to_sort_table_rows_threshold=100000 + SELECT MAX(test1.a) FROM test FULL JOIN test1 on test.b = test1.b SETTINGS join_to_sort_table_rows_threshold=100000 DROP TABLE IF EXISTS test DROP TABLE IF EXISTS test1 From b8e967ff9c5b03dcf9376ad316f0a73af133cb90 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Tue, 13 Aug 2024 09:41:19 +0800 Subject: [PATCH 114/215] add allowReadCaseInsentitive func --- src/Processors/Formats/IRowInputFormat.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Processors/Formats/IRowInputFormat.h b/src/Processors/Formats/IRowInputFormat.h index f8796df8604..07014bec452 100644 --- a/src/Processors/Formats/IRowInputFormat.h +++ b/src/Processors/Formats/IRowInputFormat.h @@ -68,6 +68,8 @@ protected: virtual bool allowSyncAfterError() const { return false; } virtual void syncAfterError(); + virtual bool allReadColumnCaseInsensitive() const { return false; } + /// In case of parse error, try to roll back and parse last one or two rows very carefully /// and collect as much as possible diagnostic information about error. /// If not implemented, returns empty string. From cfa4ca6fb122580b98e4f4630dc14fb047ba6ccb Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Tue, 13 Aug 2024 09:46:53 +0800 Subject: [PATCH 115/215] remove useless code --- src/Processors/Formats/IRowInputFormat.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Processors/Formats/IRowInputFormat.h b/src/Processors/Formats/IRowInputFormat.h index 07014bec452..f8796df8604 100644 --- a/src/Processors/Formats/IRowInputFormat.h +++ b/src/Processors/Formats/IRowInputFormat.h @@ -68,8 +68,6 @@ protected: virtual bool allowSyncAfterError() const { return false; } virtual void syncAfterError(); - virtual bool allReadColumnCaseInsensitive() const { return false; } - /// In case of parse error, try to roll back and parse last one or two rows very carefully /// and collect as much as possible diagnostic information about error. /// If not implemented, returns empty string. From add486b62a45a615b1d1c2ee08a945d08b984943 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Tue, 20 Aug 2024 17:33:08 +0800 Subject: [PATCH 116/215] rebase and reslove conflict --- src/Core/SettingsChangesHistory.cpp | 2 +- src/Interpreters/HashJoin/AddedColumns.cpp | 82 +++++----------------- src/Interpreters/HashJoin/AddedColumns.h | 8 +-- src/Interpreters/HashJoin/HashJoin.cpp | 4 +- src/Interpreters/TableJoin.cpp | 2 + src/Interpreters/TableJoin.h | 4 ++ tests/performance/all_join_opt.xml | 8 +-- 7 files changed, 33 insertions(+), 77 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 9ddf40e87b1..392f0dbc2ee 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -94,7 +94,7 @@ static std::initializer_list::buildOutput() {} template<> void AddedColumns::buildJoinGetOutput() {} -<<<<<<< HEAD template<> template void AddedColumns::buildOutputFromBlocks() {} -======= ->>>>>>> add threshold for table rows template<> void AddedColumns::buildOutput() @@ -35,15 +32,9 @@ void AddedColumns::buildOutput() buildOutputFromBlocks(); else { -<<<<<<< HEAD if (join_data_avg_perkey_rows < output_by_row_list_threshold) buildOutputFromBlocks(); - else -======= - if (join_data_avg_perkey_rows < sort_right_perkey_rows_threshold) - buildOutputFromBlocks(); else if (join_data_sorted) ->>>>>>> add threshold for table rows { for (size_t i = 0; i < this->size(); ++i) { @@ -53,19 +44,31 @@ void AddedColumns::buildOutput() if (row_ref_i) { const RowRefList * row_ref_list = reinterpret_cast(row_ref_i); -<<<<<<< HEAD - for (auto it = row_ref_list->begin(); it.ok(); ++it) - col->insertFrom(*it->block->getByPosition(right_indexes[i]).column, it->row_num); -======= col->insertRangeFrom(*row_ref_list->block->getByPosition(right_indexes[i]).column, row_ref_list->row_num, row_ref_list->rows); ->>>>>>> add threshold for table rows } else type_name[i].type->insertDefaultInto(*col); } } } -<<<<<<< HEAD + else + { + for (size_t i = 0; i < this->size(); ++i) + { + auto & col = columns[i]; + for (auto row_ref_i : lazy_output.row_refs) + { + if (row_ref_i) + { + const RowRefList * row_ref_list = reinterpret_cast(row_ref_i); + for (auto it = row_ref_list->begin(); it.ok(); ++it) + col->insertFrom(*it->block->getByPosition(right_indexes[i]).column, it->row_num); + } + else + type_name[i].type->insertDefaultInto(*col); + } + } + } } } @@ -88,25 +91,6 @@ void AddedColumns::buildJoinGetOutput() nullable_col->insertFromNotNullable(*column_from_block.column, row_ref->row_num); else col->insertFrom(*column_from_block.column, row_ref->row_num); -======= - else - { - for (size_t i = 0; i < this->size(); ++i) - { - auto & col = columns[i]; - for (auto row_ref_i : lazy_output.row_refs) - { - if (row_ref_i) - { - const RowRefList * row_ref_list = reinterpret_cast(row_ref_i); - for (auto it = row_ref_list->begin(); it.ok(); ++it) - col->insertFrom(*it->block->getByPosition(right_indexes[i]).column, it->row_num); - } - else - type_name[i].type->insertDefaultInto(*col); - } - } ->>>>>>> add threshold for table rows } } } @@ -115,11 +99,7 @@ template<> template void AddedColumns::buildOutputFromBlocks() { -<<<<<<< HEAD if (this->size() == 0) -======= - if (this->size() == 0) ->>>>>>> add threshold for table rows return; std::vector blocks; std::vector row_nums; @@ -160,32 +140,6 @@ void AddedColumns::buildOutputFromBlocks() col->insertFrom(*blocks[j]->getByPosition(right_indexes[i]).column, row_nums[j]); else type_name[i].type->insertDefaultInto(*col); -<<<<<<< HEAD -======= - } - } -} - -template<> -void AddedColumns::buildJoinGetOutput() -{ - for (size_t i = 0; i < this->size(); ++i) - { - auto & col = columns[i]; - for (auto row_ref_i : lazy_output.row_refs) - { - if (!row_ref_i) - { - type_name[i].type->insertDefaultInto(*col); - continue; - } - const auto * row_ref = reinterpret_cast(row_ref_i); - const auto & column_from_block = row_ref->block->getByPosition(right_indexes[i]); - if (auto * nullable_col = typeid_cast(col.get()); nullable_col && !column_from_block.column->isNullable()) - nullable_col->insertFromNotNullable(*column_from_block.column, row_ref->row_num); - else - col->insertFrom(*column_from_block.column, row_ref->row_num); ->>>>>>> add threshold for table rows } } } diff --git a/src/Interpreters/HashJoin/AddedColumns.h b/src/Interpreters/HashJoin/AddedColumns.h index 5ae69fbbf66..3f90b215602 100644 --- a/src/Interpreters/HashJoin/AddedColumns.h +++ b/src/Interpreters/HashJoin/AddedColumns.h @@ -115,6 +115,7 @@ public: } join_data_avg_perkey_rows = join.getJoinedData()->avgPerKeyRows(); output_by_row_list_threshold = join.getTableJoin().outputByRowListPerkeyRowsThreshold(); + join_data_sorted = join.getJoinedData()->sorted; } size_t size() const { return columns.size(); } @@ -147,6 +148,7 @@ public: std::unique_ptr offsets_to_replicate; bool need_filter = false; bool output_by_row_list = false; + bool join_data_sorted = false; size_t join_data_avg_perkey_rows = 0; size_t output_by_row_list_threshold = 0; IColumn::Filter filter; @@ -196,12 +198,6 @@ private: } } - /** Build output from the blocks that extract from `RowRef` or `RowRefList`, to avoid block cache miss which may cause performance slow down. - * And This problem would happen it we directly build output from `RowRef` or `RowRefList`. - */ - template - void buildOutputFromBlocks(); - MutableColumns columns; bool is_join_get; std::vector right_indexes; diff --git a/src/Interpreters/HashJoin/HashJoin.cpp b/src/Interpreters/HashJoin/HashJoin.cpp index e394b9913b5..59888d7a71d 100644 --- a/src/Interpreters/HashJoin/HashJoin.cpp +++ b/src/Interpreters/HashJoin/HashJoin.cpp @@ -1422,12 +1422,12 @@ void HashJoin::tryRerangeRightTableData() if ((kind != JoinKind::Inner && kind != JoinKind::Left) || strictness != JoinStrictness::All || table_join->getMixedJoinExpression()) return; - if (!data || data->sorted || data->blocks.empty() || data->maps.size() > 1) + if (!data || data->sorted || data->blocks.empty() || data->maps.size() > 1 || data->rows_to_join > table_join->sortRightTableRowsThreshold() || data->avgPerKeyRows() < table_join->sortRightPerkeyRowsThreshold()) return; if (data->keys_to_join == 0) data->keys_to_join = getTotalRowCount(); - if (sample_block_with_columns_to_add.columns() == 0 || data->rows_to_join > table_join->sortRightTableRowsThreshold() || data->avgPerKeyRows() < table_join->sortRightPerkeyRowsThreshold()) + if (sample_block_with_columns_to_add.columns() == 0) { LOG_DEBUG(log, "The joined right table total rows :{}, total keys :{}, columns added:{}", data->rows_to_join, data->keys_to_join, sample_block_with_columns_to_add.columns()); diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp index 138085f0710..8bcaef77939 100644 --- a/src/Interpreters/TableJoin.cpp +++ b/src/Interpreters/TableJoin.cpp @@ -116,6 +116,8 @@ TableJoin::TableJoin(const Settings & settings, VolumePtr tmp_volume_, Temporary , max_files_to_merge(settings.join_on_disk_max_files_to_merge) , temporary_files_codec(settings.temporary_files_codec) , output_by_rowlist_perkey_rows_threshold(settings.join_output_by_rowlist_perkey_rows_threshold) + , sort_right_perkey_rows_threshold(settings.join_to_sort_perkey_rows_threshold) + , sort_right_table_rows_threshold(settings.join_to_sort_table_rows_threshold) , max_memory_usage(settings.max_memory_usage) , tmp_volume(tmp_volume_) , tmp_data(tmp_data_) diff --git a/src/Interpreters/TableJoin.h b/src/Interpreters/TableJoin.h index 4d626084d81..09d7f0f2b2a 100644 --- a/src/Interpreters/TableJoin.h +++ b/src/Interpreters/TableJoin.h @@ -149,6 +149,8 @@ private: const size_t max_files_to_merge = 0; const String temporary_files_codec = "LZ4"; const size_t output_by_rowlist_perkey_rows_threshold = 0; + const size_t sort_right_perkey_rows_threshold = 0; + const size_t sort_right_table_rows_threshold = 0; /// Value if setting max_memory_usage for query, can be used when max_bytes_in_join is not specified. size_t max_memory_usage = 0; @@ -297,6 +299,8 @@ public: } size_t outputByRowListPerkeyRowsThreshold() const { return output_by_rowlist_perkey_rows_threshold; } + size_t sortRightPerkeyRowsThreshold() const { return sort_right_perkey_rows_threshold; } + size_t sortRightTableRowsThreshold() const { return sort_right_table_rows_threshold; } size_t defaultMaxBytes() const { return default_max_bytes; } size_t maxJoinedBlockRows() const { return max_joined_block_rows; } size_t maxRowsInRightBlock() const { return partial_merge_join_rows_in_right_blocks; } diff --git a/tests/performance/all_join_opt.xml b/tests/performance/all_join_opt.xml index 2ecd76ee976..0ab9c39f67c 100644 --- a/tests/performance/all_join_opt.xml +++ b/tests/performance/all_join_opt.xml @@ -5,10 +5,10 @@ INSERT INTO test SELECT number % 10000, number % 10000, number % 10000 FROM numbers(10000000) INSERT INTO test1 SELECT number % 1000 , number % 1000, number % 1000 FROM numbers(100000) - SELECT MAX(test1.a) FROM test INNER JOIN test1 on test.b = test1.b SETTINGS join_to_sort_table_rows_threshold=100000 - SELECT MAX(test1.a) FROM test LEFT JOIN test1 on test.b = test1.b SETTINGS join_to_sort_table_rows_threshold=100000 - SELECT MAX(test1.a) FROM test RIGHT JOIN test1 on test.b = test1.b SETTINGS join_to_sort_table_rows_threshold=100000 - SELECT MAX(test1.a) FROM test FULL JOIN test1 on test.b = test1.b SETTINGS join_to_sort_table_rows_threshold=100000 + SELECT MAX(test1.a) FROM test INNER JOIN test1 on test.b = test1.b + SELECT MAX(test1.a) FROM test LEFT JOIN test1 on test.b = test1.b + SELECT MAX(test1.a) FROM test RIGHT JOIN test1 on test.b = test1.b + SELECT MAX(test1.a) FROM test FULL JOIN test1 on test.b = test1.b DROP TABLE IF EXISTS test DROP TABLE IF EXISTS test1 From d12aac7d12de9632e4a97ae7240a0e0dabe5ef59 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 23 Aug 2024 20:11:33 +0000 Subject: [PATCH 117/215] Test 03228_pr_subquery_view_order_by PR + subquery with view and order by --- .../03228_pr_subquery_view_order_by.reference | 20 +++++++++++++++++++ .../03228_pr_subquery_view_order_by.sql | 18 +++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 tests/queries/0_stateless/03228_pr_subquery_view_order_by.reference create mode 100644 tests/queries/0_stateless/03228_pr_subquery_view_order_by.sql diff --git a/tests/queries/0_stateless/03228_pr_subquery_view_order_by.reference b/tests/queries/0_stateless/03228_pr_subquery_view_order_by.reference new file mode 100644 index 00000000000..aad720e27cf --- /dev/null +++ b/tests/queries/0_stateless/03228_pr_subquery_view_order_by.reference @@ -0,0 +1,20 @@ +300 +299 +298 +297 +296 +295 +294 +293 +292 +291 +290 +289 +288 +287 +286 +285 +284 +283 +282 +281 diff --git a/tests/queries/0_stateless/03228_pr_subquery_view_order_by.sql b/tests/queries/0_stateless/03228_pr_subquery_view_order_by.sql new file mode 100644 index 00000000000..b85392e0521 --- /dev/null +++ b/tests/queries/0_stateless/03228_pr_subquery_view_order_by.sql @@ -0,0 +1,18 @@ +DROP TABLE IF EXISTS view1; +DROP TABLE IF EXISTS table1; +CREATE TABLE table1 (number UInt64) ENGINE=MergeTree ORDER BY number SETTINGS index_granularity=1; +INSERT INTO table1 SELECT number FROM numbers(1, 300); +CREATE VIEW view1 AS SELECT number FROM table1; + +SELECT * +FROM +( + SELECT * + FROM view1 +) +ORDER BY number DESC +LIMIT 20 +SETTINGS cluster_for_parallel_replicas = 'parallel_replicas', allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 3, parallel_replicas_for_non_replicated_merge_tree = 1, parallel_replicas_local_plan = 1, query_plan_lift_up_union = 0; + +DROP TABLE view1; +DROP TABLE table1; From 3c29f27dd2a1cc8d4649860a9b5021f8784e467b Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 30 Aug 2024 20:04:43 +0000 Subject: [PATCH 118/215] Fix 03228_pr_subquery_view_order_by --- .../QueryPlan/Optimizations/liftUpUnion.cpp | 1 + .../Optimizations/optimizeReadInOrder.cpp | 23 +++++++++++++++++++ .../03228_pr_subquery_view_order_by.sql | 2 +- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp b/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp index c48551732c9..43cf166002e 100644 --- a/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp +++ b/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp @@ -50,6 +50,7 @@ size_t tryLiftUpUnion(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes) expr_node.step = std::make_unique( expr_node.children.front()->step->getOutputStream(), expression->getExpression().clone()); + expr_node.step->setStepDescription(expression->getStepDescription()); } /// - Expression - Something diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index 5df7d7b4e82..2eac1896066 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -899,6 +900,18 @@ AggregationInputOrder buildInputOrderInfo(AggregatingStep & aggregating, QueryPl return {}; } +static bool readingFromParallelReplicas(const QueryPlan::Node * node) +{ + IQueryPlanStep * step = node->step.get(); + while (!node->children.empty()) + { + step = node->children.front()->step.get(); + node = node->children.front(); + } + + return typeid_cast(step); +} + void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes) { if (node.children.size() != 1) @@ -924,6 +937,16 @@ void optimizeReadInOrder(QueryPlan::Node & node, QueryPlan::Nodes & nodes) std::vector infos; infos.reserve(node.children.size()); + for (const auto * child : union_node->children) + { + /// in case of parallel replicas + /// avoid applying read-in-order optimization for local replica + /// since it will lead to different parallel replicas modes + /// between local and remote nodes + if (readingFromParallelReplicas(child)) + return; + } + for (auto * child : union_node->children) { infos.push_back(buildInputOrderInfo(*sorting, *child, steps_to_update)); diff --git a/tests/queries/0_stateless/03228_pr_subquery_view_order_by.sql b/tests/queries/0_stateless/03228_pr_subquery_view_order_by.sql index b85392e0521..804a97f737f 100644 --- a/tests/queries/0_stateless/03228_pr_subquery_view_order_by.sql +++ b/tests/queries/0_stateless/03228_pr_subquery_view_order_by.sql @@ -12,7 +12,7 @@ FROM ) ORDER BY number DESC LIMIT 20 -SETTINGS cluster_for_parallel_replicas = 'parallel_replicas', allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 3, parallel_replicas_for_non_replicated_merge_tree = 1, parallel_replicas_local_plan = 1, query_plan_lift_up_union = 0; +SETTINGS cluster_for_parallel_replicas = 'parallel_replicas', allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 3, parallel_replicas_for_non_replicated_merge_tree = 1, parallel_replicas_local_plan = 1; DROP TABLE view1; DROP TABLE table1; From 81b8f8594c044b1de6e1b18877e1d9b80401f617 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 30 Aug 2024 21:23:07 +0000 Subject: [PATCH 119/215] Fix tests --- ...imize_distributed_group_by_sharding_key.reference | 4 ++-- .../02496_remove_redundant_sorting.reference | 12 ++++++------ ...02496_remove_redundant_sorting_analyzer.reference | 12 ++++++------ ...2500_remove_redundant_distinct_analyzer.reference | 4 ++-- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/queries/0_stateless/01952_optimize_distributed_group_by_sharding_key.reference b/tests/queries/0_stateless/01952_optimize_distributed_group_by_sharding_key.reference index e786532f25a..a807bf7096e 100644 --- a/tests/queries/0_stateless/01952_optimize_distributed_group_by_sharding_key.reference +++ b/tests/queries/0_stateless/01952_optimize_distributed_group_by_sharding_key.reference @@ -21,7 +21,7 @@ Expression (Projection) Union Expression ((Before LIMIT BY + (Before ORDER BY + (Convert VIEW subquery result to VIEW table structure + (Materialize constants after VIEW subquery + (Projection + Before ORDER BY)))))) ReadFromSystemNumbers - Expression + Expression (Before LIMIT BY) ReadFromRemote (Read from remote replica) explain select distinct on (k1, k2) v from remote('127.{1,2}', view(select 1 k1, 2 k2, 3 v from numbers(2)), cityHash64(k1, k2)); -- optimized Union @@ -96,7 +96,7 @@ Expression (Project names) LimitBy Expression ((Before LIMIT BY + (Projection + (Change column names to column identifiers + (Convert VIEW subquery result to VIEW table structure + (Materialize constants after VIEW subquery + (Project names + (Projection + (Change column names to column identifiers + (Project names + (Projection + Change column names to column identifiers))))))))))) ReadFromSystemNumbers - Expression + Expression (Before LIMIT BY) ReadFromRemote (Read from remote replica) explain select distinct on (k1, k2) v from remote('127.{1,2}', view(select 1 k1, 2 k2, 3 v from numbers(2)), cityHash64(k1, k2)); -- optimized Union diff --git a/tests/queries/0_stateless/02496_remove_redundant_sorting.reference b/tests/queries/0_stateless/02496_remove_redundant_sorting.reference index 4d004f2f78f..7824fd8cba9 100644 --- a/tests/queries/0_stateless/02496_remove_redundant_sorting.reference +++ b/tests/queries/0_stateless/02496_remove_redundant_sorting.reference @@ -395,9 +395,9 @@ Expression ((Projection + Before ORDER BY)) Union Expression ((Before ORDER BY + (Conversion before UNION + (Projection + Before ORDER BY)))) ReadFromStorage (SystemOne) - Expression (( + (Conversion before UNION + (Projection + Before ORDER BY)))) + Expression ((Before ORDER BY + (Conversion before UNION + (Projection + Before ORDER BY)))) ReadFromStorage (SystemOne) - Expression (( + (Conversion before UNION + (Projection + Before ORDER BY)))) + Expression ((Before ORDER BY + (Conversion before UNION + (Projection + Before ORDER BY)))) ReadFromStorage (SystemOne) -- execute Float64 9007199254740994 @@ -427,9 +427,9 @@ Expression ((Projection + Before ORDER BY)) Union Expression ((Before ORDER BY + (Conversion before UNION + (Projection + Before ORDER BY)))) ReadFromStorage (SystemOne) - Expression (( + (Conversion before UNION + (Projection + Before ORDER BY)))) + Expression ((Before ORDER BY + (Conversion before UNION + (Projection + Before ORDER BY)))) ReadFromStorage (SystemOne) - Expression (( + (Conversion before UNION + (Projection + Before ORDER BY)))) + Expression ((Before ORDER BY + (Conversion before UNION + (Projection + Before ORDER BY)))) ReadFromStorage (SystemOne) -- execute Nullable(Float64) 9007199254740994 @@ -459,9 +459,9 @@ Expression ((Projection + Before ORDER BY)) Union Expression ((Before ORDER BY + (Conversion before UNION + (Projection + Before ORDER BY)))) ReadFromStorage (SystemOne) - Expression (( + (Conversion before UNION + (Projection + Before ORDER BY)))) + Expression ((Before ORDER BY + (Conversion before UNION + (Projection + Before ORDER BY)))) ReadFromStorage (SystemOne) - Expression (( + (Conversion before UNION + (Projection + Before ORDER BY)))) + Expression ((Before ORDER BY + (Conversion before UNION + (Projection + Before ORDER BY)))) ReadFromStorage (SystemOne) -- execute Float64 9007199254740994 diff --git a/tests/queries/0_stateless/02496_remove_redundant_sorting_analyzer.reference b/tests/queries/0_stateless/02496_remove_redundant_sorting_analyzer.reference index dd5ac7bf706..3c68d14fdf2 100644 --- a/tests/queries/0_stateless/02496_remove_redundant_sorting_analyzer.reference +++ b/tests/queries/0_stateless/02496_remove_redundant_sorting_analyzer.reference @@ -394,9 +394,9 @@ Expression ((Project names + Projection)) Union Expression ((Before ORDER BY + (Projection + (Change column names to column identifiers + (Conversion before UNION + (Project names + (Projection + Change column names to column identifiers))))))) ReadFromStorage (SystemOne) - Expression (( + ( + ( + (Conversion before UNION + (Project names + (Projection + Change column names to column identifiers))))))) + Expression ((Before ORDER BY + (Projection + (Change column names to column identifiers + (Conversion before UNION + (Project names + (Projection + Change column names to column identifiers))))))) ReadFromStorage (SystemOne) - Expression (( + ( + ( + (Conversion before UNION + (Project names + (Projection + Change column names to column identifiers))))))) + Expression ((Before ORDER BY + (Projection + (Change column names to column identifiers + (Conversion before UNION + (Project names + (Projection + Change column names to column identifiers))))))) ReadFromStorage (SystemOne) -- execute Float64 9007199254740994 @@ -426,9 +426,9 @@ Expression ((Project names + Projection)) Union Expression ((Before ORDER BY + (Projection + (Change column names to column identifiers + (Conversion before UNION + (Project names + (Projection + Change column names to column identifiers))))))) ReadFromStorage (SystemOne) - Expression (( + ( + ( + (Conversion before UNION + (Project names + (Projection + Change column names to column identifiers))))))) + Expression ((Before ORDER BY + (Projection + (Change column names to column identifiers + (Conversion before UNION + (Project names + (Projection + Change column names to column identifiers))))))) ReadFromStorage (SystemOne) - Expression (( + ( + ( + (Conversion before UNION + (Project names + (Projection + Change column names to column identifiers))))))) + Expression ((Before ORDER BY + (Projection + (Change column names to column identifiers + (Conversion before UNION + (Project names + (Projection + Change column names to column identifiers))))))) ReadFromStorage (SystemOne) -- execute Nullable(Float64) 9007199254740994 @@ -458,9 +458,9 @@ Expression ((Project names + Projection)) Union Expression ((Before ORDER BY + (Projection + (Change column names to column identifiers + (Conversion before UNION + (Project names + (Projection + Change column names to column identifiers))))))) ReadFromStorage (SystemOne) - Expression (( + ( + ( + (Conversion before UNION + (Project names + (Projection + Change column names to column identifiers))))))) + Expression ((Before ORDER BY + (Projection + (Change column names to column identifiers + (Conversion before UNION + (Project names + (Projection + Change column names to column identifiers))))))) ReadFromStorage (SystemOne) - Expression (( + ( + ( + (Conversion before UNION + (Project names + (Projection + Change column names to column identifiers))))))) + Expression ((Before ORDER BY + (Projection + (Change column names to column identifiers + (Conversion before UNION + (Project names + (Projection + Change column names to column identifiers))))))) ReadFromStorage (SystemOne) -- execute Float64 9007199254740994 diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct_analyzer.reference b/tests/queries/0_stateless/02500_remove_redundant_distinct_analyzer.reference index b79f6310166..27b01cf1158 100644 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct_analyzer.reference +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct_analyzer.reference @@ -54,7 +54,7 @@ Expression (Project names) Distinct (Preliminary DISTINCT) Expression ((Projection + Change column names to column identifiers)) ReadFromSystemNumbers - Expression (( + ( + Project names))) + Expression ((Projection + (Change column names to column identifiers + Project names))) Distinct (DISTINCT) Distinct (Preliminary DISTINCT) Expression ((Projection + Change column names to column identifiers)) @@ -542,7 +542,7 @@ Expression (Project names) Distinct (Preliminary DISTINCT) Expression ((Projection + Change column names to column identifiers)) ReadFromSystemNumbers - Expression (( + ( + Project names))) + Expression ((Projection + (Change column names to column identifiers + Project names))) Distinct (DISTINCT) Distinct (Preliminary DISTINCT) Expression ((Projection + Change column names to column identifiers)) From bfa27d6650484063b0f8c9b340f5a85b3df2c27d Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Sat, 31 Aug 2024 06:11:50 +0000 Subject: [PATCH 120/215] Fix 02500_remove_redundant_distinct --- .../0_stateless/02500_remove_redundant_distinct.reference | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference index d7623cd5541..9bb8f4a4017 100644 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference @@ -53,7 +53,7 @@ Expression (Projection) Distinct (Preliminary DISTINCT) Expression (Before ORDER BY) ReadFromSystemNumbers - Expression (( + Projection)) + Expression ((Before ORDER BY + Projection)) Distinct Distinct (Preliminary DISTINCT) Expression (Before ORDER BY) @@ -536,7 +536,7 @@ Expression (Projection) Distinct (Preliminary DISTINCT) Expression (Before ORDER BY) ReadFromSystemNumbers - Expression (( + Projection)) + Expression ((Before ORDER BY + Projection)) Distinct Distinct (Preliminary DISTINCT) Expression (Before ORDER BY) From d7aaf053f992d4d6b27e9f88133818bb0a48d011 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 2 Sep 2024 12:50:12 +0000 Subject: [PATCH 121/215] Fix propogating structure argument in s3Cluster --- .../ObjectStorage/Azure/Configuration.cpp | 23 +++++-- .../ObjectStorage/Azure/Configuration.h | 2 +- .../ObjectStorage/HDFS/Configuration.cpp | 23 +++++-- .../ObjectStorage/HDFS/Configuration.h | 2 +- .../ObjectStorage/Local/Configuration.h | 2 +- .../ObjectStorage/S3/Configuration.cpp | 66 ++++++++++++++----- src/Storages/ObjectStorage/S3/Configuration.h | 2 +- .../ObjectStorage/StorageObjectStorage.h | 4 +- .../StorageObjectStorageCluster.cpp | 2 +- .../TableFunctionObjectStorage.h | 2 +- .../configs/named_collections.xml | 7 ++ tests/integration/test_s3_cluster/test.py | 41 ++++++++++++ 12 files changed, 139 insertions(+), 37 deletions(-) diff --git a/src/Storages/ObjectStorage/Azure/Configuration.cpp b/src/Storages/ObjectStorage/Azure/Configuration.cpp index 9730391d429..fa5346e9288 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.cpp +++ b/src/Storages/ObjectStorage/Azure/Configuration.cpp @@ -272,16 +272,25 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context, connection_params = getConnectionParams(connection_url, container_name, account_name, account_key, context); } -void StorageAzureConfiguration::addStructureAndFormatToArgs( +void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, ContextPtr context) { - if (tryGetNamedCollectionWithOverrides(args, context)) + if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) { - /// In case of named collection, just add key-value pair "structure='...'" - /// at the end of arguments to override existed structure. - ASTs equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; - auto equal_func = makeASTFunction("equals", std::move(equal_func_args)); - args.push_back(equal_func); + /// In case of named collection, just add key-value pairs "format='...', structure='...'" + /// at the end of arguments to override existed format and structure with "auto" values. + if (collection->getOrDefault("format", "auto") == "auto") + { + ASTs format_equal_func_args = {std::make_shared("format"), std::make_shared(format_)}; + auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); + args.push_back(format_equal_func); + } + if (collection->getOrDefault("structure", "auto") == "auto") + { + ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; + auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); + args.push_back(structure_equal_func); + } } else { diff --git a/src/Storages/ObjectStorage/Azure/Configuration.h b/src/Storages/ObjectStorage/Azure/Configuration.h index 4e6bfbc0745..8cf583c32db 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.h +++ b/src/Storages/ObjectStorage/Azure/Configuration.h @@ -44,7 +44,7 @@ public: ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override; - void addStructureAndFormatToArgs( + void addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.cpp b/src/Storages/ObjectStorage/HDFS/Configuration.cpp index 85eb29a3868..8abc51a7526 100644 --- a/src/Storages/ObjectStorage/HDFS/Configuration.cpp +++ b/src/Storages/ObjectStorage/HDFS/Configuration.cpp @@ -158,19 +158,28 @@ void StorageHDFSConfiguration::setURL(const std::string & url_) LOG_TRACE(getLogger("StorageHDFSConfiguration"), "Using URL: {}, path: {}", url, path); } -void StorageHDFSConfiguration::addStructureAndFormatToArgs( +void StorageHDFSConfiguration::addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, ContextPtr context) { - if (tryGetNamedCollectionWithOverrides(args, context)) + if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) { - /// In case of named collection, just add key-value pair "structure='...'" - /// at the end of arguments to override existed structure. - ASTs equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; - auto equal_func = makeASTFunction("equals", std::move(equal_func_args)); - args.push_back(equal_func); + /// In case of named collection, just add key-value pairs "format='...', structure='...'" + /// at the end of arguments to override existed format and structure with "auto" values. + if (collection->getOrDefault("format", "auto") == "auto") + { + ASTs format_equal_func_args = {std::make_shared("format"), std::make_shared(format_)}; + auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); + args.push_back(format_equal_func); + } + if (collection->getOrDefault("structure", "auto") == "auto") + { + ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; + auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); + args.push_back(structure_equal_func); + } } else { diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.h b/src/Storages/ObjectStorage/HDFS/Configuration.h index 04884542908..85f11aa6a98 100644 --- a/src/Storages/ObjectStorage/HDFS/Configuration.h +++ b/src/Storages/ObjectStorage/HDFS/Configuration.h @@ -39,7 +39,7 @@ public: ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override; - void addStructureAndFormatToArgs( + void addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, diff --git a/src/Storages/ObjectStorage/Local/Configuration.h b/src/Storages/ObjectStorage/Local/Configuration.h index ba4de63ac47..c713a81975b 100644 --- a/src/Storages/ObjectStorage/Local/Configuration.h +++ b/src/Storages/ObjectStorage/Local/Configuration.h @@ -40,7 +40,7 @@ public: ObjectStoragePtr createObjectStorage(ContextPtr, bool) override { return std::make_shared("/"); } - void addStructureAndFormatToArgs(ASTs &, const String &, const String &, ContextPtr) override { } + void addStructureAndFormatToArgsIfNeeded(ASTs &, const String &, const String &, ContextPtr) override { } private: void fromNamedCollection(const NamedCollection & collection, ContextPtr context) override; diff --git a/src/Storages/ObjectStorage/S3/Configuration.cpp b/src/Storages/ObjectStorage/S3/Configuration.cpp index 7542f59dcc4..73f25009422 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.cpp +++ b/src/Storages/ObjectStorage/S3/Configuration.cpp @@ -365,16 +365,25 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ keys = {url.key}; } -void StorageS3Configuration::addStructureAndFormatToArgs( +void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, ContextPtr context) { - if (tryGetNamedCollectionWithOverrides(args, context)) + if (auto collection = tryGetNamedCollectionWithOverrides(args, context)) { - /// In case of named collection, just add key-value pair "structure='...'" - /// at the end of arguments to override existed structure. - ASTs equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; - auto equal_func = makeASTFunction("equals", std::move(equal_func_args)); - args.push_back(equal_func); + /// In case of named collection, just add key-value pairs "format='...', structure='...'" + /// at the end of arguments to override existed format and structure with "auto" values. + if (collection->getOrDefault("format", "auto") == "auto") + { + ASTs format_equal_func_args = {std::make_shared("format"), std::make_shared(format_)}; + auto format_equal_func = makeASTFunction("equals", std::move(format_equal_func_args)); + args.push_back(format_equal_func); + } + if (collection->getOrDefault("structure", "auto") == "auto") + { + ASTs structure_equal_func_args = {std::make_shared("structure"), std::make_shared(structure_)}; + auto structure_equal_func = makeASTFunction("equals", std::move(structure_equal_func_args)); + args.push_back(structure_equal_func); + } } else { @@ -401,7 +410,10 @@ void StorageS3Configuration::addStructureAndFormatToArgs( auto second_arg = checkAndGetLiteralArgument(args[1], "format/NOSIGN"); /// If there is NOSIGN, add format=auto before structure. if (boost::iequals(second_arg, "NOSIGN")) - args.push_back(std::make_shared("auto")); + args.push_back(format_literal); + else if (checkAndGetLiteralArgument(args[1], "format") == "auto") + args[1] = format_literal; + args.push_back(structure_literal); } /// s3(source, format, structure) or @@ -413,16 +425,21 @@ void StorageS3Configuration::addStructureAndFormatToArgs( auto second_arg = checkAndGetLiteralArgument(args[1], "format/NOSIGN"); if (boost::iequals(second_arg, "NOSIGN")) { + if (checkAndGetLiteralArgument(args[2], "format") == "auto") + args[2] = format_literal; args.push_back(structure_literal); } else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg)) { - args[count - 1] = structure_literal; + if (second_arg == "auto") + args[1] = format_literal; + if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + args[2] = structure_literal; } else { - /// Add format=auto before structure argument. - args.push_back(std::make_shared("auto")); + /// Add format and structure arguments. + args.push_back(format_literal); args.push_back(structure_literal); } } @@ -435,14 +452,22 @@ void StorageS3Configuration::addStructureAndFormatToArgs( auto second_arg = checkAndGetLiteralArgument(args[1], "format/NOSIGN"); if (boost::iequals(second_arg, "NOSIGN")) { - args[count - 1] = structure_literal; + if (checkAndGetLiteralArgument(args[2], "format") == "auto") + args[2] = format_literal; + if (checkAndGetLiteralArgument(args[3], "structure") == "auto") + args[3] = structure_literal; } else if (second_arg == "auto" || FormatFactory::instance().exists(second_arg)) { - args[count - 2] = structure_literal; + if (second_arg == "auto") + args[1] = format_literal; + if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + args[2] = structure_literal; } else { + if (checkAndGetLiteralArgument(args[3], "format") == "auto") + args[3] = format_literal; args.push_back(structure_literal); } } @@ -454,17 +479,26 @@ void StorageS3Configuration::addStructureAndFormatToArgs( auto sedond_arg = checkAndGetLiteralArgument(args[1], "format/NOSIGN"); if (boost::iequals(sedond_arg, "NOSIGN")) { - args[count - 2] = structure_literal; + if (checkAndGetLiteralArgument(args[2], "format") == "auto") + args[2] = format_literal; + if (checkAndGetLiteralArgument(args[2], "structure") == "auto") + args[3] = structure_literal; } else { - args[count - 1] = structure_literal; + if (checkAndGetLiteralArgument(args[3], "format") == "auto") + args[3] = format_literal; + if (checkAndGetLiteralArgument(args[4], "structure") == "auto") + args[4] = structure_literal; } } /// s3(source, access_key_id, secret_access_key, format, structure, compression) else if (count == 6) { - args[count - 2] = structure_literal; + if (checkAndGetLiteralArgument(args[3], "format") == "auto") + args[3] = format_literal; + if (checkAndGetLiteralArgument(args[4], "structure") == "auto") + args[4] = structure_literal; } } } diff --git a/src/Storages/ObjectStorage/S3/Configuration.h b/src/Storages/ObjectStorage/S3/Configuration.h index 39a646c7df2..0aed0f366a6 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.h +++ b/src/Storages/ObjectStorage/S3/Configuration.h @@ -44,7 +44,7 @@ public: ObjectStoragePtr createObjectStorage(ContextPtr context, bool is_readonly) override; - void addStructureAndFormatToArgs( + void addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure, const String & format, diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index 562ca259089..f39586c23b4 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -180,7 +180,9 @@ public: virtual String getNamespace() const = 0; virtual StorageObjectStorage::QuerySettings getQuerySettings(const ContextPtr &) const = 0; - virtual void addStructureAndFormatToArgs( + + /// Add/replace structure and format arguments in the AST arguments if they have 'auto' values. + virtual void addStructureAndFormatToArgsIfNeeded( ASTs & args, const String & structure_, const String & format_, ContextPtr context) = 0; bool withPartitionWildcard() const; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp index 08a0739d929..d712e4eec20 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp @@ -103,7 +103,7 @@ void StorageObjectStorageCluster::updateQueryToSendIfNeeded( ASTPtr cluster_name_arg = args.front(); args.erase(args.begin()); - configuration->addStructureAndFormatToArgs(args, structure, configuration->format, context); + configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context); args.insert(args.begin(), cluster_name_arg); } diff --git a/src/TableFunctions/TableFunctionObjectStorage.h b/src/TableFunctions/TableFunctionObjectStorage.h index 3468e5c5007..6fa10face82 100644 --- a/src/TableFunctions/TableFunctionObjectStorage.h +++ b/src/TableFunctions/TableFunctionObjectStorage.h @@ -142,7 +142,7 @@ public: const String & format, const ContextPtr & context) { - Configuration().addStructureAndFormatToArgs(args, structure, format, context); + Configuration().addStructureAndFormatToArgsIfNeeded(args, structure, format, context); } protected: diff --git a/tests/integration/test_s3_cluster/configs/named_collections.xml b/tests/integration/test_s3_cluster/configs/named_collections.xml index 64d1bd98df2..2d3a69a8c38 100644 --- a/tests/integration/test_s3_cluster/configs/named_collections.xml +++ b/tests/integration/test_s3_cluster/configs/named_collections.xml @@ -6,5 +6,12 @@ minio123 CSV> + + http://minio1:9001/root/data/data{1,2,3} + minio + minio123 + JSONEachRow> + id UInt32, date Date DEFAULT 18262 + diff --git a/tests/integration/test_s3_cluster/test.py b/tests/integration/test_s3_cluster/test.py index 03919ee6a4d..b2954276ca5 100644 --- a/tests/integration/test_s3_cluster/test.py +++ b/tests/integration/test_s3_cluster/test.py @@ -459,3 +459,44 @@ def test_cluster_format_detection(started_cluster): ) assert result == expected_result + +def test_cluster_default_expression(started_cluster): + node = started_cluster.instances["s0_0_0"] + + node.query("insert into function s3('http://minio1:9001/root/data/data1', 'minio', 'minio123', JSONEachRow) select 1 as id settings s3_truncate_on_insert=1") + node.query("insert into function s3('http://minio1:9001/root/data/data2', 'minio', 'minio123', JSONEachRow) select * from numbers(0) settings s3_truncate_on_insert=1") + node.query("insert into function s3('http://minio1:9001/root/data/data3', 'minio', 'minio123', JSONEachRow) select 2 as id settings s3_truncate_on_insert=1") + + expected_result = node.query( + "SELECT * FROM s3('http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262') order by id" + ) + + result = node.query( + "SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262') order by id" + ) + + assert result == expected_result + + result = node.query( + "SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'auto', 'id UInt32, date Date DEFAULT 18262') order by id" + ) + + assert result == expected_result + + result = node.query( + "SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262', 'auto') order by id" + ) + + assert result == expected_result + + result = node.query( + "SELECT * FROM s3Cluster(cluster_simple, 'http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'auto', 'id UInt32, date Date DEFAULT 18262', 'auto') order by id" + ) + + assert result == expected_result + + result = node.query( + "SELECT * FROM s3Cluster(cluster_simple, test_s3_with_default) order by id" + ) + + assert result == expected_result \ No newline at end of file From 0cb8c9f1483674229165b8a2d40fa56a9bce75fa Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Mon, 2 Sep 2024 14:58:09 +0200 Subject: [PATCH 122/215] Fix typo --- src/Storages/ObjectStorage/S3/Configuration.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/ObjectStorage/S3/Configuration.cpp b/src/Storages/ObjectStorage/S3/Configuration.cpp index 73f25009422..8bf2669f2af 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.cpp +++ b/src/Storages/ObjectStorage/S3/Configuration.cpp @@ -476,7 +476,7 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( /// We can distinguish them by looking at the 2-nd argument: check if it's a NOSIGN keyword name or not. else if (count == 5) { - auto sedond_arg = checkAndGetLiteralArgument(args[1], "format/NOSIGN"); + auto second_arg = checkAndGetLiteralArgument(args[1], "format/NOSIGN"); if (boost::iequals(sedond_arg, "NOSIGN")) { if (checkAndGetLiteralArgument(args[2], "format") == "auto") From db8ce31bb785ae6a843089a02396abc2f986003e Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 2 Sep 2024 14:08:22 +0000 Subject: [PATCH 123/215] Increase flaky check timeout --- tests/ci/ci_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 7b0a7850576..fb4780d48c9 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -470,7 +470,7 @@ class CI: JobNames.STATELESS_TEST_FLAKY_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties( required_builds=[BuildNames.PACKAGE_ASAN], pr_only=True, - timeout=3600, + timeout=3 * 3600, # TODO: approach with reference job names does not work because digest may not be calculated if job skipped in wf # reference_job_name=JobNames.STATELESS_TEST_RELEASE, ), From 7e444136bbad7e80f3a1905bbea7fa4c7e9a8337 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Mon, 2 Sep 2024 18:27:24 +0200 Subject: [PATCH 124/215] Use QueryPlan for horizontal part of merge --- src/Interpreters/MutationsInterpreter.cpp | 13 +- src/Storages/MergeTree/MergeTask.cpp | 412 ++++++++++++------ .../MergeTree/MergeTreeSequentialSource.cpp | 51 ++- .../MergeTree/MergeTreeSequentialSource.h | 3 + 4 files changed, 334 insertions(+), 145 deletions(-) diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 0b93b5989b1..a8d45caeeaf 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -1217,9 +1217,16 @@ void MutationsInterpreter::Source::read( createReadFromPartStep( MergeTreeSequentialSourceType::Mutation, - plan, *data, storage_snapshot, - part, required_columns, - apply_deleted_mask_, std::move(filter), context_, + plan, + *data, storage_snapshot, + part, + required_columns, + nullptr, + apply_deleted_mask_, + std::move(filter), + false, + false, + context_, getLogger("MutationsInterpreter")); } else diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index fa86bb31629..3bee2ecb0d9 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -38,6 +38,11 @@ #include #include #include +#include +#include +#include +#include +#include #include #include #include @@ -1206,12 +1211,204 @@ bool MergeTask::execute() } +/// Apply merge strategy (Ordinary, Colapsing, Aggregating, etc) to the stream +class ApplyMergeStep : public ITransformingStep /// TODO: is this transformation step? +{ +public: + ApplyMergeStep( + const DataStream & input_stream_, + const SortDescription & sort_description_, + const Names partition_key_columns_, + const MergeTreeData::MergingParams & merging_params_, + WriteBuffer * rows_sources_write_buf_, + UInt64 merge_block_size_rows_, + UInt64 merge_block_size_bytes_, + bool blocks_are_granules_size_, + bool cleanup_) + : ITransformingStep(input_stream_, input_stream_.header, Traits{}) // TODO proper traits? + , sort_description(sort_description_) + , partition_key_columns(partition_key_columns_) + , merging_params(merging_params_) + , rows_sources_write_buf(rows_sources_write_buf_) + , merge_block_size_rows(merge_block_size_rows_) + , merge_block_size_bytes(merge_block_size_bytes_) + , blocks_are_granules_size(blocks_are_granules_size_) + , cleanup(cleanup_) + {} + + String getName() const override { return "ApplyMergePolicy"; } + + void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & /*pipelineSettings*/) override + { + /// The order of the streams is important: when the key is matched, the elements go in the order of the source stream number. + /// In the merged part, the lines with the same key must be in the ascending order of the identifier of original part, + /// that is going in insertion order. + ProcessorPtr merged_transform; + +// /// There is no sense to have the block size bigger than one granule for merge operations. +// const UInt64 merge_block_size_rows = data_settings->merge_max_block_size; +// const UInt64 merge_block_size_bytes = data_settings->merge_max_block_size_bytes; + + const auto &header = pipeline.getHeader(); + const auto input_streams_count = pipeline.getNumStreams(); + + switch (merging_params.mode) + { + case MergeTreeData::MergingParams::Ordinary: + merged_transform = std::make_shared( + header, + input_streams_count, + sort_description, + merge_block_size_rows, + merge_block_size_bytes, + SortingQueueStrategy::Default, + /* limit_= */0, + /* always_read_till_end_= */false, + rows_sources_write_buf, + blocks_are_granules_size); + break; + + case MergeTreeData::MergingParams::Collapsing: + merged_transform = std::make_shared( + header, input_streams_count, sort_description, merging_params.sign_column, false, + merge_block_size_rows, merge_block_size_bytes, rows_sources_write_buf, blocks_are_granules_size); + break; + + case MergeTreeData::MergingParams::Summing: + merged_transform = std::make_shared( + header, input_streams_count, sort_description, merging_params.columns_to_sum, partition_key_columns, merge_block_size_rows, merge_block_size_bytes); + break; + + case MergeTreeData::MergingParams::Aggregating: + merged_transform = std::make_shared(header, input_streams_count, sort_description, merge_block_size_rows, merge_block_size_bytes); + break; + + case MergeTreeData::MergingParams::Replacing: + merged_transform = std::make_shared( + header, input_streams_count, sort_description, merging_params.is_deleted_column, merging_params.version_column, + merge_block_size_rows, merge_block_size_bytes, rows_sources_write_buf, blocks_are_granules_size, + cleanup); + break; + + case MergeTreeData::MergingParams::Graphite: + merged_transform = std::make_shared( + header, input_streams_count, sort_description, merge_block_size_rows, merge_block_size_bytes, + merging_params.graphite_params, time_of_merge); + break; + + case MergeTreeData::MergingParams::VersionedCollapsing: + merged_transform = std::make_shared( + header, input_streams_count, sort_description, merging_params.sign_column, + merge_block_size_rows, merge_block_size_bytes, rows_sources_write_buf, blocks_are_granules_size); + break; + } + + pipeline.addTransform(std::move(merged_transform)); + +#ifndef NDEBUG + if (!sort_description.empty()) + { + pipeline.addSimpleTransform([&](const Block & header_) + { + auto transform = std::make_shared(header_, sort_description); + return transform; + }); + } +#endif + } + + void updateOutputStream() override + { + output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); + output_stream->sort_description = sort_description; + + /// TODO: is this correct? +// if (partition_key_columns.empty()) + output_stream->sort_scope = DataStream::SortScope::Global; +// else +// output_stream->sort_scope = DataStream::SortScope::Stream; + } + +private: + SortDescription sort_description; + Names partition_key_columns; + MergeTreeData::MergingParams merging_params{}; + WriteBuffer * rows_sources_write_buf; + const UInt64 merge_block_size_rows; + const UInt64 merge_block_size_bytes; + bool blocks_are_granules_size; + bool cleanup{false}; + time_t time_of_merge{0}; +}; + + +class MaterializingStep : public ITransformingStep /// TODO: is this transformation step? +{ +public: + explicit MaterializingStep( + const DataStream & input_stream_) + : ITransformingStep(input_stream_, input_stream_.header, Traits{}) // TODO proper traits? + {} + + String getName() const override { return "Materializing"; } + + void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override + { + pipeline.addTransform(std::make_shared(input_streams.front().header)); + } + + void updateOutputStream() override + { + /// TODO: can this be simplified? + output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); + output_stream->sort_description = input_streams.front().sort_description; + } +}; + + +class TTLStep : public ITransformingStep +{ +public: + TTLStep( + const DataStream & input_stream_, + const ContextPtr & context_, + const MergeTreeData & storage_, + const StorageMetadataPtr & metadata_snapshot_, + const MergeTreeData::MutableDataPartPtr & data_part_, + time_t current_time, + bool force_) + : ITransformingStep(input_stream_, input_stream_.header, Traits{}) // TODO proper traits? + { + transform = std::make_shared(context_, input_stream_.header, storage_, metadata_snapshot_, data_part_, current_time, force_); + subqueries_for_sets = transform->getSubqueries(); + } + + String getName() const override { return "Materializing"; } + + PreparedSets::Subqueries getSubqueries() { return std::move(subqueries_for_sets); } + + void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override + { + pipeline.addTransform(transform); + } + + void updateOutputStream() override + { + // TODO: implement? + } + +private: + std::shared_ptr transform; + PreparedSets::Subqueries subqueries_for_sets; +}; + + void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() { /** Read from all parts, merge and write into a new one. * In passing, we calculate expression for sorting. */ - Pipes pipes; + global_ctx->watch_prev_elapsed = 0; /// We count total amount of bytes in parts @@ -1238,143 +1435,92 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() global_ctx->horizontal_stage_progress = std::make_unique( ctx->column_sizes ? ctx->column_sizes->keyColumnsWeight() : 1.0); + auto sorting_key_expression_dag = global_ctx->metadata_snapshot->getSortingKey().expression->getActionsDAG().clone(); + + /// Read from all parts + std::vector plans; for (const auto & part : global_ctx->future_part->parts) { - Pipe pipe = createMergeTreeSequentialSource( + /// TODO: this is just for debugging purposes, remove it later + if (part->getMarksCount() == 0) + LOG_DEBUG(ctx->log, "Part {} is empty", part->name); + + auto plan_for_part = std::make_unique(); + createReadFromPartStep( MergeTreeSequentialSourceType::Merge, + *plan_for_part, *global_ctx->data, global_ctx->storage_snapshot, part, global_ctx->merging_columns.getNames(), - /*mark_ranges=*/ {}, global_ctx->input_rows_filtered, /*apply_deleted_mask=*/ true, + /*filter=*/ std::nullopt, ctx->read_with_direct_io, - /*prefetch=*/ false); + /*prefetch=*/ false, + global_ctx->context, + ctx->log); if (global_ctx->metadata_snapshot->hasSortingKey()) { - pipe.addSimpleTransform([this](const Block & header) - { - return std::make_shared(header, global_ctx->metadata_snapshot->getSortingKey().expression); - }); + /// Calculate sorting key expressions so that they are available for merge sorting. + auto calculate_sorting_key_expression_step = std::make_unique( + plan_for_part->getCurrentDataStream(), + sorting_key_expression_dag.clone()); /// TODO: can we avoid cloning here? + plan_for_part->addStep(std::move(calculate_sorting_key_expression_step)); } - pipes.emplace_back(std::move(pipe)); + plans.emplace_back(std::move(plan_for_part)); } + QueryPlan merge_parts_query_plan; - Names sort_columns = global_ctx->metadata_snapshot->getSortingKeyColumns(); - SortDescription sort_description; - sort_description.compile_sort_description = global_ctx->data->getContext()->getSettingsRef().compile_sort_description; - sort_description.min_count_to_compile_sort_description = global_ctx->data->getContext()->getSettingsRef().min_count_to_compile_sort_description; - - size_t sort_columns_size = sort_columns.size(); - sort_description.reserve(sort_columns_size); - - Names partition_key_columns = global_ctx->metadata_snapshot->getPartitionKey().column_names; - - Block header = pipes.at(0).getHeader(); - for (size_t i = 0; i < sort_columns_size; ++i) - sort_description.emplace_back(sort_columns[i], 1, 1); - -#ifndef NDEBUG - if (!sort_description.empty()) + /// Union of all parts streams { - for (size_t i = 0; i < pipes.size(); ++i) - { - auto & pipe = pipes[i]; - pipe.addSimpleTransform([&](const Block & header_) - { - auto transform = std::make_shared(header_, sort_description); - transform->setDescription(global_ctx->future_part->parts[i]->name); - return transform; - }); - } + DataStreams input_streams; + input_streams.reserve(plans.size()); + for (auto & plan : plans) + input_streams.emplace_back(plan->getCurrentDataStream()); + + auto union_step = std::make_unique(std::move(input_streams)); + merge_parts_query_plan.unitePlans(std::move(union_step), std::move(plans)); } -#endif - /// The order of the streams is important: when the key is matched, the elements go in the order of the source stream number. - /// In the merged part, the lines with the same key must be in the ascending order of the identifier of original part, - /// that is going in insertion order. - ProcessorPtr merged_transform; - - /// If merge is vertical we cannot calculate it - ctx->blocks_are_granules_size = (global_ctx->chosen_merge_algorithm == MergeAlgorithm::Vertical); - - /// There is no sense to have the block size bigger than one granule for merge operations. - const UInt64 merge_block_size_rows = data_settings->merge_max_block_size; - const UInt64 merge_block_size_bytes = data_settings->merge_max_block_size_bytes; - - switch (ctx->merging_params.mode) + /// Merge { - case MergeTreeData::MergingParams::Ordinary: - merged_transform = std::make_shared( - header, - pipes.size(), - sort_description, - merge_block_size_rows, - merge_block_size_bytes, - SortingQueueStrategy::Default, - /* limit_= */0, - /* always_read_till_end_= */false, - ctx->rows_sources_write_buf.get(), - ctx->blocks_are_granules_size); - break; + Names sort_columns = global_ctx->metadata_snapshot->getSortingKeyColumns(); + SortDescription sort_description; + sort_description.compile_sort_description = global_ctx->data->getContext()->getSettingsRef().compile_sort_description; + sort_description.min_count_to_compile_sort_description = global_ctx->data->getContext()->getSettingsRef().min_count_to_compile_sort_description; - case MergeTreeData::MergingParams::Collapsing: - merged_transform = std::make_shared( - header, pipes.size(), sort_description, ctx->merging_params.sign_column, false, - merge_block_size_rows, merge_block_size_bytes, ctx->rows_sources_write_buf.get(), ctx->blocks_are_granules_size); - break; + size_t sort_columns_size = sort_columns.size(); + sort_description.reserve(sort_columns_size); - case MergeTreeData::MergingParams::Summing: - merged_transform = std::make_shared( - header, pipes.size(), sort_description, ctx->merging_params.columns_to_sum, partition_key_columns, merge_block_size_rows, merge_block_size_bytes); - break; + Names partition_key_columns = global_ctx->metadata_snapshot->getPartitionKey().column_names; - case MergeTreeData::MergingParams::Aggregating: - merged_transform = std::make_shared(header, pipes.size(), sort_description, merge_block_size_rows, merge_block_size_bytes); - break; + for (size_t i = 0; i < sort_columns_size; ++i) + sort_description.emplace_back(sort_columns[i], 1, 1); - case MergeTreeData::MergingParams::Replacing: - if (global_ctx->cleanup && !data_settings->allow_experimental_replacing_merge_with_cleanup) - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental merges with CLEANUP are not allowed"); + /// If merge is vertical we cannot calculate it + ctx->blocks_are_granules_size = (global_ctx->chosen_merge_algorithm == MergeAlgorithm::Vertical); - merged_transform = std::make_shared( - header, pipes.size(), sort_description, ctx->merging_params.is_deleted_column, ctx->merging_params.version_column, - merge_block_size_rows, merge_block_size_bytes, ctx->rows_sources_write_buf.get(), ctx->blocks_are_granules_size, - global_ctx->cleanup); - break; + if (global_ctx->cleanup && !data_settings->allow_experimental_replacing_merge_with_cleanup) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental merges with CLEANUP are not allowed"); - case MergeTreeData::MergingParams::Graphite: - merged_transform = std::make_shared( - header, pipes.size(), sort_description, merge_block_size_rows, merge_block_size_bytes, - ctx->merging_params.graphite_params, global_ctx->time_of_merge); - break; - - case MergeTreeData::MergingParams::VersionedCollapsing: - merged_transform = std::make_shared( - header, pipes.size(), sort_description, ctx->merging_params.sign_column, - merge_block_size_rows, merge_block_size_bytes, ctx->rows_sources_write_buf.get(), ctx->blocks_are_granules_size); - break; + auto merge_step = std::make_unique( + merge_parts_query_plan.getCurrentDataStream(), + sort_description, + partition_key_columns, + ctx->merging_params, + ctx->rows_sources_write_buf.get(), + data_settings->merge_max_block_size, + data_settings->merge_max_block_size_bytes, + ctx->blocks_are_granules_size, + global_ctx->cleanup); + merge_step->setStepDescription("Merge sorted parts"); + merge_parts_query_plan.addStep(std::move(merge_step)); } - auto builder = std::make_unique(); - builder->init(Pipe::unitePipes(std::move(pipes))); - builder->addTransform(std::move(merged_transform)); - -#ifndef NDEBUG - if (!sort_description.empty()) - { - builder->addSimpleTransform([&](const Block & header_) - { - auto transform = std::make_shared(header_, sort_description); - return transform; - }); - } -#endif - if (global_ctx->deduplicate) { const auto & virtuals = *global_ctx->data->getVirtualsPtr(); @@ -1383,44 +1529,56 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() /// If deduplicate_by_columns is empty, add all columns except virtuals. if (global_ctx->deduplicate_by_columns.empty()) { - for (const auto & column : global_ctx->merging_columns) + for (const auto & column_name : global_ctx->merging_columns.getNames()) { - if (virtuals.tryGet(column.name, VirtualsKind::Persistent)) + if (virtuals.tryGet(column_name, VirtualsKind::Persistent)) continue; - global_ctx->deduplicate_by_columns.emplace_back(column.name); + global_ctx->deduplicate_by_columns.emplace_back(column_name); } } - if (DistinctSortedTransform::isApplicable(header, sort_description, global_ctx->deduplicate_by_columns)) - builder->addTransform(std::make_shared( - builder->getHeader(), sort_description, SizeLimits(), 0 /*limit_hint*/, global_ctx->deduplicate_by_columns)); - else - builder->addTransform(std::make_shared( - builder->getHeader(), SizeLimits(), 0 /*limit_hint*/, global_ctx->deduplicate_by_columns)); + auto deduplication_step = std::make_unique( + merge_parts_query_plan.getCurrentDataStream(), + SizeLimits(), 0 /*limit_hint*/, + global_ctx->deduplicate_by_columns, + false, + true /*TODO: ??*/); + deduplication_step->setStepDescription("Deduplication step"); + merge_parts_query_plan.addStep(std::move(deduplication_step)); } PreparedSets::Subqueries subqueries; + /// TTL step if (ctx->need_remove_expired_values) { - auto transform = std::make_shared(global_ctx->context, builder->getHeader(), *global_ctx->data, global_ctx->metadata_snapshot, global_ctx->new_data_part, global_ctx->time_of_merge, ctx->force_ttl); - subqueries = transform->getSubqueries(); - builder->addTransform(std::move(transform)); + auto ttl_step = std::make_unique( + merge_parts_query_plan.getCurrentDataStream(), global_ctx->context, *global_ctx->data, global_ctx->metadata_snapshot, global_ctx->new_data_part, global_ctx->time_of_merge, ctx->force_ttl); + subqueries = ttl_step->getSubqueries(); + ttl_step->setStepDescription("TTL step"); + merge_parts_query_plan.addStep(std::move(ttl_step)); } + /// Secondary indices expressions if (!global_ctx->merging_skip_indexes.empty()) { - builder->addTransform(std::make_shared( - builder->getHeader(), - global_ctx->merging_skip_indexes.getSingleExpressionForIndices(global_ctx->metadata_snapshot->getColumns(), - global_ctx->data->getContext()))); - - builder->addTransform(std::make_shared(builder->getHeader())); + auto indices_expression_dag = global_ctx->merging_skip_indexes.getSingleExpressionForIndices(global_ctx->metadata_snapshot->getColumns(), global_ctx->data->getContext())->getActionsDAG().clone(); + auto calculate_indices_expression_step = std::make_unique( + merge_parts_query_plan.getCurrentDataStream(), + std::move(indices_expression_dag)); + merge_parts_query_plan.addStep(std::move(calculate_indices_expression_step)); + /// TODO: what is the purpose of MaterializingTransform in the original code? + merge_parts_query_plan.addStep(std::make_unique(merge_parts_query_plan.getCurrentDataStream())); } if (!subqueries.empty()) - builder = addCreatingSetsTransform(std::move(builder), std::move(subqueries), global_ctx->context); + addCreatingSetsStep(merge_parts_query_plan, std::move(subqueries), global_ctx->context); + + auto pipelineSettings = BuildQueryPipelineSettings::fromContext(global_ctx->context); + auto builder = merge_parts_query_plan.buildQueryPipeline( + QueryPlanOptimizationSettings::fromContext(global_ctx->context), + pipelineSettings); global_ctx->merged_pipeline = QueryPipelineBuilder::getPipeline(std::move(*builder)); /// Dereference unique_ptr and pass horizontal_stage_progress by reference diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index 39aa191a3d2..444a59b5590 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -347,8 +347,11 @@ public: const StorageSnapshotPtr & storage_snapshot_, MergeTreeData::DataPartPtr data_part_, Names columns_to_read_, + std::shared_ptr> filtered_rows_count_, bool apply_deleted_mask_, std::optional filter_, + bool read_with_direct_io_, + bool prefetch_, ContextPtr context_, LoggerPtr log_) : ISourceStep(DataStream{.header = storage_snapshot_->getSampleBlockForColumns(columns_to_read_)}) @@ -357,8 +360,11 @@ public: , storage_snapshot(storage_snapshot_) , data_part(std::move(data_part_)) , columns_to_read(std::move(columns_to_read_)) + , filtered_rows_count(std::move(filtered_rows_count_)) , apply_deleted_mask(apply_deleted_mask_) , filter(std::move(filter_)) + , read_with_direct_io(read_with_direct_io_) + , prefetch(prefetch_) , context(std::move(context_)) , log(log_) { @@ -401,24 +407,27 @@ public: data_part, columns_to_read, std::move(mark_ranges), - /*filtered_rows_count=*/ nullptr, + filtered_rows_count, apply_deleted_mask, - /*read_with_direct_io=*/ false, - /*prefetch=*/ false); + read_with_direct_io, + prefetch); pipeline.init(Pipe(std::move(source))); } private: - MergeTreeSequentialSourceType type; + const MergeTreeSequentialSourceType type; const MergeTreeData & storage; - StorageSnapshotPtr storage_snapshot; - MergeTreeData::DataPartPtr data_part; - Names columns_to_read; - bool apply_deleted_mask; - std::optional filter; - ContextPtr context; - LoggerPtr log; + const StorageSnapshotPtr storage_snapshot; + const MergeTreeData::DataPartPtr data_part; + const Names columns_to_read; + const std::shared_ptr> filtered_rows_count; + const bool apply_deleted_mask; + const std::optional filter; + const bool read_with_direct_io; + const bool prefetch; + const ContextPtr context; + const LoggerPtr log; }; void createReadFromPartStep( @@ -428,15 +437,27 @@ void createReadFromPartStep( const StorageSnapshotPtr & storage_snapshot, MergeTreeData::DataPartPtr data_part, Names columns_to_read, + std::shared_ptr> filtered_rows_count, bool apply_deleted_mask, std::optional filter, + bool read_with_direct_io, + bool prefetch, ContextPtr context, LoggerPtr log) { - auto reading = std::make_unique(type, - storage, storage_snapshot, std::move(data_part), - std::move(columns_to_read), apply_deleted_mask, - std::move(filter), std::move(context), log); + auto reading = std::make_unique( + type, + storage, + storage_snapshot, + std::move(data_part), + std::move(columns_to_read), + filtered_rows_count, + apply_deleted_mask, + std::move(filter), + read_with_direct_io, + prefetch, + std::move(context), + log); plan.addStep(std::move(reading)); } diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.h b/src/Storages/MergeTree/MergeTreeSequentialSource.h index 1b05512b9a3..543d1f60d10 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.h +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.h @@ -37,8 +37,11 @@ void createReadFromPartStep( const StorageSnapshotPtr & storage_snapshot, MergeTreeData::DataPartPtr data_part, Names columns_to_read, + std::shared_ptr> filtered_rows_count, bool apply_deleted_mask, std::optional filter, + bool read_with_direct_io, + bool prefetch, ContextPtr context, LoggerPtr log); From 547276780c688feed8e3dd216f661c861cdd631b Mon Sep 17 00:00:00 2001 From: marco-vb Date: Mon, 2 Sep 2024 17:11:15 +0000 Subject: [PATCH 125/215] Removed unnecessary code and restored test certificates. --- src/Access/Authentication.cpp | 3 - .../certs/ca-cert.pem | 56 +++++----- .../certs/ca-cert.srl | 2 +- .../certs/ca-key.pem | 100 +++++++++--------- .../certs/client1-cert.pem | 52 ++++----- .../certs/client1-key.pem | 100 +++++++++--------- .../certs/client1-req.pem | 46 ++++---- .../certs/client2-cert.pem | 52 ++++----- .../certs/client2-key.pem | 100 +++++++++--------- .../certs/client2-req.pem | 46 ++++---- .../certs/client3-cert.pem | 52 ++++----- .../certs/client3-key.pem | 100 +++++++++--------- .../certs/client3-req.pem | 46 ++++---- .../certs/client4-cert.pem | 53 +++++----- .../certs/client4-key.pem | 100 +++++++++--------- .../certs/client4-req.pem | 46 ++++---- .../certs/client5-cert.pem | 56 +++++----- .../certs/client5-key.pem | 100 +++++++++--------- .../certs/client5-req.pem | 46 ++++---- .../certs/client6-cert.pem | 56 +++++----- .../certs/client6-key.pem | 100 +++++++++--------- .../certs/client6-req.pem | 46 ++++---- .../certs/server-cert.pem | 56 +++++----- .../certs/server-key.pem | 100 +++++++++--------- .../certs/server-req.pem | 46 ++++---- .../certs/wrong-cert.pem | 52 ++++----- .../certs/wrong-key.pem | 100 +++++++++--------- 27 files changed, 853 insertions(+), 859 deletions(-) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 994aec57f0c..3acf8a89b70 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -6,13 +5,11 @@ #include #include #include -#include "Common/LoggingFormatStringHelpers.h" #include #include #include #include -#include "base/types.h" #include "config.h" namespace DB diff --git a/tests/integration/test_ssl_cert_authentication/certs/ca-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/ca-cert.pem index bec7f132e23..d1e4a3a88d9 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/ca-cert.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/ca-cert.pem @@ -1,32 +1,32 @@ -----BEGIN CERTIFICATE----- -MIIFhTCCA22gAwIBAgIULizd839OeWFia09eiBiQGk7m/wYwDQYJKoZIhvcNAQEL +MIIFhTCCA22gAwIBAgIUZmPYBB6vdp8uxKlJcS8mI0SArqQwDQYJKoZIhvcNAQEL BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwODEy -MTAzOTI4WhcNMzQwODEwMTAzOTI4WjBSMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwNjI2 +MTAyNTAwWhcNMzQwNjI0MTAyNTAwWjBSMQswCQYDVQQGEwJSVTETMBEGA1UECAwK U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQsw -CQYDVQQDDAJjYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAME6XxeV -Nl6RnBKhr1yygLIXYIVFmqgcEyAm/5hWhV2bERP6urmIYLWcokaFpBMCXkH5tumA -2B5PQQgVVTTVv51qGNgCHR7Vfmtf3s0BlAn1XI719TelWrtVP7XzLlDjDl5ubb3I -WB0ZhckU57of0kZrUIfrqJKaSyKqYp/Hgjdczuk2uIgv12eX7r47qhfq6B9rmhVp -a/WRB2LbBoyJyjsY6UdRLld7vO2wVSWdmy9wxVhssbkP9zNY3JbE81HUMhyw4F/W -j+ryTesILSYUGvx4DjxvP8zlMvL4N81/V7q8nVkwiu9A3owECqUz5pAHfVk6U4zF -vhU+7TTLP3eyFwJk9o1ctxjT0zKtxLRv+ss8eER/7loLOD+zvq3Y/5XCHGd5WILv -9cDfz44XS0UMRdE/M2RQnFmTDPDKy1IY2TxpQLEv4txtY9h/edRDwcPPaU17gY8Z -boXZR61d90zOIDGITAr8DD/xrsei57ly/W9cbIKfNj0QfH0T6ESP5lW3fS9GRwjY -djoeOZ8uK0d+pzHGUVTkE/9Hf9Zl4NfnwvvFvvF3X/U2VeBkxw+ZgcEiphI+uUjx -aXsn7leNuAemPB50LEnQD5bWKIT1qIeVIQWOHzP5g+tGxCsfg/YzoUb8MvZSe+n2 -FLDqSYKUPy6mZf0U38teh7eIfrIT6yux5IzXAgMBAAGjUzBRMB0GA1UdDgQWBBRG -jKB4cx0JVNfQH8o4feWwhu6NRDAfBgNVHSMEGDAWgBRGjKB4cx0JVNfQH8o4feWw -hu6NRDAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBtD9HsFU2D -28aJozGf2sr2CL1wjlpuOSB5LYu3rcuMrU8R+6xR7rSSUsxuXGB5drnnur9UQ8gt -9Q/sxWUyErHTycfFSQy3mhsFIUy0bGwebSeEPCK3DJo5q3QsncxidcIpTJMgUHe3 -MfyEA2ow+qnlTKROHitiALA+OzEcDdTbU1+JSUM4w0JPfHSdHcwlZzTQu5BhJOjQ -msS3letdX5iqD8xdfIG7EWn/LRupTJjSMsMwspOq9Cfx9kcgI+n9rPJt9iL9eSE7 -gBhUc6Df3JDuX/qImH8lMzE/F7H/7XReCdWXinApW94SZttTbnEWhMhpsMw1psNo -vfw0HSQnu6G6eRdbZK2TCXgzc9BwGcowfQXbv1UvWAhoNBw75UO5nOzCR2T7SakE -R1fNHEQyjFBmq60r7AYitRjcX+nE7F+T/MatpBKFoEz1aDxWfcRx+kwuJZi0sw+B -US4PWVIGN2SFfShp9JpVa8Q3PQmUFyIWCPJlEbBohTTja/nvppr8/OSTg3A3a1s1 -J2DTscxDmQGCWRBbO2LlQE495ylNMfS7BdTDrp/Xv3h+pM+qi4my8+JOC7uX/9dL -X6b7Kz9d8HlxQGug/HvnaNN6k5sQpRJN7LBGPHUj1RG5/kw0WdEDs12SfXfMle6b -q+kwPJhMAGv7CfFXKWtCjt4M5sLCAagdxA== +CQYDVQQDDAJjYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALHzp+SR +T8EtrMBYNlgwIUGMZaXomQPwsjOpjt3RUsdE3LCQ15rLuyBwZ0SbLMDOazCDA7Xr ++AXHDYwg/PCJhe2N4NTzfgnWqbkIGwYLhys95Xxq+q+kV7Csk/27JSk1rsK3Nru/ +Iuj/NWgBkAQC8n10qfbJSXHcXm9wDVy/L8t7DXwxkX70LToGWsb6QW6VkiWe/QOW +QYTMVncqjrtLTCTnqGIk0s6ZIifgPLfaYJxtwK+LdIoJioSbAuVfHmVTe10vjonO +YqJ56KfqOn/G7dIf1l9gOLFBAe4jUf3RS1wkQMdk+oEGLbGnShgW+BQMfB48RkrQ +486h7slzwC29jBJLRARI2Oc9p8/zBDVph0pxkjVGka8dfIkZbmTD932h/1gfMgQl +F20G/H5FF1jk37bDcsczns0c24S1F2uJbzOlHjFLhqmH1IaVCWsYawfBs9khModW +VS6+WAv//cqWE3KmmJ2EdtAmzMCJzAQUEyrMZWmrFrBzpACyMq2zFEtyIXsCXpgq +eW4odxIIZlClibo1FGrflqN+hXnAhRyCj7WJBQ0ZPrEdRMRpBYhYdmygPJ+lWfsg +HOtNnshSuJTXGJTtsuJVr4Ioeq6tKfWGofRu4vvT6cILZjbp9VaxxfVLS7bTsjWR +c+5xHp+KbcjSfw2cJHQN85hlWTpMe9jPhgt/AgMBAAGjUzBRMB0GA1UdDgQWBBSJ +Kj1dSYN0jW+Mu0xFQrobZFmQpTAfBgNVHSMEGDAWgBSJKj1dSYN0jW+Mu0xFQrob +ZFmQpTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQCEm/epjNWe +w0A+1M3i0y8adrXWdTeRXuP3Klgtjh3NGFKNm9WGnOAufsqUIpVwKLw1uYNIhiEA +fj5KuD7YIVU9CrWx4Z/+lILUZhyn/br5mACtRiRTF/75/QwLcDz5z7K9MyMzdL99 +DLQ9bd3JCuEgVj6zacPrALwWVhM8u9fNGJxdQANnIC8yTY5+ZE59/fn7UI787JuR +4njOGWSVnDavbTyJCPMPiUkgwqL+QSWBcNbGAPzMaAblvc1SL2Lj/ikFDAETAZs2 +T/3ZqBqHEOuVhFQYTAvMAdMQX3w8bYv/CGL8++W+qHazY+uqPypd9CLnICbnkZmr +P+id9WleGl2F//u1CQ+YA2Q3EazSFhwRLA7IKIVCrYVaBsbe/bpxxZb6+AQVfM/i ++7+fCbr7A5HDe9Fi4dClv6xPI0GZZkarhQPsoLPaDQeqM4OE+K6oPSHJnqfAB8v3 +NgTt1QuVnKhwveX5DDEP4t/Qt4j2n7AFpeoZDEA8aM33K0noXNrwqHz3991O1RWz +t/gd+cFG/Z1jRP8kYtfAV8go2nzt8QvqBhfIcNnMwD8cwuKJ5G7SdqLvDFj3XCCO +YqQAALl4QFs046eVDEWLajSESmj4fCaTmO05sHHLy7U5asoAo/MWGbmGmL+8ExUX +sPO9r12lPJ7IThJ13PSqbJIJnnloL/XCxA== -----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/ca-cert.srl b/tests/integration/test_ssl_cert_authentication/certs/ca-cert.srl index 71859de222d..cf47b0dc79c 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/ca-cert.srl +++ b/tests/integration/test_ssl_cert_authentication/certs/ca-cert.srl @@ -1 +1 @@ -05F10C67567FE30795D77AF2540F6AC8D4CF247E +05F10C67567FE30795D77AF2540F6AC8D4CF2470 diff --git a/tests/integration/test_ssl_cert_authentication/certs/ca-key.pem b/tests/integration/test_ssl_cert_authentication/certs/ca-key.pem index d4466037e77..2dea2ccd837 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/ca-key.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/ca-key.pem @@ -1,52 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDBOl8XlTZekZwS -oa9csoCyF2CFRZqoHBMgJv+YVoVdmxET+rq5iGC1nKJGhaQTAl5B+bbpgNgeT0EI -FVU01b+dahjYAh0e1X5rX97NAZQJ9VyO9fU3pVq7VT+18y5Q4w5ebm29yFgdGYXJ -FOe6H9JGa1CH66iSmksiqmKfx4I3XM7pNriIL9dnl+6+O6oX6ugfa5oVaWv1kQdi -2waMico7GOlHUS5Xe7ztsFUlnZsvcMVYbLG5D/czWNyWxPNR1DIcsOBf1o/q8k3r -CC0mFBr8eA48bz/M5TLy+DfNf1e6vJ1ZMIrvQN6MBAqlM+aQB31ZOlOMxb4VPu00 -yz93shcCZPaNXLcY09MyrcS0b/rLPHhEf+5aCzg/s76t2P+VwhxneViC7/XA38+O -F0tFDEXRPzNkUJxZkwzwystSGNk8aUCxL+LcbWPYf3nUQ8HDz2lNe4GPGW6F2Uet -XfdMziAxiEwK/Aw/8a7Houe5cv1vXGyCnzY9EHx9E+hEj+ZVt30vRkcI2HY6Hjmf -LitHfqcxxlFU5BP/R3/WZeDX58L7xb7xd1/1NlXgZMcPmYHBIqYSPrlI8Wl7J+5X -jbgHpjwedCxJ0A+W1iiE9aiHlSEFjh8z+YPrRsQrH4P2M6FG/DL2Unvp9hSw6kmC -lD8upmX9FN/LXoe3iH6yE+srseSM1wIDAQABAoICAESmzQdPc9A4srdp66inIRNl -O91N5AtrYh4vzOrefqbJw9ER6Yqmmhmca4lbzPYWBtAw/RxMMzzdkrDhB3NsqBYx -Dl+crKT6XF6g4sN6lpLNMpP2bifVn31kZezq2B9T9JR3bpUZm7jHCyHsxAH8cVSk -pymLgEqJ+Dz6RW0YULsDxd3VG84DFWiQcfnzr9SsmMklDUsjOAC5BONKBzMid4/0 -o1k7zhe19mOKnU3uh6bczbjDcYdbVUxQe5szoXGGz0EjVKyoSlzCtw1Uy9m2Ffpy -Wzh8nDHbkvUBXK4t2skVX2BDX3Fu3kLn4rJcVMQCbTYpA2tGDGl3AIcb2VJZG1QX -lWlrdnWIxsn69Yzqp9f4XE7PByJRcjLVRTROjzh8Zq2ygthAV3t9LOLUb34HehSz -56fENAIihwUZXHJoJVIcN4/1fn64TNxzeXGHc3CzFeeV/NhZb2ERBXT2SO5p3m65 -jX+vuqxsBphR5eHhXATIZnEOUwHGk4FcFu/Zv907ssxF5RqGNG2UZv8MNc6O8dmd -/uRHuCaNg0LpESS5Y6yu3lLfuVjxmZd5k364gf2U9bxedKMED8LLyzLWMAVLJ0wP -P4GuJlAKldoyhOK1R1QkIS84fWn+1YZbtorEvVtauN6fJeZzxjwEGE4TCTN89tjp -7Q6HvXzo+QUvYBzThiM9AoIBAQDtM8AkkLKEl4w/8pz1kkYXID8u9A7u9ltzfqEX -gn+StuhhOb50EkMP5LkguvHS9u4tHGl/pAfaaugdR2/m0658YZqSxFuMRQk9JUYl -fkL2zl5kQh1xC3dRhB7Wv/LGh701hFKnn54qpkzBbeMTQEkLNxkmeCEBVcJTwl83 -TnDnIchvC97CbHO5z7cnkF3glSVSVW4+EO4Qywly4n7A8yU9RFG/d20bnc/OwCah -Qxt2rHzesrJF2T5tQyqIpZNTMdyLQ7qwXNSCiLWjyF8jbah/czqE1E6vLsxi+JCA -MLc/5JEWERFJA+ck25m5EjWy7dkyAR5zBZbkVavgEHy3HR3DAoIBAQDQin5bjETG -HMC/N7hc7eijRs02qFvqHI/6nGszXPt+q4kTOMFnSFFcJEnshx4dlZ+MaPJoN6vD -6Vp6TA0F8UcgTTCxIpGrjfdTG+UwmOU8bZXjfJ0C4TDAaZhOGe2qSZRKM7ilZ8mH -+tZtEIj8qeFfOsWzeWYVdYFem4xLaJzHE8/O2s311gP5VSCeDhv9wCfoajz1h69G -AGYxCkHs7WEYdkuAHr+2/ln2CUgVIsI/Mq3zeNqkH2W6EcvoPaE33y81L+ccT+O2 -Xzm0E7gqyT5KMpS73MyKqMdUWIgdrXVolMRe5Uu8hY6RaqdD3JeEOriZdz2CklqI -bAtqQcaxb39dAoIBAQDNddkGEyv+e8KpFX5tnBEIMQ9pnU9eH5ihVts0oP3d2H86 -0CivI65hEu5kcmnLIca23nhnbm66+4Uo71r3Zv4pkOWyIVFnE9ln1GSJT2+zlpfe -YgJci+EDg9yuiT9CRgtXUtHBjhoTuU6c8ZESeMmMZGJtlvqHzH/xy1hscI6NNg8G -WRLP997Z4Rk1tPAL+0SDp6pBKqr7ctMxZw0CFtp30ji5Dzo8BJse7APKBXXBerEC -uWh/5igTvm0WRc1x76uoGbeY0h/lHYOwZXw7FN4x316d9cReK7wwMjwAPbJNLq+Z -nJkLVM0fePQTgblvzmQm0x9L5FiKWcbx43YTqq2/AoIBAGgyGopRXneCHEguZf6A -nHEgSh6aJjqmBMZZ6zIhrkMPIBErlW6ucoBQXWNB66kqfipgsWWjEz5y1NMwNn2W -nUwlZUPMGu6Vo1KyKGSMsFTuCCs+sxCiRYKOABxd1iL0WpRsnUB/x/EaQpMB5QAh -PUsgiXFwIbJtILONkp6G6XaKjpUzPB8tzJSClSaDtDaoEQiONGEVZ1zZjDXKg0AH -JzFawcSFGJMtKPPGlW6CDH4ZK1LwOoYwbdp3U+N5D5lj+SlhAt9hh0wy7hjslWND -Y80c0rW+z7AkPlwFVBVH+tbDJg6oEJxkOuSrXmGI0WevE1cSffhEedmZ6mYv8T/s -jr0CggEBAI9xrRumnEM2Xc5Ui58xnbI7/29DUx4bZRQgGsplXGzaS7puI7YKBcSI -sq3wW/jhFzDJZVZLa5RUOyLJJbLkSiFshD2gdFmlwbn3mEwDeUyInCOcTpWQjCTJ -4CS5Jq0aUOMwYxgnXKFgQOZoXAF5BfeKrBKwYYVz9BM+vi7ytKyZLjzuE4P0ov1Y -It2xCffK5EvpegjeTfv45UguDNOKn9yEnofv20v20q7MvPsLhPOEHJoLjlH+Vwu7 -ITmkZ+Cww5qLuRuHO2/y0VcRGxqcpk578QlNssO9dLkq0Bs8ohvHdFBna/PfCC2a -JLQFvYyizF5x7A50l4gv6PKcsyHFEyQ= +MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQCx86fkkU/BLazA +WDZYMCFBjGWl6JkD8LIzqY7d0VLHRNywkNeay7sgcGdEmyzAzmswgwO16/gFxw2M +IPzwiYXtjeDU834J1qm5CBsGC4crPeV8avqvpFewrJP9uyUpNa7Ctza7vyLo/zVo +AZAEAvJ9dKn2yUlx3F5vcA1cvy/Lew18MZF+9C06BlrG+kFulZIlnv0DlkGEzFZ3 +Ko67S0wk56hiJNLOmSIn4Dy32mCcbcCvi3SKCYqEmwLlXx5lU3tdL46JzmKieein +6jp/xu3SH9ZfYDixQQHuI1H90UtcJEDHZPqBBi2xp0oYFvgUDHwePEZK0OPOoe7J +c8AtvYwSS0QESNjnPafP8wQ1aYdKcZI1RpGvHXyJGW5kw/d9of9YHzIEJRdtBvx+ +RRdY5N+2w3LHM57NHNuEtRdriW8zpR4xS4aph9SGlQlrGGsHwbPZITKHVlUuvlgL +//3KlhNyppidhHbQJszAicwEFBMqzGVpqxawc6QAsjKtsxRLciF7Al6YKnluKHcS +CGZQpYm6NRRq35ajfoV5wIUcgo+1iQUNGT6xHUTEaQWIWHZsoDyfpVn7IBzrTZ7I +UriU1xiU7bLiVa+CKHqurSn1hqH0buL70+nCC2Y26fVWscX1S0u207I1kXPucR6f +im3I0n8NnCR0DfOYZVk6THvYz4YLfwIDAQABAoICABZRI14j5yen7cFVjsMyjgkl +bV4INKBs4DxgaF1jMglxlmfCUnfEUxx3XEwbVdp8SK8VzzJSfJFk7EsFnBMifBxV +rbunKchcFn7xCEqSyYnfwlb/J589cg3jJtAsVzW62MbsqT2Uc/FaiD0Z7RDDuduH +9QTRK5fO9jzthY97HqhbL07C/Kc6Qi3DvEC2A9y1f1WegcagYmkgIzvgp3PPtqXu +M4zTZ2inhcQQeCzqgzE7Bm49hAkHt0p4Ej3n1u0IMjF2lF6t9mq/9TCRzHJX5V1z +xrPBYnrAV1ihL1gwlk3g8grPnCbwOmzMORuaTdRd2HcGQh6B4f/5CPRUwuY5nkY7 +UMcX4rbRCcBDzG8s/NllTRaVC6yvEJPN4B/zOek6DI+tRy4tRj+BZO/1771bindD +nsYAklxbbpTiS7B073b+as3RFZW0xvmoqLyhzRQNW6GqSGj4C0KVigkWQ7Br69b7 +O2++oEurmLqI5Hm3RsJeJJ9obG4vKhPUPdtStF/W0W2TO2i0gpTOm+NeO5uYBRB1 +6OvhJH9dzMi+a0ekCpdQeD2xG4NLzwSe62/Ozz9Oi0rpAqHHhu+SvF+WEepLbkyO +2zx/OYpFK47idBRCAlHLC/9UyXpvw2yU9sleElVuHM41CzMe8Pwj3Qk0YdiHHDzS +Y19XEVHh/riXUufsAHVhAoIBAQDpUe+UJLgAIT6boz+MPLrPIGLeazCySEHEsdnp +jGuAx0M59PIDn5OeXQpMqCIRFDw6zhA/4gji0UegFRPIGA3Qduq+GsjVYRt6SHLC +N/rBRi2xg77yyKOMxv/+nwKFh/3TKIQbUc9EQj63WGBGCHu/EyMV7i9V0j8e7D2u +v/Z23nV/+XqawJXi4u2bHB3M/upOoclKfb3ewZBVouajzZc92kNC8XYfPn10Eofu +Pz7jxDX10IJNmzIYOozO9mlBsds7nFIBXS5wMz3iVg3GEUB05pPEKoEtZGrw474u +0M+gW9d7PV3qYdFgjSogiQf4JrSrOwXJQL/26nyfRX9QVplxAoIBAQDDP+fFT7Zl +eFLvouZu73lr++bV1+LogHgX+GHCSIujBELPyFEAyAeElFKFqnJ/NEOuPLG9X7tL +PDhh9NUijcBTPhVvwbH2/CRBdSX7Yf6RHh5fY+2Ik3hTF81L4bQcf0fgyX4roJY9 +YqpjQWFYGmSk4niCqWd+re/ZrYx/zpF+qgN21v37BAZNOCI+KidFScrq29p8kpLj +MdBWa0m7bhJcv4MPO46s2EJZVdczBU7iK86v5NVrGz7fPVm+tGxEDpwhyfYiu961 +U05XzT+trAaBa88KlSKdmUFq3qDdC4bFb6D+Ra4g+mhqNGtfBYXsavnENZgt0N99 +9M/dgaAEa/vvAoIBAQCm4YcFo9nDpgOc2H/Mc2d+XIC661oyOkJoSHk/dcMyodNw +scUkWFACdjO2ro9nPdzyho7S0n5elSew1UKH3MSMtXGjNY8uJ726kfUa+2UsNmDa +VgwOpPlt6KwTV3I7RhCDprgOvk4MWYF4LAr4LHsuKKbwuaM7tByXpotb4UuMrALI +3Q0XgOX0GTGvvsWF6VJ3mXpbAGL839+3kMN8p8EkaWewivpc0Jp0mgiFnNEDokSi +JFf+4CFNeRtbsJ2KcocHNQDmntpnQA9kQv6pC4/ZzU4lge1RJUDkOVC/NXU8ElSm +fjcdPIfAklduW/TKRgz1aEr0Lo7fMcqfNNsiAD7RAoIBAQCaeplZ13OsXMLhrrU6 +2GXtNeSxFJoG8n4SGQbfvJ4eYGSsGQVd5OVt1BxmfTERy7wwwvytpGx/XioN9rQb +HqQoOFqljU7M5zmYQKPIfQP4tSe6uUlaYbM1qwNXIkBqu5mXFFSrF+dGsiW1Wik2 +l8tBWZ2XY4jrBZtbUqBzDnC3ErSi9f8E92408lDFdnyTqYrOvxviq+VjtCnt9fzk +OnZ0w2FiT/DWeFQmcnBNgcmj0J07NYZVs7zOy6+R3xY50oVdhnkjihjuxfaaKV5U +fmK3SyEIcm5s2rCTaYlE2rXKyENMar0WgojSXp8FE02efBUZVH4O4c+xzFwaGVEN +rpIpAoIBAQDnAmhVy85n9oX6Bi+bXg9aZFa/teiCX33lCjNvNeJ5GljGqO0X6423 +6FVg91BYvnCbHQ0frqyKimVxNO/XYxCnid+Or48Cb9LwD31Wyip3UZABljg5sTb0 +fiNK0pxx6G8x1MsX0u97LogGwd5bMuff2lMi4xpinkz6ydB+fYsmM0UXGKsNkB/d +zR1izlqm87TMeQVi+pVZuOLmg/+hXVgISI2M7TlSoytODmpnSg5SZbaT7ut1IIwm +hJdWMTPHLv0X7NwsvhV4Knu27BJBw4470+okrOLfDuZ9+LZ6JHvYg22MywjCJx9s +MJuwAZJiZb+dQc0uwMkAEPMEfOQ1BI1+ -----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client1-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client1-cert.pem index c8fd22c9e40..068855d96ea 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client1-cert.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client1-cert.pem @@ -1,30 +1,30 @@ -----BEGIN CERTIFICATE----- -MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyR5MA0GCSqGSIb3DQEBCwUAMFIx +MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyRtMA0GCSqGSIb3DQEBCwUAMFIx CzAJBgNVBAYTAlJVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl -cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTI0MDgxMjEwMzk0 -NFoXDTM0MDgxMDEwMzk0NFowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt +cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTI0MDYyNjEwMjUw +NFoXDTM0MDYyNDEwMjUwNFowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt U3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UE -AwwHY2xpZW50MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALygXybg -ZQVWMyk3lYQS3DNZ+1Cs4qEgCHREtEKR2MgcE71ShJeWNzZEtPjjG18pCCpc2g3+ -TDM6fHlSmCx1mH0QsiEXEZsq6zbQVzA5a8ts1tHNOQKBK3BxDKDS38MScYTOo8nz -27zvVIJGSOPiuZKzr/aZ72yUlbhDEUEuf90vruAnZz3CbcD7COQ8E6XdlJfkCBMR -8MGXd4onHyWcfWU89dRvV/bfw4FU66MQkQlHa/z5DTQIH3Tst9Sdj8lp+w6sS8nP -p53yJbAOJ9hq4zUNyUjuNoeO+KFnJ2yETIHVGFPUuXqpMPzSk7AofEwxRHc1xtYa -tYM7KXKKZ2L50PMxRHEMbp/xgaSB+xOW/725uB6EqlUNQZZiuyhsk6c5G8alTc1k -nYhoNIleyfn+tmG3jLPgV+gIZnIorVtoZKh8bio3KAuqusONnWFHyhLhr4Y9tEMR -1lNK4vGCdoqVEPRB+Jm4oY7UQ75Dz1d3ivKktGzA4oSNwxfc3tBccKVV6j7qqPmq -IcqkWBZ4M8/6ulAvUB+9zEQ/HzajWZCW+PjDXHTSnSVybelzFv+ixBAFt+IGFGNW -SJRmu4zWOj8ZBMYJ834Ft7/kqfiTmGQzuZ4kQg+4qM5PQpkDnA+EwjrJ/OYeKbIc -9kB4WY02tKTiziW1hMr0Yk2gdWewmyo29bplAgMBAAEwDQYJKoZIhvcNAQELBQAD -ggIBAK/dfx3lQt/ySYQ3POrAxZETvSk+Z7Fw2G8nJ3CNPtrA2nWJFvOS11iUjR3n -nsF6e7CfApHNE91CE8x+eCm7nukkuJTq8Vt0MszMQXyEIOKuVzTMYcIm7o3MtiHy -lGVkueOfYTV0d14aZyY9SKSVodtP3aJDzTTdcj4r/zVeSqWAD3qwfTxHTWoWEBLG -7/6yV5fWVN0ntzlf3EwADaV4qtazXmn1/a7HokJ0PlWRkB5LqqhU6c1dSAYvQIdG -fmZYZXAQKuhfBaOk4mLM8uNc3WzbdFeOMhVa4JpYdqVjboD5r+dWXyIrZPkHa+oQ -rZAe9GTBrZjZ7dfi7g/kdDCsLiRC4it3+cH2M1wwOcO/j8ZGvHil0bXDD4ZeRNzi -rgj7/U4mtaUn+LPZaAn5Re9fMTXE6VWADMim2aQnYz+OULlLu1O+tAcwGteUpuuJ -IESit3T2DTlC7ohHhDo9aq/LblsIZ1fPsITI83RIovYArdBqz52ULPYkkXf3Gsu7 -bhSU/XoEu7245AkOWr+dqcGcspU7bS7bBErXTI+G2uwNxY6oSBhhZNeSvIdvWPlQ -wIpUUZFd3rqTHQn8b7KHXoMbpZoTXBWrKxMgITSHWsxyK6htbGpqN/nlGf/U1tcF -+Ww7lXzJSA0HGJvgJcSwBSs7vaLBkAZpBW2rzoWGip64zrCL +AwwHY2xpZW50MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALaJgDgT +PPqKnBNHrxu/jVwMpfPa9ENvaHbtSi88jQrvu2O+nRV/EoOBb/ol3pQn7RveMyj0 +PV6Io3KzmkMQRcX2Z5H0BNPAM0WijXpzfxWSdS9yZ/BKbpyq3QfHPX9FTAcnnn0S +gBgKUDfSxnQt4tn32kNv09q8nk0H1JXzjxILwKAhllHweJlyVBt1a6AG/yNe9K+9 +atyhYgwuQHv7z0OdbF8NfBWhWODLPkZ4Lj4LktVakYSi5XS5sdTtYXwi8mEDkucZ +1CrlZswIzR44nlmkJVoZssMbGphLzxWw/XiDn2ZtO4QNU1cJYU/WbJiMY+SUjIyr +AsLf2GAbU5bUiluG+XZsYKUjvrzVV6gYHWncXPDVWvjz96PDRsc6M19rsrhMT9N1 +tMgydCPqpxJkweC8/IRt7GZhTlxQNL3ioFsjksPmvm112012XYHd+NiuVIZucY6P +c0dFRWi0VKZUjvLVRbtxWMlSawTk0S/C6sbL2r556GwxJTwkm+EIuK6nGDKg7Kmw +yLFlSyqtXkvUOnnAnIOzEH3VdjUyUniUbfFT4ODs6TLzIkFSSJDN7W4klP6p1Ot1 +ZUkB030FYpFt1r39AfWLPWLjwzKvMWenWaTSpZIRO3h8sXbh6gt7zVZKNMrf8AFJ +uyOnfYaQpUwrxvWvuJdWZETS7lFgoRrJxGDvAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggIBAJ8XAILsGKVXyCBMazuq3k2rup8kkNU/hVg1RuYGmy4bNClNN6havzu/S1SO +/g00+tlXGBlSiPlRXq/p/oZgOI/efqIWSBfcmHuR9STdELfa7zugdFpscgaTOhTq +Ko5o1B81pZKw6wzVfn2JlGxnEy9e+lCC7ptMdFiBqc7SGrapkjCjLReszn1Jctk/ +9lsSvhWZ/7GhvRO/L93X3/ZM51K7VZxEwFnibULApObDZQBFipYdfKlrrVrqtdj2 +M7Plx2Hh+Ivt16Kj/DqRcRLcWVlaM8rp4QAtjn4bDYArFEGGi8ElWFRNjs5ztE12 +f0Iu+yqGmvDn0lHEocNf8fgxHIN1uJ2sYImS11Yn7xHp5FPb7efvYh8Ih6voCaTg +NojHi61q26YIU112A1ylStV4xMKgxt2rqRvmc6UTnWDtzNO9jp3NscQVHtUEJpv2 +Jd+JsDf1c/w42KTwTyOAz5j+D0acRmw1YRsv2BpO5tcly8nvdMX9k7JesdiQL9bx +ik863yOLG0AOMdWtZPVG1BAuiUUlbBS2RRUp3qsP4OuJ+eVKlXFieX+2NuzqyddV +CywaA+R05nutX5R34h3Cm2MmQOERAk9FUeHFX7cZMAXQRcmoBZKtUfKOGUKF0zOT +ZEs7xmHSqAOTx8ufDU26pnbyCxOBYwn1DVX9nYSskMGMSfGU -----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client1-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client1-key.pem index b4186a06ba4..8d9b887b033 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client1-key.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client1-key.pem @@ -1,52 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC8oF8m4GUFVjMp -N5WEEtwzWftQrOKhIAh0RLRCkdjIHBO9UoSXljc2RLT44xtfKQgqXNoN/kwzOnx5 -UpgsdZh9ELIhFxGbKus20FcwOWvLbNbRzTkCgStwcQyg0t/DEnGEzqPJ89u871SC -Rkjj4rmSs6/2me9slJW4QxFBLn/dL67gJ2c9wm3A+wjkPBOl3ZSX5AgTEfDBl3eK -Jx8lnH1lPPXUb1f238OBVOujEJEJR2v8+Q00CB907LfUnY/JafsOrEvJz6ed8iWw -DifYauM1DclI7jaHjvihZydshEyB1RhT1Ll6qTD80pOwKHxMMUR3NcbWGrWDOyly -imdi+dDzMURxDG6f8YGkgfsTlv+9ubgehKpVDUGWYrsobJOnORvGpU3NZJ2IaDSJ -Xsn5/rZht4yz4FfoCGZyKK1baGSofG4qNygLqrrDjZ1hR8oS4a+GPbRDEdZTSuLx -gnaKlRD0QfiZuKGO1EO+Q89Xd4rypLRswOKEjcMX3N7QXHClVeo+6qj5qiHKpFgW -eDPP+rpQL1AfvcxEPx82o1mQlvj4w1x00p0lcm3pcxb/osQQBbfiBhRjVkiUZruM -1jo/GQTGCfN+Bbe/5Kn4k5hkM7meJEIPuKjOT0KZA5wPhMI6yfzmHimyHPZAeFmN -NrSk4s4ltYTK9GJNoHVnsJsqNvW6ZQIDAQABAoICAB+qwt65fGw9cQBh8wmFyw8Z -862zIKcRRCoU4LhuWNkbkZrOsDDsyJ8lQMaqBg3fSx4lkXwY7b0J/By4J5ELOwsA -Ew+oGk4VEgEVUBXqBfVUc+osDH6lpd/OMuMLFALhZUp5EJlWhdT9zw/B8fnENghx -f0CkzkPJ3Crfo5VwU4oUN8UtCFGfgyKLYo5Cob/OZ+RqJYriD6vInIqGj92WHJ0y -80hzWu642U8srRSmstq+cVw6iaNrG7DbDu+0dg/H6Zyog17BrG/BV99rOPhF1Zgy -FNFYILJ4Z7hdI4q92URQvLFSPMNhhTZkueOwoFGAUfC7iAKo4EFEXOx8XOgKbbAk -gumbWjDMuLcl5NINpnh1yZz5vvDph7rkBjGBpja4FDMyxbCMjsLoTXtM0jCr/59r -hhriPavGZTQ5YhaF8dv/WaoOjXD0PDo8ujS925CoUJteuS/gNHe28chy9rI+dFVR -5XpO7kVQsoAb/54NdeLd0AKD/dQHfStMcxMucRRtZ9muYsXvIsXaSw9HnoG9RAyF -+VuHIUEs76+EAwYrKroUqzPwe6FD/LjmoQCiD6lVudpJQkY507eoCkWMrERXgGlF -y33YFVUmQ8+Uh1Tfq6XWGNGCxU3e1b36s557Lixbc4n45I6Zhso5lBTokch7dPHb -Gj1Tlx8fCSKmV8Gekqc/AoIBAQDlUEBcKtGK7l08jQyTMLfz8XMazbSuguVkIyU2 -aSW/qEl17F8BV2BgRnqCneIZER490RtooOpnAbTZ2AMw9Xx2ZCarCbYv/2ipp+fq -TwzCm96LipLXQCeu/Khwsez+04J3xwg3ziRGCOCwDT3aI1yvuTA4+DN5qa7gJc5I -qTyzQ7FkPcq0JOVhpqqNgDf2bHxteGUQz4Dnh6KVJ5WktVRS/x1fN7l/InUz4pWR -UmRGEmHcY4m5OS0mTDioRq6IoMDktjX7b/hicFC0lKBNGsYKW5FZgMZawseF6B9M -J4UBS1kuCacktumDROhxFvKym5n+qF8Yxe8D7usedg05pKrPAoIBAQDSk/cGkYoL -AYXG0h0Pu5GCw52MpgmQF4FKU82nUnWCFdHJQDKklBplD3e7IfjgjgVz51omPhq5 -jqcqvQJP+PaM1dPAP83Qg8IVJccrhKr8Uhm+yLsM/G6/zR/Jh+fvOXuOimYQptSB -LmMV9R/WS4GJAAWbrEkFRfXnV8U7hnjsQyb92fy3i32FZyB9wbhvA2830rEW/7X+ -bEFnqFliamxj3QfiKMH3pWKmBps1HiejgwScMzrAshD+SYdegNIsBU25M9fduBj0 -P4vHijfpikCW7KI8g/Ua7zomNwvPMAcrqf2K12D50g7sppPLeZVyiNxVupeU0iK9 -mwz7EllTckSLAoIBAQCrYXL4bzf0MXmVS4VebEbPFmEEY0DqPAHc+TdSlvsTZl6g -t2u5wOfSjFn7TKwyN8z2PlXRuJH3MhtLxFl+ObovcWJE6hkzfhtm0IpPX40V1RBa -EdL09Im0CjiJITESslkL08CclIAfp3rlmL/lTBB1dF5J9ZXjE6IS+65UN8eZ4rmt -RVE9mWWlub41yuIIVcQYvKYsDik04PXtq4v3cV34OZpmE2ubTNFzWaCW35D5nbtt -zyabO9Q2jzi46zp9+kavqE5oAKMOTwrCK2IyvhPMoerbjZaxNmmP9o+Nhuusr/eS -pLw6gEm9Rb7C5ne0bP6GiKeqCCQ+OE2YknoH3r2ZAoIBABMZSHZYDOUipjd6Jxie -MnxhsZBoHsXq29cZ8rUSTYYW5IgUQoOayyoPVRSr6B61wtjYStzdFR9vSMofim4m -bnYnjqUJJGmdCaVRX9camUd5Ssx6+DwdSQ4sVI26vdZ8qDAtbURlXTC6PFJpgRG2 -l3ENh3SvE5Be4AazdzgG4kDLdmXft4QOKrxGtjOxtkCbVyMBAP7NAnyo5OW55yF9 -9Ya5LVXdzHnhjijwwHL44S97WpewLcv8Wn41vQldNbK4aGw7v4Bkih7dBJh8eMW9 -vE3z+GL0tO5UdDmzFGcePAhByrUlA66k1qWwFgeGZ3+T2igMXGMrC+uWfH7pEyD2 -ZDECggEAT0o1NvWbsnms65w1oF8/2RFsvlTVeb0tpS9D3ijzCoeRv72fThiZ/diF -WtQPEW7WdYPWLG5UMOggake/GS4zF4+TBDNqZDcdVgu0esk738CUvLHzBl7n1Los -7CIOH0dT2Qook3XoOe2lLUD1arfqrpj/ygUW0MoHo9ihSPOact1jusNNyOlMAcma -OzSY6jJnRDE7vJr9h4+1fDJYd4PbSq83KNHJyVayaSk6RBxijPEjmkBZ6bWNdDt7 -/kihyxT3+ZBGvuFcvnBGFgRtUV6ZGsz/74MGDvKnmIyvt+PuvZueMJIqMKytUva5 -CG/KQT7wu3i0IjmX91IiDXTzH8oa4w== +MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQC2iYA4Ezz6ipwT +R68bv41cDKXz2vRDb2h27UovPI0K77tjvp0VfxKDgW/6Jd6UJ+0b3jMo9D1eiKNy +s5pDEEXF9meR9ATTwDNFoo16c38VknUvcmfwSm6cqt0Hxz1/RUwHJ559EoAYClA3 +0sZ0LeLZ99pDb9PavJ5NB9SV848SC8CgIZZR8HiZclQbdWugBv8jXvSvvWrcoWIM +LkB7+89DnWxfDXwVoVjgyz5GeC4+C5LVWpGEouV0ubHU7WF8IvJhA5LnGdQq5WbM +CM0eOJ5ZpCVaGbLDGxqYS88VsP14g59mbTuEDVNXCWFP1myYjGPklIyMqwLC39hg +G1OW1Ipbhvl2bGClI7681VeoGB1p3Fzw1Vr48/ejw0bHOjNfa7K4TE/TdbTIMnQj +6qcSZMHgvPyEbexmYU5cUDS94qBbI5LD5r5tddtNdl2B3fjYrlSGbnGOj3NHRUVo +tFSmVI7y1UW7cVjJUmsE5NEvwurGy9q+eehsMSU8JJvhCLiupxgyoOypsMixZUsq +rV5L1Dp5wJyDsxB91XY1MlJ4lG3xU+Dg7Oky8yJBUkiQze1uJJT+qdTrdWVJAdN9 +BWKRbda9/QH1iz1i48MyrzFnp1mk0qWSETt4fLF24eoLe81WSjTK3/ABSbsjp32G +kKVMK8b1r7iXVmRE0u5RYKEaycRg7wIDAQABAoICACgRktW8U1xj5NLOn3+l0q/s +DtmyrH/JCtNgTzKDRiqqaSYCB5VaaYP4e84bVfqLsR627eAFjRsdP1PEXQ5vmgFU +j3OYbx7UR+z3O7svcywXFCYwJOS4UgON9iro73Tqjz/a0I1/7CJa0TUPzYRfNjbG +k2DOQWD4mn8qQt4Pss4xSj1cYhTmhnKYiCHm6pMcNhFbnLafC8AWpOErnfgZVGvx +OIK9AQn2ev4NX0Q0yWHRRJAU63CEGX4/7OtimE2Zlj75e9vC7bHk3WXYYL5Li2b+ +Azz9+yGc53+a1IBcc6dqrSjcvX3FNxAZ/QR7eycZWiwo95lBSL/iRysBlJ29VglW +YScc2Il/xWNp97PORwsJEDpeWq5UYdinARFK6PAGjnxmADZNAeZHP+r1C5CQaw72 +y31aIrhL2s9wRPZ2DytIlkSmvffIoNpZJW2AyVdJn8L37Aot0Hwr0SsU8/zibvZ7 +4d+7/6rnPnE1jTZlpnDgyH5e5Mtn3YUYDlPAEQudfYyvh0QrNfSOMnetWSYTh/Oi +40iQM2vaKDiK91deTR50g90A88eSgxWMGG6WUzqNoE5CwwiNQxHPhrmFi4H1V6y2 +uaF3s0Gx6aF6j+ws1ImbgrkpAbvgTCENoDtmS8MYbZjXgTzwnG4UtIwqdc5bK2B5 +i9mdb5w1v2v6XLUxVvKhAoIBAQDhVgmw/ssCscde91dWLMylm5Wf+Q7Ry32eYSr0 +UXqYUbChHkYNK5HpVY5b6Br7C13FrhboDk0hsz3agOFexcrua1N2huyJ8gGjlAzz +i+286WuwbhsX9rYgqTvAZmQYpklAfWLZH8nlwtt3iafNhgSVaa//A2m4hhZagElT +pctVakSyG3OYaNDTXBDOnZi9xagc3eWmxkS8PWFaYC0DJCw9yf+9ynH6+FRZg75x +t7nnDd/eSxtW9QUALUCheOO+yIp/uJUiIyWR69cfojQ2vNx5t8FzpK6EqHFCujhq +e+kJB81BAc2P59O8oGqw9fvc9pzCQXyFbx7dtl/Xu/JyMEqnAoIBAQDPYH0afED6 +qyvtQ1le6bjLW3jGCaymflgGwc0sm/pm/3XY4WXhrmqeSIF3tbhP2HnANGinW0wP +nFd0vAi8RU9UxB7iIUAZ6wXRS8/YQmv5zIouPzSCpmvW0l0IcnqjDUS0IZPRo+UA +FTzS2KIQ/yOsHSZoVNQe/Tsdk7Z8XVAJlq+YZ7o7pGR25yGPleUUbVwbIhoEiBPq +EFA+4BErok4CFQB9J8jLRdzmTEQFjQ/w4w066ZkplcIy009a4+LXIvL+MCPG3qMD ++2K/HlTYfMd+CyozaF0ZGTECtakrK+PWbbTj+VV30SD9Sckk8ZqIFUq18Fb574gF +K2KSq5SkYSh5AoIBAQDdenJ2HEkvcctzJQsbsVbII58yKFsPi8IBjKHql7c2xXwl +MJtL0JpOzH/rB7yVKXvWk6ECHyRizkkqXeil/STTqHvVkRInF83SmO8N5mgaeRcW +x3Ir4JrsiUoodrtFmxN+pn8kx+DqytZprMxY7rPMo5+PuCwOaQTJmTP5Woj7gELb +CK5ajBNM2z3Nxwrc48yz6soRXOksV+w7JzK21rQBW2zZf4T+V1yYyyvBnALF/lYe +qJXLp3Jt1QykaSz4VSYEGUnDzuXbggHknspRTtopbJpg7ul1jBYeruhKiVXoQVnV +3k7MdeEgkk+rdWtDqMU1Daa1hB3Db8DOS3YmFB8bAoIBAQDPDD476F0UKTzlWf3r +9pzLZNuTlmsrnC+VJ4ALjvwWQ+7MiFapWfQXbrrc47FO/wqoLWtj1JJ/b5Ad+/MY +znajYmCXU61lczLOwcuV1tNph59bBz4NR82ZoVTDr1DkZMX4tyGYCPQF/i5JMYO2 +Rpa+LCiBuFhFTH3uTOHBD4Vu3WUaXE4jaEHqOWBXtMgQehOg/45MgfSoGHuWGy7p +itYp3AAt9T/UPD+OLA0qIaoNzxQRgtOqIlzPVA0B6U89jyZfRX8i+nx16FKyEL2T +nBmtrcYHp6Zz/aPiWa+6a8rB96zIhNOhmko+uaG7YgHw5pk+R+T/C/mZd7SmTetN +p7e5AoIBAQDXqOVl33+eRw3CxNCJZsfglrD/Jz8VuZv5MZwRolEB4IQwm/whXdnT +34y0UUpUQHnVepzAP3QjsLKANPUY2rKO+f8NAX4Uakzn43QLI+hZcxx6hVkV6OkJ +Hi9fwSEBZzx5DWEbxmYMIGlaRL1yVff8wevQci7WA4rrztb9D5B0c49ItCrMkLNs +X6+9Bh4zafL/FxJSkTahQLe+KGNXSGGGrYB9M31oLSKKM955ZTRnICPxuyA2hffx +8lmHZ/5hmP+eMKoAJ9khilX4LmnkdXJEZ2w5lQTPUTNP8ggaXvFWpijjUsaXEdkR +NMnXQHpKE2RaT22UJ6z3W+biQqNlhlVW -----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client1-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client1-req.pem index 37288c0a23f..d5cd522bc8f 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client1-req.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client1-req.pem @@ -1,27 +1,27 @@ -----BEGIN CERTIFICATE REQUEST----- MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp -ZW50MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALygXybgZQVWMyk3 -lYQS3DNZ+1Cs4qEgCHREtEKR2MgcE71ShJeWNzZEtPjjG18pCCpc2g3+TDM6fHlS -mCx1mH0QsiEXEZsq6zbQVzA5a8ts1tHNOQKBK3BxDKDS38MScYTOo8nz27zvVIJG -SOPiuZKzr/aZ72yUlbhDEUEuf90vruAnZz3CbcD7COQ8E6XdlJfkCBMR8MGXd4on -HyWcfWU89dRvV/bfw4FU66MQkQlHa/z5DTQIH3Tst9Sdj8lp+w6sS8nPp53yJbAO -J9hq4zUNyUjuNoeO+KFnJ2yETIHVGFPUuXqpMPzSk7AofEwxRHc1xtYatYM7KXKK -Z2L50PMxRHEMbp/xgaSB+xOW/725uB6EqlUNQZZiuyhsk6c5G8alTc1knYhoNIle -yfn+tmG3jLPgV+gIZnIorVtoZKh8bio3KAuqusONnWFHyhLhr4Y9tEMR1lNK4vGC -doqVEPRB+Jm4oY7UQ75Dz1d3ivKktGzA4oSNwxfc3tBccKVV6j7qqPmqIcqkWBZ4 -M8/6ulAvUB+9zEQ/HzajWZCW+PjDXHTSnSVybelzFv+ixBAFt+IGFGNWSJRmu4zW -Oj8ZBMYJ834Ft7/kqfiTmGQzuZ4kQg+4qM5PQpkDnA+EwjrJ/OYeKbIc9kB4WY02 -tKTiziW1hMr0Yk2gdWewmyo29bplAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA -BQWFyVKS6BQF+5jNHtyV1mozn39/+YVfHKN+QkNIF6EBBLE69swO4spVCyT0vkdg -5Dok6qomBf1bvqySyfvRNNNbFaUE3ySbUS0Egwtn9NonPZRS1oUNUM7O5f/hmRAL -wJI2nbsvc5XxeWPRpm7VKw9z3Rq9oFeZy6iMbQyRlYKDbUMcR+4x4o0p5GuFqXDD -NWJMAC1iilYiid3vE0Odjsy77yzLnTdzCuelxxOWNVQccF/Dv5vACpRkX7A6GHJS -Cc/hwuBzJxuz9K+oq4bsGDtuFOV4SyIbfW7OfYtVwfDNPjkPQmZbt4aSmpxtD8s4 -00qfYxCq7ZPU/yCnX3Qrz+VEj5vzVpzeeAKaHAeBFtJXzZdI1o+fUJVFmJ4pvNiP -Q4kJbkhTn+NuVQ2GycsGcg2rutcbNYVypP5wrujbWJll1DDB2O6AbCMgZvYR/uIP -SHB1kXvrY0rEGfbWoEQ66mhR+hrLQtcgkEwEJGQ0US1TcDi7T6imeSZQxvIqOuUW -Z4w72gaet20sSczKPKHIokJsEPgM+5dVCsWwZ92bJdADEVcf9jDMtUYRJw0wL8cm -3df2dY2mlioDm/Rj990jbdM4K453vJODZmuSfCX0yFh3Kxa6oAbe8GmuB5hiV9B6 -WxZvadyxvop6y1wUnHYv/q1ZfgufsJ3GaejPEm897bg= +ZW50MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALaJgDgTPPqKnBNH +rxu/jVwMpfPa9ENvaHbtSi88jQrvu2O+nRV/EoOBb/ol3pQn7RveMyj0PV6Io3Kz +mkMQRcX2Z5H0BNPAM0WijXpzfxWSdS9yZ/BKbpyq3QfHPX9FTAcnnn0SgBgKUDfS +xnQt4tn32kNv09q8nk0H1JXzjxILwKAhllHweJlyVBt1a6AG/yNe9K+9atyhYgwu +QHv7z0OdbF8NfBWhWODLPkZ4Lj4LktVakYSi5XS5sdTtYXwi8mEDkucZ1CrlZswI +zR44nlmkJVoZssMbGphLzxWw/XiDn2ZtO4QNU1cJYU/WbJiMY+SUjIyrAsLf2GAb +U5bUiluG+XZsYKUjvrzVV6gYHWncXPDVWvjz96PDRsc6M19rsrhMT9N1tMgydCPq +pxJkweC8/IRt7GZhTlxQNL3ioFsjksPmvm112012XYHd+NiuVIZucY6Pc0dFRWi0 +VKZUjvLVRbtxWMlSawTk0S/C6sbL2r556GwxJTwkm+EIuK6nGDKg7KmwyLFlSyqt +XkvUOnnAnIOzEH3VdjUyUniUbfFT4ODs6TLzIkFSSJDN7W4klP6p1Ot1ZUkB030F +YpFt1r39AfWLPWLjwzKvMWenWaTSpZIRO3h8sXbh6gt7zVZKNMrf8AFJuyOnfYaQ +pUwrxvWvuJdWZETS7lFgoRrJxGDvAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +L2KVSFlGtuYjmV6sTYF0GlA4V0RvUTbLM7qnd321957ReDR8iRqj9ZnqKDEHeH9g +jfuW+TV/BeQFjTTAXM5Gy+LRz23xOwqcvxKh0WYjGLhDXIjWx1zzzGBaAcyWMllq ++o6LAUadST3yhfncP0A79hPLxOsPgMGd3OXGajKskDNmU3MTTsbtQ065HsBwo07P +leMx2jDkapesUaNaTmXERg6MT86hsknGsbO/tU3dGNy4hBuOX5O6bEkijF6eESLd +U4Xc54yScVvxpWqTSEAz9xHjIOpOZNfW+enbLdpApxq6IZkeVM8z7yy8DNjTJ2SD +aS/xKexqrWjWFxNa/CtKezkaZgmLs9jGGan+hmlNBeuixvJEekPliv6Syj3wvLp/ +L3/PmLgBzZj6iRdw5fky0swCn1qwpgwYRjBSN+SL0E8yG6BGKFWByQfwWbdOu9DS +lN/CPBe73yi8kYY5gBvBmPsrt3VMVRbXBLNM16jO6lkyYzyC48jTdicqpLsHazZn +Z4I6GZoUQKc9WPzSdu6tEXjM6e/2lkT8kaPmrae3JOKnP+lzjZjfplV1NylICNQY +whPWBVGaRg0dy8dZSTGtzygTNMoHS3zYsBGE4MuGZtm/4+x/XLkz32n1k58wAKxJ +JKafNaOReYFxJKd+ML5XnYOVICuw3nxQY+CeVZlz1Bc= -----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client2-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client2-cert.pem index 5eea9eedccd..9d317f07963 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client2-cert.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client2-cert.pem @@ -1,30 +1,30 @@ -----BEGIN CERTIFICATE----- -MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyR6MA0GCSqGSIb3DQEBCwUAMFIx +MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyRuMA0GCSqGSIb3DQEBCwUAMFIx CzAJBgNVBAYTAlJVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl -cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTI0MDgxMjEwMzk0 -NFoXDTM0MDgxMDEwMzk0NFowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt +cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTI0MDYyNjEwMjUw +NFoXDTM0MDYyNDEwMjUwNFowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt U3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UE -AwwHY2xpZW50MjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALU3+Z07 -LANG4lgyVe9pjWQEHyAZn4iadESZqpkcVKFpeHXmEmCvDkt4f+xs/tJle2YmzmVI -OgTrhOeY+3Et9ydfkNM8N1LUiem6i0LQq4yhg8/xM9jeKbOzol1CNM9TbRzZXHKy -PX1Ykics3iRGE3wqwK9quQqYIvBnUT367N7w9kvsgvMr+E825BaQ2hmmefi61vXb -CRO7dyRPIt4z9ERMeAHY1Y4ahM1OwWcGNfhqPnMoscWrWarwIE78SoiUMXd3mVUE -27m/opmLbvk59qh8o9bu5sje+4iKW7P5LVEXWhVnbUptht786f6zTFzWEeG8XUF6 -TFQIAtO5P+QxVmKhsliEt/vCJdTWVt4TKT7c6T+S6ivFl9Fn3/KWkscKLITsILLC -+DKxBracohoROMfzYSGgDmE2d6jnQZ8F4U2SxtGelk1BVYTOLP2y3Rg/xlPB7FgH -iq9OJitIuUIN8yAtjEedQN47OpUt8fMMZHukcslTjXCDgzFpZVh/J+oIULBoRBah -NWsroT45rrDhum/n1W8jc8R7VfLVXhb/YK84unxxGduElbNwW3T0mtDMqHBdvwj2 -pfNmGj/kgtSSuD9kREnIuywyiQCLMM8N3UacMwPZadjFxeugBuKCXX0U4j/llnas -w31ljfEL9EMSuk1a1PpLyPM3VM9guOzX6FKXAgMBAAEwDQYJKoZIhvcNAQELBQAD -ggIBAC369EfL0SqOqGSleQfoQS88UDazzyysbTsNeqKqieSkK7Uf09dtXflg33mi -W+dEvEdcMbhouQT1nIi/nfeFktlhqtrFav5lUDQljVHbNteBQPMWgvMSGbuYlDoo -pxEmvCcof7C3IAzj9btEtMdo99V6Ecp6PfDAhDJWX9VQqvmKiwbpdVhl0Z32o9lV -XoQZvIYGMi15Ny/j6/8m9xeTql7fLq3o7xTg1fOVSJweT8MNYu8zYeohwDOIcLx1 -6mJLdx0DS7ok/7swMECdyd4zwCTgn7wDworS3MyPty5nVOIinlr51NXOKjbpNBTI -41z2NPfX793Qx43Er6zRBKhrRmF10DwiMgtrxJdF0khFjVuMczHZuOL3gVn573X/ -FTVHW/A5SyDmilMZRn+VYbnad1JAy/RqIb7e2EkkPzNMFMbUT8f++HQF6Uo5XALi -EsBiabf5KZ+uF9N9XN/7fOR+mPltc466etUqFCP+MjHRiLyisDYwcAay0V1Q350h -GEWWQj53akIVi3cEV73dW9co391xQ33EGRkqY4+kY6Uh7fKN+roRoyu62e8kKV60 -p3FjCtjDoOFSHknGITSofERfc7TMwBmhFBU1anooMZs2+zs352TAzk/qcnKfmSzV -PQoZSDuUzt+aaj+w4XisguJEq2Hdbt+41a8Sb4XA8+hzXUo8 +AwwHY2xpZW50MjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANUgBLgZ +EjiW3LmjGlCu17h2ERYc75YX+7esbw/iMWMrcrXfvNNutJ4H7hOsV81ZpMEQouHR +fog8InraKCuwmb28DLhJZJlpE49hZxeUJws4yN8VCCTgr/tLtTsvsSS+TGA58B29 +HHaglJwY0w2mOlOVcUxOkjne4VHokLOomhzzqcqTLCjwUslZqRn+SDgXyRw9P5re +J/m6E36dFyBeVglZvealVp4uK/TTqVVFJYBljD22M9wrOeo9AIvrru5VlgDNHu+r +wqgSeE/bGcwhX2J9J++lwOxsFDR7OILY33yoD7/6NN61SJEoRh7xZTQkr/Dc3Hpv +jYyj4YagdWKq2xDZzIxH+QfEMQssuFvulq5L76TJpXB3ceyCu2NEvb/563p1EvXQ +bp5Zjz7Ojo1elobs9epvfbCmiANmTxG8GLKteLXfjcph492gdIw0nsV9/bIzE5C7 +lnff4nEU9E/uEJz0FTw61VjcZPtpqEzLE/8abBU48pTj5HqKo8Nsjx9hPl6trO4h +81yMaqwbQDmza1KsU+CPIFiycyv8Hn4w6JEjwnUc08rOoQ3e7HjqLNpn8X6RirVQ +UrwSU7L8RTKeOCOBLg6AMXfH/frPRnNQjUG/Z7tBTjTJhm38qucxyHI3J5jwX6vn +/jBfdFHfT6510V0Q9fzgzp3H3fyHpnLW1qxDAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggIBAF9fs1tF/yL+eBTf7F/RMdDrb1q9/YZCZ6btJH6CnxCuj4M3o4EkTW2PPSY5 +AeTX0zYaNXvlHT54vjdO+9H3ocyY0HfjVSzttw7qiTkvsssRLLW0PMpc8QMRBpz4 +CmD8vfjY63hKzE2cF5GyP1RveCuFVf7//wM2dfPwrQkIOtKrctejYjn1tOAfgJtX +It+RWvJ8T9t4e3KxYgKSa6eyYxyNMZV67X91C3jIJLgTTLwXXGQF5G8hH3KsclSl +RDE3CAYoyDTtaMlI6A3qDtmvfFKzeltKZc8w7uIbjgHvF49p+n4oh1WwDc/C8SUy +1QAx6DSSW1f470Egtfp0hJKT9yJh7C+/EdeAq8Oh1vMxYKBrtjswCsrFQ+bayEcl +2SzMLez2S/bIFSF0WaDqqIOZDzcjpXjbFlm/px01qoPDk5lkTPGA18Zq8mVc0y2N +R3vYzvfpigjkjXgMcOIfP1Jnlrx1x/4+txR723hUkHQd38nKENepsoEoLrcpmbIl +VAKYTALTle6jJKGf6oZf1TIs09Bc1Qs8Oo4IymubOXD+FlUSmggVwMiST15O5vQu +zdvidRHhAE581DKK04GLmWn0UE0Ko4uaNHAgl2gzZsuJQ5oZynOxmh/z6t+mgA7L +l2qS1WOq29Cq2qWrrfvqbl21LWLrf2X75UyTd3GAlQ19aqLV -----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client2-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client2-key.pem index 26cdf77769f..ed0d179712c 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client2-key.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client2-key.pem @@ -1,52 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC1N/mdOywDRuJY -MlXvaY1kBB8gGZ+ImnREmaqZHFShaXh15hJgrw5LeH/sbP7SZXtmJs5lSDoE64Tn -mPtxLfcnX5DTPDdS1InpuotC0KuMoYPP8TPY3imzs6JdQjTPU20c2Vxysj19WJIn -LN4kRhN8KsCvarkKmCLwZ1E9+uze8PZL7ILzK/hPNuQWkNoZpnn4utb12wkTu3ck -TyLeM/RETHgB2NWOGoTNTsFnBjX4aj5zKLHFq1mq8CBO/EqIlDF3d5lVBNu5v6KZ -i275OfaofKPW7ubI3vuIiluz+S1RF1oVZ21KbYbe/On+s0xc1hHhvF1BekxUCALT -uT/kMVZiobJYhLf7wiXU1lbeEyk+3Ok/kuorxZfRZ9/ylpLHCiyE7CCywvgysQa2 -nKIaETjH82EhoA5hNneo50GfBeFNksbRnpZNQVWEziz9st0YP8ZTwexYB4qvTiYr -SLlCDfMgLYxHnUDeOzqVLfHzDGR7pHLJU41wg4MxaWVYfyfqCFCwaEQWoTVrK6E+ -Oa6w4bpv59VvI3PEe1Xy1V4W/2CvOLp8cRnbhJWzcFt09JrQzKhwXb8I9qXzZho/ -5ILUkrg/ZERJyLssMokAizDPDd1GnDMD2WnYxcXroAbigl19FOI/5ZZ2rMN9ZY3x -C/RDErpNWtT6S8jzN1TPYLjs1+hSlwIDAQABAoICACQBPBH2OLdp9Pyq+5H2ucXe -X8TD8oN0lJcwpMmI7HOpfeqGzK3y+fcauOfje/mXeh4Apc9Pu19Q3/YX2hVRmaJ0 -BVVCyIcnfUM38gVTalIloQfGl5OGCGkQzriSt+VdYPdzZD/RlA/8cgVgj75LQgHQ -iLoRjTRCp+Z10Jls6nUdEQJiul3QJaSvdj/ZhcvE3MUDckQkwbfu7iwDoRze27Ba -NRnA2CVEtZAJDroGbOCvUzsUIcXxVn50+SZYUAK01uMymv9eWL2eCpNRNbpEvvIY -2YCRtL7CNt2ZB8FJR6yt5BMOmpNIEZzqSR92xjxHuAPOpgSlZNbpRI6GZuBSwSO/ -04O/xGGCw7H3iOZdUurUd9+liVpNGjCk8vcgRDcacc7ibJTuXPnbNqcrpTU+Ohhw -PDgwOyxAhcpFREvofwHAVG0OlDngEb5AM6E/ocYB/nOPYvkgmVb39gOzoDHhEreG -KWzbs9w+gbt+UZHMnvjZe2LMOpovBZMFwReyuYjljeq8yt8MAi4uaB82I2XDGMhA -W3NcdF0BSNeVYQkxCF5Qrleogu84i+ldxwlPBr3Y2ve9kE59dditRZYY0ghlRi4b -p5ytq00cSog572tKiKOWTwR6HTDqKeS2xiqPtIcuRvI3F5ArautsyPJdZ/oF28UD -tY0iX6gg4Bxylkj3nmfxAoIBAQDljokxPggXH/4q1yAwVC7UHN7PsVjNUoU/oEzU -Kt6QiUog0TbFgY2km0xD+qSueK94IQrywdvBjpWU4GL2musPjGRx4g1BzIgBmLk1 -dQgCvaZNsZHL3K/c9CENb0U7iW5QSnf3l/Ua9sXHSFVXVxFaYZBJYB4JPjonZQZ4 -uAQUN7wgypdYFNzjmOEJ5lXesFyU5uGwr+Ek+3VDnQBqXD9OAVM2dS0hvPW71IG0 -kjoBdTvCKXBqVUQDcn0uv/K4wzP4mfd0RlFXKavfSrfXeijOyqccV7Niui7xHsOF -1ePN36Aaf35xy2PbD+7+oVBgeFLDQkewYFgkZAilTxPsFRE/AoIBAQDKF/uJgEHk -/2oyuhOaBhLHfekUrPZPfrENa24YiDP/fQXS+AwBM2a9kh6WneLT1Mj8wteyAqWq -qVwVmREvIG9iNvVnCAjJGkDESOZZmq1Ai0qVlUMB411z98raYltTLYCJ0Zab/gq3 -/F0c/kWk1kITF30w12rJEtmpiAxF6ptwMtJe1sonx0YEbwn1W/L/c0/B+wDRebgr -cABx/YN1+2LwoNCLEByX/Itra9s0PHnIX178RiWE0WFet9DlnMumCsMSr+sWOzHh -7SbabDDZyZCuE5cWpp83Yi/XcvoWDBzyqfwZTcIqCia4KAeM0OKT8fuLyQXZ7uqw -UYoGt/esFBCpAoIBAH8m1Y0mC/srKHD4WgcLCilJmI04cbTBKbeVhhjbag5rTVki -KZF68kckEjlxQvi1NpR20QaoJJ5w6R/erlJkeyFQpW6DRM6WiRtBSg13Nqm1/MD6 -Q4TFR9A+lXRIlvOkayskJJFLZWO7HtAEm+jV/HGwquhJyjlac4HQrqX9X50HRjhv -nycV30heLQB7ykKgM5fyzz48HZyLgGekNk+zqwj9KOd4Pjh63y5cRkAL4v9dvW0N -4Qu1EQhLyVU00zBOR6JDPlL0hCegmGgFjhFXw/TPQYMsfNuJv2ilnq2qTRWC8b7b -seE8RKJZkAsQtWHKJCWtt+HVa900X6mZRLK29KcCggEAS4jBNDhi6KNqXwFbeDMS -68ssxdSLz1SL+ncP1E18Gd7xwVOCsSQ0H2Cw3Byw/fnHEWoF0nvYzbu1NkFaweF+ -oBfoixTeq8OLN5IDvrPoU3p8awI55EfF0yVnhU5D0gfsxhY3E6DVAsVpS//hqeWH -FrmtygaJ3BOWlR3LcIpqhHoKVPY04r9BdUDHWRR/82h3BAq72I+E/X9EO1bEvuEx -rrx8la108n2q4oCSkqiT4Ad9ag8R0N2tZgiQJHIBT9Mv7BkiZdHEKCxKXFMZwxzq -XZXm38lxMvMY7I8F2B246B4OoqqEis/2ftg/Uit0tFU6bpm4/SamQzaGv1IdWQy6 -2QKCAQEA1Nr2xcoHH1XoSMI9gB3TNWXSEUvT4Rs1jFLabPNjJ2s4y3kEm+Ii1/Uv -KAyL12k7nb5uCpkWO//Seu3LcyUeiOIzUqlqcP1RtpqWsOz+5tJ9okN90x526FuH -vNsv7uzCQC3qXZ48u3P6bEZONc8QMx8a32pHNvvvPjjUHai07EoyMOH1WQc71m10 -3DTqHwTN1drOY/wkomzzCYojQ45KN84noLUJdLxUb8IBBwe3RipCJ/no25zBUdhV -tWMOVjg7f/7iOjygM8EAbR2MsgdmKiLjVUGCML1vtE78PC03OlFCKx6wOL9/pM9A -VXHl/tHRg27YjKQT1VZIyhcSLcxS2A== +MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQDVIAS4GRI4lty5 +oxpQrte4dhEWHO+WF/u3rG8P4jFjK3K137zTbrSeB+4TrFfNWaTBEKLh0X6IPCJ6 +2igrsJm9vAy4SWSZaROPYWcXlCcLOMjfFQgk4K/7S7U7L7EkvkxgOfAdvRx2oJSc +GNMNpjpTlXFMTpI53uFR6JCzqJoc86nKkywo8FLJWakZ/kg4F8kcPT+a3if5uhN+ +nRcgXlYJWb3mpVaeLiv006lVRSWAZYw9tjPcKznqPQCL667uVZYAzR7vq8KoEnhP +2xnMIV9ifSfvpcDsbBQ0eziC2N98qA+/+jTetUiRKEYe8WU0JK/w3Nx6b42Mo+GG +oHViqtsQ2cyMR/kHxDELLLhb7pauS++kyaVwd3HsgrtjRL2/+et6dRL10G6eWY8+ +zo6NXpaG7PXqb32wpogDZk8RvBiyrXi1343KYePdoHSMNJ7Fff2yMxOQu5Z33+Jx +FPRP7hCc9BU8OtVY3GT7aahMyxP/GmwVOPKU4+R6iqPDbI8fYT5erazuIfNcjGqs +G0A5s2tSrFPgjyBYsnMr/B5+MOiRI8J1HNPKzqEN3ux46izaZ/F+kYq1UFK8ElOy +/EUynjgjgS4OgDF3x/36z0ZzUI1Bv2e7QU40yYZt/KrnMchyNyeY8F+r5/4wX3RR +30+uddFdEPX84M6dx938h6Zy1tasQwIDAQABAoICAQDIuNYY+OvTRipt37IaCQF8 +Zh4jgG8ZIk9dJlaXVAYFi1cG+chiLSKIr5lHCArNiT8E4gE1wtNzxYcHw00QEMxL +CL/GFMFdRrw4TpkEePDovbtZdvprmP3FJAF00673lw5hlk+SApi7FPPBrBOiCEto +ixfgsSNAw6vcM7eMrR8wY0AnXMK7b9PYdMwxge5MfgJXyUuNNOvbY6eWmKa+Qnqv +ZcjXYCKa6YtWkr4pY+005u7U9DQViNSLypYoMXlYWFzlNkqLmW3EU1jihMzgFxI5 +tPwW1TpEsGm7H84SVeTuB26F9UUz9vJ4W8DmxZz2JhNaOvifi056BaKS466KlbWo +iZgt57ajj0VmYxB0ZL7QgQqb2xDZL12kU1AU09QAXJnpy/RqvV2HloKbqrOd5h4L +oME6j8vT6Q8o1vsh2zJuLXHAsMr30XK8x1HhLDDzln49gq/d3GrZkNrPDjcumiwI +o6PYR91Q4QI11kdqR/3005wV50g847uURFNF6J4ziDeDGsHqj2phmOIt6d8vWcFo +XBEovCZkXQUSx+4NgAAy1GRBjK6tLRRQnS9bGgkELS8+Jx84NlgKkH3m6+lKNJQ1 +o5SpUqmk1dYnpTv99U2+5qvA/o9SVy56wlfuo+u0GlbMjs3OmItXErg46UBPhd4d +HipFZrBItpw0DYAF+voLQQKCAQEA9ePjYyy53VGLq2+vRx02IZOMQOLaaBDfybtP +51ksHfCPg+XZ4jWsDH4EPn5DeUmjZ2nG8GYSuv8dAQ4v9kArToLDyy/qhXHzaING +uSd6KlTGrVrPK1Dyu2p69xYrnduD6Vm06sJ4olDq792rEj4/hdzVwrtgw+d1ZLXG +3ropWgrHQT8z7+B9CAIAOXhYlKrV7+UdbAod+j8OpCIHk5X3+NkT4Ht7biqzQvbo +pJJILFA1qHi230N9YR8ng3PHQYObYJ6NFBrxhpXIfXwbuPyrEApY3zaL3HbkYC52 +aAI3zy7WOqZSqRZ6aDzXdf2EMGusNSxj9/TAZhTAiJvwHdwBowKCAQEA3eNC/iMt +kmy4R3FQgti0Zq+CBUErMn46pQhBCcQreI/a5U4UT/iY5WGutKXp45d/BM2ztyQL +T/8p+85RkasVF/rJB2PwlzUZKAAq29nGXuV0I6N6EiMYa2LfFLzrrleNamPQ9Ubn +atp0kiyLiPZ6P0+Y5wZMirHlMyup+fzG6xsS7KVy0Z5Fy4YetP63r6xCVQ+Rdu3l +dvXqGb2Bdc9g4OxES1Zj7MKHg0b3ce2lYaL0cq0z3kJ52MAVbL9tQQOstJX41VYv +/QSVIjC5VACSa2qsqzquqxYOyT1U0l/8innHfD/uY/8907/q/JqoO1hU5LtvZ7OO +ZF/e/ycZCM2U4QKCAQAXUIJQ9v6wk3jQyoguEAD/8gOMa3YWA/OUJySOZRAfzp1s +/jBImJo1nQU9/67aIzdRKOBqDuObw3C2lufJS5BPo2p5K5PrD0DrGfdsuueEeAFW +kpOuIcDCMHh0US/ViejaCV10HPhfO5jrIXOFCU3wnV3PVwD30kx5PhsbJz+ggAEg +mKOODRUN21K2IEkV35TlaC3//n2VKsFyop9hSQj4GW0fDdZIPdg9czff0tbxDLHp +xXhhdv6+ZLvUZPfxqE7lPGNYEq3v+ufFrizav2pg3PpMP9nHD6bbz8v+VKeCB4jc +isSvr6fvlkU/tMgB51OuvwTDj/tmMnWG/nIoAqJNAoIBAQDWiLYsS8zzJwUhplDm +winiosz+0Zy3jE6dZBamH7K8NbK6RLzk+YKzPbgSV9yFPeQEu/KIH2SEqzxnh3tc +cWLKtaKK77keKaux/j9yI+RlukqJbrVHNgGVSppQTb096s8DT5Eopa54pNFSx5j+ +Cvn1nrtCm9eDvi7SQ+RrnVii1qF8hxc1z2bCOmIUM7dcNhxIa+4EZE2ZsHjw/EZg +puqPbkE16khhEVC+v+3djJ17gngBLK/atMFkrYvJgmhbFPd1/w8BDf0GENk0npGB +w6/OBez+/ZUGPCR9tDv/z+i35rjWzGVs78tSodvM8qe4AVbLdOJpDLWfHQbaAm51 +EXhhAoIBAQDmZVXAS4dTDAp/cGwPoXOyFxu+UNnGnAKO0S3aJW9qV6E5N7jsLqzI +4eD2Mk6chkBrO4Upmwgx4sLVnDMlHQGvoqpWUZES9k6oIgNZ6N7KXnvFm5GI7mlR +ySA2LftCeSb4BzQmwyX5wcVjOzfB6bkSgEkvuMFrRStSL6+79XZCoh54jBm+fW6g +up6oXa0+lJbyO4Qrx+oWoe2G9nrUJzsjV1Gj1njnxDECCMrmB+X5P4D02Ac2FgxP +rN+bxs0TpvO4fXsvBN4B+/dtF2Hjgo3rQm5FQ/NmpoO5lAs1VZPjVUiFCjhm3Fyk +Xe2nzT23gDTuPny4yivLAMHPZPfGLLg4 -----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client2-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client2-req.pem index 225a6eb1212..f36eb94205b 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client2-req.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client2-req.pem @@ -1,27 +1,27 @@ -----BEGIN CERTIFICATE REQUEST----- MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp -ZW50MjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALU3+Z07LANG4lgy -Ve9pjWQEHyAZn4iadESZqpkcVKFpeHXmEmCvDkt4f+xs/tJle2YmzmVIOgTrhOeY -+3Et9ydfkNM8N1LUiem6i0LQq4yhg8/xM9jeKbOzol1CNM9TbRzZXHKyPX1Ykics -3iRGE3wqwK9quQqYIvBnUT367N7w9kvsgvMr+E825BaQ2hmmefi61vXbCRO7dyRP -It4z9ERMeAHY1Y4ahM1OwWcGNfhqPnMoscWrWarwIE78SoiUMXd3mVUE27m/opmL -bvk59qh8o9bu5sje+4iKW7P5LVEXWhVnbUptht786f6zTFzWEeG8XUF6TFQIAtO5 -P+QxVmKhsliEt/vCJdTWVt4TKT7c6T+S6ivFl9Fn3/KWkscKLITsILLC+DKxBrac -ohoROMfzYSGgDmE2d6jnQZ8F4U2SxtGelk1BVYTOLP2y3Rg/xlPB7FgHiq9OJitI -uUIN8yAtjEedQN47OpUt8fMMZHukcslTjXCDgzFpZVh/J+oIULBoRBahNWsroT45 -rrDhum/n1W8jc8R7VfLVXhb/YK84unxxGduElbNwW3T0mtDMqHBdvwj2pfNmGj/k -gtSSuD9kREnIuywyiQCLMM8N3UacMwPZadjFxeugBuKCXX0U4j/llnasw31ljfEL -9EMSuk1a1PpLyPM3VM9guOzX6FKXAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA -b1qAUlTncymAKrwibT26XK8txk2ldN290bqpMAAGIYxQqn6ogjKAt6s2Tz3LOnZr -rKbLBVB6RJgevu8OEJdk+6HlqzCRLFwk10w4+aU+FIkLPMUsqyD0I4cfXoW2kSkP -LOMfjXjuxFE5wjiMgzKf9oIdeKcFaiMH7ThkxbWt/HXnI1VSbLH1Ff/vciA7tShu -Y8drVmps7+yGeNW2fzHkx//vIRLR6cufEZpAnvn0HyHiDHBmjCEPwihu7g+H2s57 -m2B17Pd4/kcuYEvv4pExjlslewBjqAfe5peN5/Lo2rful9oGB7OPwPCO3Ndb1kY9 -VQjjYkw8Ie3bQa/v6MQSfpBjuKDEkpG47mNQXy3n+pfl8Bo0E+FQQx0pE3cWrifZ -Rk+PmNqlaERuJvNpDmXhq22IWuUs5BwtttqKPJTox6e6wggIMXCukXxWE9ovSzu6 -VlNM9QbyfrACvLuP+4YPboXFiBHK4afZbpqWl1uT48XJxg6/caJvazKSx9KmV6PN -zaMFCtyGYhY/G14NWDfG3lW4VBRhwXHxHA3p8Qci3ldVgxCsakMtpnwCe/BLo1vP -aCdMrnUEwjs31wldiRFtOA5dP9TaWXtm57nsOSfL9tbE702wfT4oO8JehvbD1iLu -ozapJfOOc4nC0vUqVXQ+TZb7MWGdsV/t8Lly4qi8Ydg= +ZW50MjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANUgBLgZEjiW3Lmj +GlCu17h2ERYc75YX+7esbw/iMWMrcrXfvNNutJ4H7hOsV81ZpMEQouHRfog8Inra +KCuwmb28DLhJZJlpE49hZxeUJws4yN8VCCTgr/tLtTsvsSS+TGA58B29HHaglJwY +0w2mOlOVcUxOkjne4VHokLOomhzzqcqTLCjwUslZqRn+SDgXyRw9P5reJ/m6E36d +FyBeVglZvealVp4uK/TTqVVFJYBljD22M9wrOeo9AIvrru5VlgDNHu+rwqgSeE/b +GcwhX2J9J++lwOxsFDR7OILY33yoD7/6NN61SJEoRh7xZTQkr/Dc3HpvjYyj4Yag +dWKq2xDZzIxH+QfEMQssuFvulq5L76TJpXB3ceyCu2NEvb/563p1EvXQbp5Zjz7O +jo1elobs9epvfbCmiANmTxG8GLKteLXfjcph492gdIw0nsV9/bIzE5C7lnff4nEU +9E/uEJz0FTw61VjcZPtpqEzLE/8abBU48pTj5HqKo8Nsjx9hPl6trO4h81yMaqwb +QDmza1KsU+CPIFiycyv8Hn4w6JEjwnUc08rOoQ3e7HjqLNpn8X6RirVQUrwSU7L8 +RTKeOCOBLg6AMXfH/frPRnNQjUG/Z7tBTjTJhm38qucxyHI3J5jwX6vn/jBfdFHf +T6510V0Q9fzgzp3H3fyHpnLW1qxDAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +dr0LKtpOa+Xu9PKwnlsM48/ltph4q9+tsu4CeC8XGoLFNbVIALuZZsKZDehTf+/d +bgEtjW8vVnBGAvVodo1MgCHnhPPensDLfyggAULT2X400cly+suGbKeu3kIOlKCs +TQsFdNKOPm17NcpuM1wTik2UT2EWLdzZ25Wy3Coid+ILrf5YZ75djqtxZlYbRiw4 +4IndIjN0bYsn8l6Z8Pt5HdJ1nQnbDZhQrx6FXWZ3eSSmpklfl4O07z0KlXi1Nmaf +OiVcOMvZUnM8pYmNvul8Jus/XmP8x3jSbYzJDNOJ3YV8+OD8DVG3pLM8U1FmjCZ7 +KiR5DNSxZFpHGXhUqDpTrhLgoqGK9chOqPdzU7Mp4taEO9FV8Goc7BCeOKB3Znxb +XDIszs0oBIHO/tsqUwEcWBI0vjyC2pBYQAYK++qwwmvbfWg5lrb7eH1ZO42DU9QD +AVR/5luxImAA11AmSsGf8i+FJ3F63PzSr0uUG7BnTLC03xna7dPdKXS/pGojNVBT +Q5A5J0rB3+4L2mZLE3mjst3t1xHfLW/0RVRqGwz0QUIloZkO6wPN6Jz6l5Q+TgCY +uEks1YN/qlwjHwI3ycT+Hr/sY5igT0OAySo7qa7lN13qTiO2z7eAMDgafNnq34kJ +4OQDCE28Bni0fFRIaqVCqTU31Kei5jbORif2wK81Zmw= -----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client3-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client3-cert.pem index b674ad4bb5b..376c85ab8f7 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client3-cert.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client3-cert.pem @@ -1,30 +1,30 @@ -----BEGIN CERTIFICATE----- -MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyR7MA0GCSqGSIb3DQEBCwUAMFIx +MIIFMDCCAxgCFAXxDGdWf+MHldd68lQPasjUzyRvMA0GCSqGSIb3DQEBCwUAMFIx CzAJBgNVBAYTAlJVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl -cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTI0MDgxMjEwMzk0 -NFoXDTM0MDgxMDEwMzk0NFowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt +cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTI0MDYyNjEwMjUw +NFoXDTM0MDYyNDEwMjUwNFowVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt U3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UE -AwwHY2xpZW50MzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOpzzJkJ -jTA4iBVvh4G+z4QNXoOyFsco6zFllwNNWyIGa6kemP057eSJLnacNNPboHEIK1zl -NinF9Sfj1R9PfdCrVRLnopX/6Vkj3L/pxPS/J+NpAZPWW2FosqkrFm8Ud8/p1AOm -dNNubgjmHaq4xAXPWeb/CQglTxYbsswHJtt3G+0ll5V6oJkGR89TYmko8Hqlgdw1 -3LDuJwlIfjxllLUsKB1cBQFxbeCPbG76F+tW3WatJXtEP7foKC+PZjeh4ZyLwSZr -AhbebrYdSyIwG8hCnAS9Dif1W4xPbyhZZuMe73t9APrdxZLHzmLaSzSOzEqFhEND -ab9fJ+goBWEt5p3qY+aL74jnJwwcneJAcsu8Jwsf5T8XOg6deNfpFTD1vax/A086 -8LzIeNTigoRo0Nt/Df0+s1qAr966TKcxkdBAVamNbbGR7Bj5OQO4GjKWPc3pNLEc -Xh/f441ZLoBoCRaWxIawEl5ZBJ5pwWuZdEVZFZj5oc4OPkKIfG6WkA6gy0kluuvm -3ArC8e2/6tQ7GytPIGot3HDz7QKIAvCWXS7uMb5/WBhw4R7pQ9rSYXLKdh3hPOg/ -GscwskBjQJTQk4XVtj3UQQFX3KgBXKOHg5XmNQRmDYNcNZs45PkgfwXM5copJEZN -TTwo1TDMF76CNA6V8akP2VsBM9hOFcHlHqlBAgMBAAEwDQYJKoZIhvcNAQELBQAD -ggIBAA1n8oB//2uheGDoRmRWXM4o7uPLcjN1JkRoCkj6ysQ79X4qM0y9hkH8S6fF -k/sz3zSnpB4MpA+eNbZPMZQzcFMSVdM6iau0QN6gnDA6gy3f24J251AHLC0itTGb -2uh3f7ENgO/oMwqTR46ATh15yS9RnL5Fi+/coLvvUPpT/rChPbQUgzKOCpV2OoEg -2IDpp0UD7GmS3Xq/CxS/Q3+dKtXlnlSodr+pwx93ZD/ViHltv5p74pkJOXtadQ5Z -qF3+gMB/7qoPsbarkPz+UiLjefqhCD4WLWBkg3WEouejrvrpHL61Wl50nryx0IM1 -funVy0H+DEzLJD7i5l8IvAIZTIZC1Ewsh66bavYpyCySemapTsEnnVxWLG2LkTlo -iRchS5W3S8Bm/b3z6XY19uhOhguYpHBOnRhoykbR+jV8oyrt23r/RegqfMVSPqbz -1rmetb0qCYhhoFwJ2/kattLas1lHEbD86e1WwJ5Hc3P4Vd7JdU5zrNd/YGhV5hmF -1xDCoFzkS80yvnjHrsUiz6r1fgM1YcpsfYuDLhOGVHO8CQywrScGoWe+CUPD0f63 -azylUumnexFZVG9CMCxrePwVktbBOJvMN1IY47P5h+UyStHg4PJ3gHBbTIJJxVdP -W7LqqSw3gynzT3UMLWuYCc/KH/87Cf355hERFLetmTeIY6i3 +AwwHY2xpZW50MzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAPrdk2YZ +HRhd4RPfa89z2Ay/TOby545N5n9R+kpQvyhnCbr41afzIU5DrBX7y8cKcvo7y9Dk +Cdd17Xqn4oYliSvVNI8B2nwkBz87BUYT8pNVD+QUc3Jf2P4Wj6XQM4pd9Ntaw7rO +yIf2Oo3Tq58SSjbXYrqAbCYuX4cs/VWLWyq9PapBwxEPeQ756GZS3Xvq1IfCNIKv +NLYRgExctHzkUWPf6WAS0lKydBcCobYvSMfEzPkbBlIhuDRdMmzuji4FefODY4lI +zvnaD7+IIiRC4+IY9xNhdH6s0UobpSIqLCSxOJNYwqhUQt6gNAO1mdJhUirV/XIl +xG5nCGbQS77yeoBLIBEL1t7tpo3/AdEzkR+/wS11dSpcllSj+7BJaKBhTKVZrX1i +gMqUSbiTF1Et9PnAkS1jtUy1w3Ja7FyPWfM8nt/K6vfNRudg/xwY0iY1RFdCXuMw +kPZSr4W+QryGaPqm0RlpCpLHZBOxBDpf0ZcA37ullh5hjXsn5CRw/0ZYpanqrkrq +2cVZLnY98IrJI1QfQhHlDqUP7prR4Omk8C7edXBqQqE/0mqL7AEhppOyLedLFC7W +wUBepmc1bNH+Ho11CZeSdTZfIgwAcD3v6MiMA5kMTRcW6HAHNS309zNJeDf3Eesz +TBXOSCqNBBbk+oW8bxkTLRdHRgdlLT7N6qzLAgMBAAEwDQYJKoZIhvcNAQELBQAD +ggIBAADJZ+I3CJs6E9U2RjIzi1lMo2sYgdcKJS5+yWW8CNjB+DibKfkWjgWvq2K0 +i3hT0Uc6y+ter4OOeIkGtofOiUPekaZsQkPpi73sabwhDVnlki9QL9Ayrd1qDX82 +fMM5roL7w/a+YdKzTQE9hiwPoQhrpj/2mhu7LeYhidSqwzH1anU5YtTKHq3ZrdGN +imhnklcmbqfcNQU0K2l2bu5vuJXFs/v5FCp72ux2p6QDPWwMbwvr413wibt8o7ZT +bBGsQ1MtfJynRVwLGLosn+2t3NPJTfjd4dMEsZhkDY0EX4vbE1/X+K09EN7jPOHe +aJ2AOt3cO3A2EHCR3Dbmt055C6Lb/YR6s05dX4lBT8zY0knsWSL3R77kQoa3+7oR +hU46ydU6K/Kt67nO938WBvFgI81IatRVKVRsXfTIP2oEa0TkwzuvS7nzj3czNU8o +EOa9ixawVYRlEkcuE4KE7x3TcLEGa1gYJDGbsXAfJct1Hur1SJ/rTDwZvlc+qp3o +wWOLtN0mVHEH1OaGlWmeeTuRG16CuTcku2DYiqeuRNy5eZddSuMOag/DKnIN5ZqV +s1GNrpnxPCxd/KFKtdGl+l++Bc9dBmkd+r1dJ/kRGvhul77Zm2xEnGdyybIs64iQ +gvXq8d8ohbZOPxswiFo3p8fbBjWjv0qm3UnlU3P4B3RDMrrC -----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client3-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client3-key.pem index a5552f19c0a..f88456215fe 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client3-key.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client3-key.pem @@ -1,52 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDqc8yZCY0wOIgV -b4eBvs+EDV6DshbHKOsxZZcDTVsiBmupHpj9Oe3kiS52nDTT26BxCCtc5TYpxfUn -49UfT33Qq1US56KV/+lZI9y/6cT0vyfjaQGT1lthaLKpKxZvFHfP6dQDpnTTbm4I -5h2quMQFz1nm/wkIJU8WG7LMBybbdxvtJZeVeqCZBkfPU2JpKPB6pYHcNdyw7icJ -SH48ZZS1LCgdXAUBcW3gj2xu+hfrVt1mrSV7RD+36Cgvj2Y3oeGci8EmawIW3m62 -HUsiMBvIQpwEvQ4n9VuMT28oWWbjHu97fQD63cWSx85i2ks0jsxKhYRDQ2m/Xyfo -KAVhLead6mPmi++I5ycMHJ3iQHLLvCcLH+U/FzoOnXjX6RUw9b2sfwNPOvC8yHjU -4oKEaNDbfw39PrNagK/eukynMZHQQFWpjW2xkewY+TkDuBoylj3N6TSxHF4f3+ON -WS6AaAkWlsSGsBJeWQSeacFrmXRFWRWY+aHODj5CiHxulpAOoMtJJbrr5twKwvHt -v+rUOxsrTyBqLdxw8+0CiALwll0u7jG+f1gYcOEe6UPa0mFyynYd4TzoPxrHMLJA -Y0CU0JOF1bY91EEBV9yoAVyjh4OV5jUEZg2DXDWbOOT5IH8FzOXKKSRGTU08KNUw -zBe+gjQOlfGpD9lbATPYThXB5R6pQQIDAQABAoICAAMZGKEFEUqL1LCD0sSSIufZ -zV/sIITtM35pmz897HBcqVuIvfjKpSwZ6/VBRyltg4c61mfZ14dhyEWIqy5IvJ7f -RLaFPQ7CXPECmk4m5qVdSUemAZFUicyVt5aorRk2qgajTlvl/TE2ClovwECbRGvX -O5bj09i5tXvTTd+IUKkhv8q4bnJZNnoPLS++KFS/Z74XJcolJA9qdjWXMaPWq8ph -FP6eUqqcNxl6i7JDt8EyWqaarx4b3sOtW6qVOIKPrw2Egz7gtxxaQBhD9tQy0oso -5irh3KgGg/ksq4la9RMXO47kLfkiqROxdDa2L7w4DtcFQKQq5eDTfxGAReHs21lA -XzBgd2/D9X10HQc+t37DVej60kJ0I8G2YqCcWlmcmoWcRN67LSg0Brje+m4ohYEQ -rEz2tPWDIKQBRecsfegP1p07/kFd24PgEU60MSp9D6tOunIeiI66ix6UDxoFrSA9 -wzyBNBw/12IJESJX5ggFA2RrkAA/ZX88oWGlXJHl9SxSq0MFO6t7BtgeInuUigTr -YYexKAhrkz/M6PRSH1fyNd4PYdJjb6TYT7ITarfBU0YR7x6yjTgbLGQrnddbMc5K -THF5KuyImPw0/ewCw063zyC1Ze4XVNJ/i11Gx1knqYsN1XvE/OnbeAu0EyyNjpe+ -GSHXxfd6nF4B/tKFaL4pAoIBAQD9bqupNI5Yo+q4PrlGAjoqvhovmKdjlfbi2Rbr -Zqz9ooOnLp/5E8dwEQy4ofMlRUOe4yH0Cv/DVZ2tR5Z0GVmHbIBKn289cnJw80HC -ljeth7x6k9VUVX2m/2srkjAmPy+fY/iqxsR8Sei0uKAS3v8TTq9BXuM9Woh9W8WM -z4FJtRICCAqnhLewopvxgb3wHoMm28Tug8fW5+WQhAxD0fWNs9lUe628EFD1acqp -SJ4SVAa8pMXl8k9EmLEInDtImJcLFo3DEH+WF/IJF/WXYaNTDbzb6uvw7jYW5zYp -GRDDeReErcrO6nTaRdKUwv6Kjz9H2tZAmJ5JJ09QgpmQVT6TAoIBAQDs0+ZxRUGu -qI0csPBLz1F4YSl63tbYPBS7HWPhBNg7d1PExQRhOyV51OShgtFtB99cgw5uJ2ch -OWbqKi4vQehQEHTCEdkPCpRT5Sj64AKHHNUCgd1D/Z5ZdA+Aef+e3NiDptTWaCQ5 -Eaf53KCm/VqoGBQBctzOAkh9pGvsIIyXzgDLI03i5/aiePVJNSTFfRzOROtmcsyz -4Tup0ntbgu09qigylPJQ46P9Js7MCkuK5V+Bp+kYvUMv8Yz8oknaaHYmo01yn77L -XshN9Si1zY5T8CSAfJvGpgqRLpcsbHryKRYm9uUu9b5TUYi/pPiCPiTUCdV/fnIN -0mmMZ9hykMlbAoIBAGDdYP0+Wj3lAPzE9Jmelk7p9CZHCYuVsRSJdyooR2x2Ji3L -M1fHSI475gnX1JBlRfA/ziVx4sntOmQVnZroaYDKZsqe36yzxqwHPhY6xjMsU0zi -nkIqnukqbPLtYDvuIKyiUFQtvsHmmewhOQWdeA8QHwo0U1SK+uo3Hm2wjjD25Vgj -bLcUkgUQUdxgA5H4h5Zdn4qukb5BSkwPPITbhihQGnwg/YmJDviOI+jKGajVtvF5 -ZS57i/KjDd9Fn58iu3CAgVSSRMHAi0EQiE0BA6Tl9k50HxQqaEAexWO18eNUsDmF -F6Q4lssqrs8vLI0XLU+wg/2Sl8VMIhOap0k0W8MCggEAdyCn/DZAMeErClGeriOc -8Za+TMYnACJIs063XQsY0eDWTFZmO5qK4VvLncq2Gcgp/NkXuyUq5TWApS2Oicr3 -Vr7QXIapzr0dm43cLUXdP6WrKFW2vWqn6otM+O1lPb4NUzvqN+euwN42xqLKVPWA -Uqm59niWxTG00S6R8vb0ga/oCka5+PeBwnxhte97jbO/d6qHVsNMYPddEbGEx6V2 -PNyI19jAgxve84o/37cWBMoYXpnd0MnzL/yrVLE1wR1mwUzHum7MhHQrAbvePIUn -oOVdFxyXJzeCfUEYXgo7VfCA6hyrcBHBKRwMU4piTvF/iQFhWX57VKSQ30WlRQu7 -nQKCAQB0dBfxvhJVMIdWE5IkiXj8ui+86LjYiBrdb0QBGSesYjZ38eChi57X6UF5 -Z0h0zvb95P6D9ZOFwEJan3pd9nwS9ud4/kXfnx8k4KtimOpw/41bjjHja3oliY0t -wgrSvRvJtfAqaUf+2gbbFy+eoiNdtCmu9btnwPBV1h1uVLK7wi53mkrufP0X9Cx9 -wjz4TkDtckjxzxYp1vuGa8SKZsxS+uXK9vaut9cme0gpVqwNeTq9yQD+GvIcpGs3 -eOXqHRMuyClyt2/jInwVSAJPq3hgGL+Zn+DAa++VhWbHASh9m//ZdwEtUeIiyHhM -Tea8lz7hwFEIIR1F88FY6UjN1dIn +MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQD63ZNmGR0YXeET +32vPc9gMv0zm8ueOTeZ/UfpKUL8oZwm6+NWn8yFOQ6wV+8vHCnL6O8vQ5AnXde16 +p+KGJYkr1TSPAdp8JAc/OwVGE/KTVQ/kFHNyX9j+Fo+l0DOKXfTbWsO6zsiH9jqN +06ufEko212K6gGwmLl+HLP1Vi1sqvT2qQcMRD3kO+ehmUt176tSHwjSCrzS2EYBM +XLR85FFj3+lgEtJSsnQXAqG2L0jHxMz5GwZSIbg0XTJs7o4uBXnzg2OJSM752g+/ +iCIkQuPiGPcTYXR+rNFKG6UiKiwksTiTWMKoVELeoDQDtZnSYVIq1f1yJcRuZwhm +0Eu+8nqASyARC9be7aaN/wHRM5Efv8EtdXUqXJZUo/uwSWigYUylWa19YoDKlEm4 +kxdRLfT5wJEtY7VMtcNyWuxcj1nzPJ7fyur3zUbnYP8cGNImNURXQl7jMJD2Uq+F +vkK8hmj6ptEZaQqSx2QTsQQ6X9GXAN+7pZYeYY17J+QkcP9GWKWp6q5K6tnFWS52 +PfCKySNUH0IR5Q6lD+6a0eDppPAu3nVwakKhP9Jqi+wBIaaTsi3nSxQu1sFAXqZn +NWzR/h6NdQmXknU2XyIMAHA97+jIjAOZDE0XFuhwBzUt9PczSXg39xHrM0wVzkgq +jQQW5PqFvG8ZEy0XR0YHZS0+zeqsywIDAQABAoICAQDAMTs48CqTPXEvyW6OS+EM +uw7OrO/r3RCnIIYRo1UgPfh9byA5AJLWpA/V88eF4SJ/RYp7qglEMcvTuYVZYq55 +j2kp2rCphOysa6o5qxSf/X4kLerYiEf1OhGpZh3mdt8doqbrmnqVd3YarD0CrH+B +DnhMDBFPGx4CsNwRSqd40ezJYIJyspj7eUisA/Y9doaGz6ltKY/HoRba6fc4667T +RntEKIdL5f38lv6PViB7M/IZMrQf/kdijrgQLp9s8LMiddmvFsHDN2XzRfdqMnjm +AlxgU7xtRDc/gHh9+TNClSeT81+GmK92YeQXp2yGehr6SGFYr0iTkIomQpSVYK2p +0haIIjQMHlc7E6WVkDELdpAxERgvV4uDN9iEkd4t9oNDPPRioPJQ4bhbMSxCO+CP +NdFHTxIbaDr39OdgqNNE14j7WJsFaCsYXH2NFF8jvwIkPQ3QVMQT/JPGStkyF+9P +5IjFfQ9aEF2i4mAVYiG0DE3NyD/OOI9/uF05POn15H9U+bA9hfBE0Rtm9nMqfVy+ +zgmajXkVb0jTHdL2t/UKv0YdgaglvDcWGFdEUskjJoB00NJwBGorSvcMZiSTxpLD +cGRqywRHOEqNIAbKv0Dt2AX5ZdBSQu7/z1/5Jcdmx8vp9lVhQKeMzYxsFKE4V7fr +ztDuPOlFGyffxpRenBIxUQKCAQEA/XVyoOW1cSFqeG46mjw+dbwjqRmLtEVhAMsG +TtW8pnMJHZ8u7lfM/UJyMN4NQEPJElrABns6I3dqPOwaKOy1a2leHMg5Vvd0/uqp +s5a2fduP6l9PXvhhWDN2sChbeKhl0jJDVnaTO7tiye8ZGMYOM/AlfQX/+PY4QNgd +O7UwcLKhoytxtPtHFZTOZp+cECdTvlmX9lZoNEzFp0nfzFaLVwDsy0B9e6KGt1xJ +fV3Drw7p7PeUyYBNKkyCRVee5S/pn5fT7pkIxMHvaL9BBnWVpwiH3Vi0hfTfFZk4 +8tLcVZgf3n0Y4dMVP2VQRF+kKBTL0coLne36HksQEJyk/4KZEwKCAQEA/WF4z6kc +YXwsU5847+ywq4ipq9efadkMDaGzI6Ez06TQjRYNsZGplCV9fiGxKX2YmZyFzjTf +4joqOmI6UANk+JZKW0Eyyak/TnxugrjMFq8WnK64cIz1TK054tAM/bHGkavaYb8K +bCfbKmaSkwkTbb/OasbQqsC7jbALdbM6Ae0PMrpPmI90YYIMYLRogIaBqCkB43vp +GEZN2VeNS7blhRMiq7YBDXn807aSMQ0+skNSQ7MA8F5i4BFvWyPb1nKZWux1RWLZ +O23IxGWmoGho1CAaEk55LXbqLygU5ZYlBSqkrP9N/elJykOp0LwpjoYBgjMPmanz +o6jy8XIUP78MaQKCAQEAi8+YjqaHosMTDyGG1AN9VMaWSTYdOTC4JI7ZiO0f5hU4 +pw1i/viRy/Y2NTyXxKZfqO9EU47v8BZ0FO0MNRz1qi1yS6Aq+Q0BjYh2Wek9+0j9 +JwSyLKoIUHX694sbggAqQnuVZ4F7EAz6nnd0uZSuyvmiREfl/jgbqbFM1t3IvbHb +tb1GONYPTRlLjZJnrQV0jWCwkaLyUj8zHGeEuxvWOwT4mdmWHnf1pfmTVEM/qTYp +1Zxwh4JtjnKrvYJq1PPMBEvlDQ1/p8FuxbISNXTxOzVadL/0vJvp3ukpX9Du14xV +sA4DhrZAVzsUvtKfI7jtAWlZZSGbwdAYKYGvBn7M3wKCAQAHZWX6YcxTSCWfF0G5 +NyZ9C1Mwke20UEKaz0KEYrs5jVENHTyvFzpk+actHFyogmMG8NuzBjYWy23aIG3l +UgQLgY+QFFogKtGPP/CV3kEO1HOLhUoa9vJeF5xd84a9jQfnzqVkPwhV2d/6392d +byFjDbs/wKfspA2VeDMNb3rc/Yd5CpkyMdXK1tn3pKx8O/Di8Ld+ZWqLa9nv4y9b +q24NsV5MttZXB12K7IRd7C4NVAu9sCbx3T9znO6sMWLEYrn5Pne528XNh0nZ+cGg +YwvUTU+VgzbkTdlOIRRjEzvnZ7RA3H7xT3L49XqqfiOUZnL60vS8nopfF5pn09Wl +erUpAoIBAQDWHJQT+Jvj+dXPC42oIRQKCiYtp1buM8YyL+dJNi7p73brF+2Oqx3k +XNT5eP9GthGqpVGJ732FWJDbPViuZB12zlx9tpGF3ghQTq3p/95KOhGEb2fG7mnl +bEcPqOoFEsAlc4DZYqsDDUvmsifimKm20ZWi4VjTqQJUHYCegJsjrA7D4obGbOxX +FujRMq7/idXRjEoWLloQTMPAQ0Uu4Omwnea25daRzrrrJ34uYrTO1sNgOKk9JAem +rGgrOzsRVG1aNwddcZT/t/icLKS25G7AszLnrNFxJB7DRAfgzpHkJBwNLpcPVtfR +KB6GTGRi7uqGHYScU6+wRMHjdVzdKGNM -----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client3-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client3-req.pem index 0163a020c5d..7c679b4b367 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client3-req.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client3-req.pem @@ -1,27 +1,27 @@ -----BEGIN CERTIFICATE REQUEST----- MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp -ZW50MzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOpzzJkJjTA4iBVv -h4G+z4QNXoOyFsco6zFllwNNWyIGa6kemP057eSJLnacNNPboHEIK1zlNinF9Sfj -1R9PfdCrVRLnopX/6Vkj3L/pxPS/J+NpAZPWW2FosqkrFm8Ud8/p1AOmdNNubgjm -Haq4xAXPWeb/CQglTxYbsswHJtt3G+0ll5V6oJkGR89TYmko8Hqlgdw13LDuJwlI -fjxllLUsKB1cBQFxbeCPbG76F+tW3WatJXtEP7foKC+PZjeh4ZyLwSZrAhbebrYd -SyIwG8hCnAS9Dif1W4xPbyhZZuMe73t9APrdxZLHzmLaSzSOzEqFhENDab9fJ+go -BWEt5p3qY+aL74jnJwwcneJAcsu8Jwsf5T8XOg6deNfpFTD1vax/A0868LzIeNTi -goRo0Nt/Df0+s1qAr966TKcxkdBAVamNbbGR7Bj5OQO4GjKWPc3pNLEcXh/f441Z -LoBoCRaWxIawEl5ZBJ5pwWuZdEVZFZj5oc4OPkKIfG6WkA6gy0kluuvm3ArC8e2/ -6tQ7GytPIGot3HDz7QKIAvCWXS7uMb5/WBhw4R7pQ9rSYXLKdh3hPOg/GscwskBj -QJTQk4XVtj3UQQFX3KgBXKOHg5XmNQRmDYNcNZs45PkgfwXM5copJEZNTTwo1TDM -F76CNA6V8akP2VsBM9hOFcHlHqlBAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA -m+9LdrSuXxt1vtz+sIqotYTfWOlEO5yDvxYIV5jS9BaCGrexbj7ku/kaA7AvnosA -ztEb4ZOsiFFyETSYRR6+br+A3BHDBgyiKdilEhH7AzZYPxDmkUmmYWRXeia5o1tu -qQWWoB4pncEs303a6dGY4IFjbYTCu1UPP1A46da4xuDmsTVf1OE93T3KClEyyAZ9 -DCbAzEj0Mc0iiJtEumP71nl5xr8xeq7sQ0Qf2YQHG4svV8TKNwQSRd3taaFQ82H3 -gxqL7u2wS9lk2Xg5BOvcpzGVXjxw5qwNGgL9G0g8FiAT6fWi2Wxl707WOOUx4uKn -wLJJfO6q0ojSlsd8JXlPXfvoVKIUmDUKIwuz6/UgLpDdm43X62aFyO9htSVVK56L -OawMXFHLSkf1SDHirqhhTIDVlOCM4o6KjbnEQhjU3l+kFmKbAUQ+wfDOO16i/emS -tkuu1FRouZVwwTMXeEUciVjadUSrBJr+balX9fjJdqTDSTc1/9ytkQ66h1JlJfy9 -U4T2hZTzUlMKXm+x99qxQyI4fG97zgoL0vU2tKJf/5dkqTki5b66RLtiPbuduJtH -Sdz7GyuGQ/H8qzgAQx55qWA9j3AQVCZZUr7mA02Cuv71YnwpqW1uHM4re48+pKKJ -Uys9m1yS1AEloa9P+2qwLKylbsEzO+UxFXZYImfCLRA= +ZW50MzCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAPrdk2YZHRhd4RPf +a89z2Ay/TOby545N5n9R+kpQvyhnCbr41afzIU5DrBX7y8cKcvo7y9DkCdd17Xqn +4oYliSvVNI8B2nwkBz87BUYT8pNVD+QUc3Jf2P4Wj6XQM4pd9Ntaw7rOyIf2Oo3T +q58SSjbXYrqAbCYuX4cs/VWLWyq9PapBwxEPeQ756GZS3Xvq1IfCNIKvNLYRgExc +tHzkUWPf6WAS0lKydBcCobYvSMfEzPkbBlIhuDRdMmzuji4FefODY4lIzvnaD7+I +IiRC4+IY9xNhdH6s0UobpSIqLCSxOJNYwqhUQt6gNAO1mdJhUirV/XIlxG5nCGbQ +S77yeoBLIBEL1t7tpo3/AdEzkR+/wS11dSpcllSj+7BJaKBhTKVZrX1igMqUSbiT +F1Et9PnAkS1jtUy1w3Ja7FyPWfM8nt/K6vfNRudg/xwY0iY1RFdCXuMwkPZSr4W+ +QryGaPqm0RlpCpLHZBOxBDpf0ZcA37ullh5hjXsn5CRw/0ZYpanqrkrq2cVZLnY9 +8IrJI1QfQhHlDqUP7prR4Omk8C7edXBqQqE/0mqL7AEhppOyLedLFC7WwUBepmc1 +bNH+Ho11CZeSdTZfIgwAcD3v6MiMA5kMTRcW6HAHNS309zNJeDf3EeszTBXOSCqN +BBbk+oW8bxkTLRdHRgdlLT7N6qzLAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +WLhSuFZ6pnoDe8LQx6eMPXzRkQb1qsJpyjpegUxFe71o2e23V/1yMnTfFiO+DsBQ +PP8RkLWUKAvkAvqPyttJBx9U5ZYspsSsTVhPsCjUFZ4IG+fc/dVP1ZRid5HQJz2+ +bFf4KPgErZkJZR02Q2q6ZpKq9clRzbDkho56OZXLYI/o2Z4xADbhzpa0xt8sx533 +bm0rKvz85WxH3cimRjKaGKzuKg38ZaXmmUbsigV3dzImT00KDWmMmaW9SB8lIm2R +JToms0Qs+mOr9qD2NiRoiUd1wmgG2QpFDViIqAZKJjjeesmeV2CAcPfLztOZBim4 +6bRIOIXDhYYOyDgs52XuijXUr4BR8aQmqBrjnccCMcGE8Ol5ZH/IDg4pCRSduCWe +T7ThhH7BpAWYdgF3ITcp5oEcpXK8IdAMAst1/6vk7Z1JHIOejxksbLsGDYkaLM6w +yTn4X3Ak0X6bVmLAY+xAL/WjAJhVtDPqGYAmpx4iQ6QjYG/8gRdOiUI8H7MCK8+h +P0auhyyMmO+kdhNnzwuX/eeLXZfNvnyK4n2uHWYgwV5I+Kv282zw94UIQgwVQ2DN +/IbXD7K57s7+ff9Eff8L/B8rt1i1cmv01mEgQ4kMsLOClGaceGcz/ivfzDCosmsk +Xg/zVmdunUY0lswYL4SQM3BhWB3xJ4likHikfQHklM4= -----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client4-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client4-cert.pem index 175bca39128..5eae58da627 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client4-cert.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client4-cert.pem @@ -1,32 +1,31 @@ -----BEGIN CERTIFICATE----- -MIIFmjCCA4KgAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJHwwDQYJKoZIhvcNAQEL +MIIFWjCCA0KgAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJHAwDQYJKoZIhvcNAQEL BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwODEy -MTAzOTQ0WhcNMzQwODEwMTAzOTQ0WjBXMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwNjI2 +MTAyNTA0WhcNMzQwNjI0MTAyNTA0WjBXMQswCQYDVQQGEwJSVTETMBEGA1UECAwK U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRAw DgYDVQQDDAdjbGllbnQ0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA -2fVodgIdjzIVOxWDmdmdC56f3JOhzYXrYPP8JHyrSUQqL3puJM3/J1JBQvQF1Nbm -fVR/FmnI1FT2HlqWUh8M+G6d2a1iP1I3S08glSghPzuD6rb95o+e0f8ghWfpNt6I -wpf08wWVGWS8ONBm0BVIiLJ8rKb2ftI2H8VmCGIj+xGNXtNNdsOmkpIQiIHtzgw4 -b9yH0onBA4F13TWxOUExgcyXIeZICxR3CHHHJEyDKzGb2iKFaIzBeXxOrZs2aoYF -+A7F1J8O8CYaDU/p22g15hYPbvZ+K3idemzkkZVX35usiaa4Flo3rkHRMvFNGij0 -vvJbQsbMBAdRKsQwt5pP9LbGPywFDqxDCVx8FdFbtBlWpXfZLDSKNOUNp+Bojh+O -luJF+9qRq0ISj41nAf3tAB7djn6bvVpoLJ4Yxz90IDm4KoR3L2Lmd4xscghRffza -apcHKGqjy2G7nkGpxD5itqErzMZ/JgSN1Ue91afOtHO7dLm1nSHnriLCtW4v3NXD -FZmqz2APsbRWIazgOay+5Ai2FLRljN6ieZwMELcIAofzGY/BeaDv5hk6Rj8OBi1f -eIntvDJOXj9le/lOB8P4eW8WxiArLYktaD4BAJTIkEUvA0ckjPqkYN0AN8ICuPQ1 -Jhc0Q3bMSA5FEZs/Zj7Pw828q0/2PEABakLsgIS/A6sCAwEAAaNjMGEwHwYDVR0R -BBgwFoYUc3BpZmZlOi8vZm9vLmNvbS9iYXIwHQYDVR0OBBYEFITmpe10c7Is+D2L -GCRFfgf88duQMB8GA1UdIwQYMBaAFEaMoHhzHQlU19Afyjh95bCG7o1EMA0GCSqG -SIb3DQEBCwUAA4ICAQAiklhYl9ak9up3UBXyeayXVYsX5WJRP3xpYVBN2LFRYsTj -gWv/Q4N8ChegS1xSploVH8jkQorBep+zRj+s1u2k1NCxCAaIIakG1BNN2/W2qBBl -8JLnW4hJwy/XTLitmobqcFE9WTsi/d0V0GrsT9Ojh+Zmx3GCNGS2VWHJqM5S62d5 -iJycUIKEcWy9h2vjyKb2cAEQ9+OOtl8ODonfBIu7JwpUFAqWiBGG/r9fQCwS/9Yz -OCljZqaPi6w/w+04dA+0xRO86kd685QSvkD+hmtm8DzYLdI07gAiSxXD2sMszTF2 -C8JDKVxaIW0SV2mP2LcPdKEAKpZnW51rG4QiXjWRrDYluRAWjhlSHFVHau9vJOSM -6rCHahhGtUldaIGvruEN1TuEAuxQXupeBgQDMq/OBndrHSHwTRSEHzprrEvdFzuS -rn6MVx9f6VyEAsku23nhFuidXKRObF30b8T6t7T5JOE/KmeYcE0YGczob1YEbTR6 -rSDcOn41ymOQ4Jo0NRpe3N+Zz8USh2xzkFTgx4sxwKfyhh4peGya18klkc+fZqho -CMLtvkdoK4w6/bXxxGviakIe8OJJvKqKbSv43LoIDYgnYoiFVl1TRhvVe0uVothR -Yh3/laO/+p4WzPjFUHUtFo1yw+vOOsfA1++FvODgZa/p3DaTH7VKgRw6PEZQ6Q== +353z74lGXkwEc1n6r/M0FS1XTXhoVGMYIqK7HPCBOEGLeyyGwGfSQ7lRfT1xSkii +zBGG0Nod9cRT1CAewOSJ6BjVfkQcGEjlnVYm42nD6PMd9iFJj9Y5atPeFNvvr+wF +OFX+E8FRu8u9aEu7MIj+KCqoqBukFhFgJYX8sMbRROfLOPaCq0cSC+Vod4qR83+W +ITrQ5n8+/CC39uLY/oKgAKdVnmff595Uy76BVdYzuit1IRKwJxqIWMRrfNI+szmS +hdj0AHwgmwEGCaTNcOQyqvBLxW6qB5tc1FyV4LYv4iNftroqNQvlUbJ4UqVr55Fh +vZ38C1BQ4sWgo6FSS/B6u13clwpRzDh3H8tOMTTz1inUtg61Y49p2G8k3kNVH+QU +fRM4xvCkhFzIArgSiJ+/YUKboltSG5K28pegkk8RRMsaQK8g+NScKKu7/8ddRGE8 +454AqxPpzASij+djM0vxzgad6BB4e+iIVdj77NSjAxVAfg9GIjNHG1DZ87jLLgtk +SN2jaYsBRBRnmenslEGDtwO1SeWrzzicVfP9GRdiMLJwCkwv5P5hCzwrCB5eUPhm +tGHm4K8eXDAd+Ol9pKMySC79E5/W372wdbaO1fcAUKvpHhcRZnSusNAJkLGJYCkV +2gzTWlaeX4rGqjNVs4MSmNuMT+a0IafZeZivptxdgLkCAwEAAaMjMCEwHwYDVR0R +BBgwFoYUc3BpZmZlOi8vZm9vLmNvbS9iYXIwDQYJKoZIhvcNAQELBQADggIBAFox +3myVDr9yJkrF5+vB9gUlTv14JIPRd0OFCLcPOlHpvYKEwjRjTwT9oL3zU5PoRPX0 +AiD9sL5TOo0zraiFPUi1k5X6SoW/qU/kOJ/j5CgfChyyyit/V773LitM/cVXZGui +YX32V1zV9+RaCowC/16oHvfjMA8xNOYoYW83FgQ3GrKgRuqqVMT7JAHoDebVSqyb +w5W0G7RH3hHM1nCv51tnT1SZDn+qRBcX5faPUVARzdcRrZ/VSU2RoVIU/fPPiet8 +5TRioZFslZaFDWOLOuP0ZcOj5MsY3vQZtx2/NRgNc+iLF593YBUhRJYqfT5ePW3H +LwbZp/Rvd2kLucYd/W9WhKEzJKKvzm1V2hCDnh5dl32sZgdBzrdKzgyNB723cLR2 +cHFTIEj1Q/scay+iiSoV+VNfMSDQ71vkHqFHNhEqPFUpdF/SeooDFeQaDvYkomgr +Z9BJFtbp4kZRIEuPX+niTi0S/zwi7htiUn17wOIBcydcgG2GXBer5H3JyFnCXM1N +0jFQsuBFRj8xP71xzhN8YjA2Pe+MGYrMWiwaVMLTz8mdQ+Y2aEvOkfXFSaeNUqW3 +GYxAjEkhVCvzhOd6sD3QjLRX2qhwh8NJCJDbkTok66hno8QsHWASbaDiCMG9z7le +ci4dOHzu/buwqS4LVTmFWTn7mkd+FAlSY9Hj0WVI -----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client4-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client4-key.pem index d3587f78e63..f1f17525a51 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client4-key.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client4-key.pem @@ -1,52 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDZ9Wh2Ah2PMhU7 -FYOZ2Z0Lnp/ck6HNhetg8/wkfKtJRCovem4kzf8nUkFC9AXU1uZ9VH8WacjUVPYe -WpZSHwz4bp3ZrWI/UjdLTyCVKCE/O4Pqtv3mj57R/yCFZ+k23ojCl/TzBZUZZLw4 -0GbQFUiIsnyspvZ+0jYfxWYIYiP7EY1e0012w6aSkhCIge3ODDhv3IfSicEDgXXd -NbE5QTGBzJch5kgLFHcIccckTIMrMZvaIoVojMF5fE6tmzZqhgX4DsXUnw7wJhoN -T+nbaDXmFg9u9n4reJ16bOSRlVffm6yJprgWWjeuQdEy8U0aKPS+8ltCxswEB1Eq -xDC3mk/0tsY/LAUOrEMJXHwV0Vu0GVald9ksNIo05Q2n4GiOH46W4kX72pGrQhKP -jWcB/e0AHt2Ofpu9WmgsnhjHP3QgObgqhHcvYuZ3jGxyCFF9/NpqlwcoaqPLYbue -QanEPmK2oSvMxn8mBI3VR73Vp860c7t0ubWdIeeuIsK1bi/c1cMVmarPYA+xtFYh -rOA5rL7kCLYUtGWM3qJ5nAwQtwgCh/MZj8F5oO/mGTpGPw4GLV94ie28Mk5eP2V7 -+U4Hw/h5bxbGICstiS1oPgEAlMiQRS8DRySM+qRg3QA3wgK49DUmFzRDdsxIDkUR -mz9mPs/DzbyrT/Y8QAFqQuyAhL8DqwIDAQABAoICABRNlMcACHKJLErI2hhFRKig -meyeq0utULvtzmGJuXJFkWvDIFUjrpkiQ5EHO9nT5jxA2DvQFn9CUNDgCFuq/SF7 -5blcD32th21norXc5fKcH9GH7UaCahLOt1/7Siase8SAtOEZy9tF3nX+Ym1ku3vR -qzpxV8nVhBQukQQb5up20UVG3VLrA8O91nYY1abuzX0MvRw4F8zfTN7n0alEhjuR -7CF3cDjpVZ3pl1mCxRSLdk9PYupWc5lWTS/vzZWxBAeo50sYlqLOhyO7+3akPPfA -u1GkpXkgLwSIoJFifivcC6pLkkoGv7lm886mhVOirWByIV0q4+CbhoyQzPxKuW16 -vHSjqwQDp0EhMjk1NZ1+emTOA37E8vtE2jVCLZITPwEiRtbwdl9MjgGjQKedS/oI -BJ5sGFupfwzmY1rAjkC5U014LQVOPhvXzPxwLAH6szlrXlUOJYtG/D3qlEEnW3oT -c/DfJYhEmyhaWDnHC72c74e9E4JjpMp8cbJt9uCQd+rxd/FPtzaC7zi9+b38aktH -gp9UazWOjNkBYdxk1ULn/iVf+5hxsGYuOGfdeu/62bJnNev3YTiV2IKeQRREjRgD -AG560k/IMN/aYwJ+M7hAYfUwHA1oW5CwuzdVl1bBLe6iXRdZTfqFvR1dHTJI3xRP -6Qbbo1pSZGpsGTT84Xk1AoIBAQD0C5c8A3Nnf5j/dqRS6OVUo+aTZvztyydwo/ae -AT+Ju+InmZGvBd8qpyd0pJlGPgM1MJ+74kYzLvmFrUdlCX91PfgG+1PEaK1Yh9r/ -uMTOp62K7x65/Cp6LFXfddIywB8RCDn1EfVmA6mw/Q8lAaI5B764WHGwcdncVzhH -1omYBPXz2QzOAsGh9Z86RvMvkQI7pb1oEpp5n98MxGXZq4wqvqhIYcXfFfFSAkGg -DyMm4DRKUpUMp+fE4hjD0DFAtcgOFCog/kcMsIR1W24+/R5mtwt4zet8m7EXh3yD -xtcggYT2IPqdvYYpNqRs6lewawH/jnni5oaJr9ZJWJ7rKF5/AoIBAQDkoq6V+0KF -r9uckkaSAQ2fnMDfx15icK7J6s87pTY+RoonhzsvM8YZFYgfmKtaR1y/A+8oMzBj -g/5C2nqUjBuIbwmT1cWpoODGI0yZ4tltDHf8LSehvjvws1RCKH6nBBr3VK+BFcSj -9ZX9QeKTCSYiMUvC2A08wXOipWCF6R2X1q3/VtjGYEsDcE+vbn9PVzMneHvZT7nX -u0kinRbg5el0ifclzMhT8SawSMf0UZF+Xzt89nwV8CgA2EGmkn8aVWD7AhYBz99X -6jPt/Ny89RZSpRtiWSrH7gO7U9c6Z1bWA6IfitKMqwuutg2lUUJT77KXPVynLhXu -9FwMYS+57ZzVAoIBAC2qKY9SOfro1lEe+xqEd70Bujhksfkw5V1fYNsp8qCsJmIw -iyEL4TlS88AQuJl7KxB927XFXktfg9MunEhcoZvnj4yG5KLAz4bcTO9CvJjKqV9w -5Pf6JlycBSl+G2nhpy6Bso6X4KpJDyyN/ZJ4Hsm7GEXYTtWsSA7Bx0cx7qcibn9F -Yu79dwv+MOAvvHtvD9LnbWyW5o7RimQSL9iuD9lDuatpmCPpMDGfOHZhvrscgsxR -bZBjrquZfvbM8xpfqWJxBX0M93B5ax+mjxd8N38SyzO8nE7Btz5m3Lavszti4OVJ -QHvZpcv6bIPwyZ26ODKAj3AUoKIM4/TvfzUTZbMCggEAeL35GTex78vpXIxbPGvV -9/7eVM+C5tW/WN7qRgN/2++AslNWTB05drHKS3ViuLCLKE05la9C1Ks2MbQTnDNJ -rwmEs0/nrSdKzRiDMZl9XXuCOejWzzIZkYoC61VbXXRrkUZl0PPf/v6JEVAkCaaB -Pvltmx5iDz3ShYh0qwNrH26+QORqYokj1FMRogHmtc3FNAkKzr6t1GIRw3fWRzrj -ySm7HXlCKiJmXh5xinMxNN2yGtJIwYUsexW5xfADs7J3HtgBKz/OIS2L8xjgseF0 -wnxzM8MHNsJZsioatbsxPqB4k9PKUYNbDg8SscVyE8W4OOqs7ZUG/ESL//WrMLNt -aQKCAQEAx9ZBfjVVqmMVPp1GV61+B3LySueyI7UTrFDxKEGPQAzcLxQr41iWnkCy -MNDn1E9fkvNND2pVqKC88NZ+GDk62W7j96pUP2AqxIE3lKjB3vFAWdH8izWQwUWp -P+/yKYscaqo5wAgMNV2W4lqZUmQ6QaK0rJU2SKaXtjlnUD6fbHhJaYUWdBc8VmTb -FkYGJnpOPuP0eo/ijThdDpCVTUnekSCEdoinuUmprBv04ypn2RCjx4dpHHXSK13M -X8Gnn2API9//cDuuv8TjV0UG+yOz1aHz6ciLpQX8AxBbgetkdf/jJTQFWMiFHMxU -LBnc/Bj54Ydh01rOez6w3PkOCAugFQ== +MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQDfnfPviUZeTARz +Wfqv8zQVLVdNeGhUYxgiorsc8IE4QYt7LIbAZ9JDuVF9PXFKSKLMEYbQ2h31xFPU +IB7A5InoGNV+RBwYSOWdVibjacPo8x32IUmP1jlq094U2++v7AU4Vf4TwVG7y71o +S7swiP4oKqioG6QWEWAlhfywxtFE58s49oKrRxIL5Wh3ipHzf5YhOtDmfz78ILf2 +4tj+gqAAp1WeZ9/n3lTLvoFV1jO6K3UhErAnGohYxGt80j6zOZKF2PQAfCCbAQYJ +pM1w5DKq8EvFbqoHm1zUXJXgti/iI1+2uio1C+VRsnhSpWvnkWG9nfwLUFDixaCj +oVJL8Hq7XdyXClHMOHcfy04xNPPWKdS2DrVjj2nYbyTeQ1Uf5BR9EzjG8KSEXMgC +uBKIn79hQpuiW1Ibkrbyl6CSTxFEyxpAryD41Jwoq7v/x11EYTzjngCrE+nMBKKP +52MzS/HOBp3oEHh76IhV2Pvs1KMDFUB+D0YiM0cbUNnzuMsuC2RI3aNpiwFEFGeZ +6eyUQYO3A7VJ5avPOJxV8/0ZF2IwsnAKTC/k/mELPCsIHl5Q+Ga0Yebgrx5cMB34 +6X2kozJILv0Tn9bfvbB1to7V9wBQq+keFxFmdK6w0AmQsYlgKRXaDNNaVp5fisaq +M1WzgxKY24xP5rQhp9l5mK+m3F2AuQIDAQABAoICAAQfLTflF971F7/okK5dlUAu +rcVHyuSDTxaUWU6XQEqBKskCcRlq0H1fFRlx4Hy2Cgoo6ItA+fxlugXW8bosfD5C +9ux05O+tqE3WILFgabQJhyvaQTjdggFuFlHcG/bqKs53B0/l6FPF1Z/uhWzHmaez +4Zf3qnadq2AFsDqx73mNrDlIkfAGR1bgy6QocbhDSckjBGa7QbX0BHAQjl9imQBq +FTHuSDpF5to6kLe8UwfDdU0+wvB1lL3OIQ0T8wPqs8Cz1wuLPi6dPjc/SmoiSqzL +8RmaiJfLTVK8wiZ6NTe93y3HELAZoAh5ea5MTkjebSbJmrO6r0L+0Y8ykgnETP7O +Ug9PWeDDE15sNXIQCKtRe3QpHtJaoAJU1MGhNqwm9oKMcuSvBOV8XRuZORinTYRL +Q2ZD7czaT5VZXCQI4uHwE+KIlQF+658c9M9WETxClgUlhbzqig3ilUz3QUweaPvE +tqArjiYLsT2KtrgmsZ2DaDn2IlGSIRjXMZJ3kn6i49C0uhH2YkSZgU9/7kH2myse +3opxE1EbT4ARFWUbGgqXTOc/OSb9DAsxUK2u0eR/4yOMLQJkvxNJWQgwmi6N18iU +WdvTphNtMtmdsAhst9luwNaeJItzTDm7JeWx+MPs8f7PVOOkTz8HcBAvZnISH1Md +0i+0lBrBXbAcRK5X7tvhAoIBAQDwKPQA0uNk4Hemt5yke1jrg4B03OLimHeQ/1PY +I99hThh/RLncYaMxqsd5WkXXbjsWyGidKHYh3/cG9akmgU6D2Z16CFNMRhgBgRX2 ++LJkdS2QSuHPJlB9ERtOOiWFt7IDafB+tMKHE/VRQdxFRtvLe6pQMzP4veVXZsq8 +NNJGAQ8egUa6HDvkXzR2VDf2Kc61t4ZwT4JT6C12GnCfvXobaVkU1aWhcguoX8vI +o3UOkeracEKc/80ZRdFhA/nvPPXCobyjFjLi8WGp6PUySrVhN9eAdZaUBNeFHLdg +8urNvy5Q6mBAByEfHZeJNZbBeAEAw7S5YAxgL96blj2IPOerAoIBAQDuXazpCDXD +dG6XDZ9FS7MKBWriHUa54UgbX4JfQsOOFA8uxglIe5E4IKFSEbetlDORnKZjcmGa +SKTm0MLLi/kKrkhoDgi6HNbmbo9ZmKIhmEwws5L1TLeUdLrWj8GLgNphotOBKs1V +vQQkfh6rzovyFsMj44Xea8Kgx5ONVlB1L5pEepKdIyDRiQfxhwFox719HACSqCEa +06eFNGtUOLLqNMZhpur59bqgiVQtIZKis9juwzZID0svXBElpDrNvbWS1V5MCBOT +6AStW66YkmVWFCn7qQmNqMh4x19GveW8ajgrBSr/8GP/WXiACBDEsunWRORW57iS +KiPmC0uHlMUrAoIBAQCYTrCokRZLlJvtbIb4PY3gFw7xjmCJqn4xw+wNqHpzgI7C +r/hbjsRrrE5DZP/kJ3Fr+n92JAH/a8WDcWrsE5eSwQFBMmR5e/6ffZlLft/MHBBg +cU0SDc9/8chqbS/8xMotphMymDrCZeLvvKAQg2bDftM9d6ufNfdr3bH3eFxery9C +fmQ3hc5qAAMKhFDVWiBRWGn3ckVKJ3Ylb5E7jXQSTFaFgxU+9U/1YYOg5CFJszrJ +e+aTIRuWypOGPnpUwkluPRqgJ2TwTntMwYQ3d+/eDwcp3ek4SHXSYqrd3lERWQzr +niiakqrry92d1BGe8xdXv8Yuxn4yxkkcTUUK0O1vAoIBAQDLY0LW1BqL3B1A5m6w +QhdSxaydoz1l/cP5F1W20tDpulP6JSBmqIkQy0bbMCL6CSq3ZGLVGBQQAUwzZo3Q +AG9PncZKgy8PHux/UncejA5LfBgGtjL++6bpFXEXAzKyRhAQn065ODxcnBucx8CD ++ImQ17tKNClVz70SUzijsLKWSzfmlm/jhMXMBJCyle+t6EDXL72NZchZi5+1GTU7 +d+Wx0bY0PKji/7luob8hgzQLgEnp8MewVNxiXLyE0c0bIHR+BXGgjoOmAKN9CG3B +4ah1+l6YTXPJW+syo2u4gPA2BKxIiPBX0laA22bmV/t22vKL0dzECpSCo1JeR+T6 +mwZhAoIBAQDpLqRLxfZk2TK3wJ/bloXXsRg4TjSQ4m2Y3htVRiOQF83iERVRlAwg +9yKlyd99ux8tlYAK368Q+FFAwYTvUyghmfVTPARFTmeX0F5u+MX00WOa2FhPs3du ++ImYeQH3hg2O7qyVDMCwtqgIIuGLNwsVPqUUF5bx0He7wTwwzQmx3EVCOu6yZXG7 +Aw3qpOM2VhrtWgGP1mTONiUg5dh4sGbXX70gjG9cUpo/Owr69Q4Y8/OEyx9bzqSW +5BeVN0vONzQC+LHG5EvgNF6yOU7iCkuoDirZUrVchuAf+IDapK85TLIH8bm57LKN +Etg/x+MCoqlEQBVgnY7f3suMB89XerER -----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client4-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client4-req.pem index 5180a8697d7..224484f6611 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client4-req.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client4-req.pem @@ -1,27 +1,27 @@ -----BEGIN CERTIFICATE REQUEST----- MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp -ZW50NDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANn1aHYCHY8yFTsV -g5nZnQuen9yToc2F62Dz/CR8q0lEKi96biTN/ydSQUL0BdTW5n1UfxZpyNRU9h5a -llIfDPhundmtYj9SN0tPIJUoIT87g+q2/eaPntH/IIVn6TbeiMKX9PMFlRlkvDjQ -ZtAVSIiyfKym9n7SNh/FZghiI/sRjV7TTXbDppKSEIiB7c4MOG/ch9KJwQOBdd01 -sTlBMYHMlyHmSAsUdwhxxyRMgysxm9oihWiMwXl8Tq2bNmqGBfgOxdSfDvAmGg1P -6dtoNeYWD272fit4nXps5JGVV9+brImmuBZaN65B0TLxTRoo9L7yW0LGzAQHUSrE -MLeaT/S2xj8sBQ6sQwlcfBXRW7QZVqV32Sw0ijTlDafgaI4fjpbiRfvakatCEo+N -ZwH97QAe3Y5+m71aaCyeGMc/dCA5uCqEdy9i5neMbHIIUX382mqXByhqo8thu55B -qcQ+YrahK8zGfyYEjdVHvdWnzrRzu3S5tZ0h564iwrVuL9zVwxWZqs9gD7G0ViGs -4DmsvuQIthS0ZYzeonmcDBC3CAKH8xmPwXmg7+YZOkY/DgYtX3iJ7bwyTl4/ZXv5 -TgfD+HlvFsYgKy2JLWg+AQCUyJBFLwNHJIz6pGDdADfCArj0NSYXNEN2zEgORRGb -P2Y+z8PNvKtP9jxAAWpC7ICEvwOrAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA -liXJjRTg9O4AEadeTV7BMMFtRIKXdUvj7WBbUgim1UsbDYjCvURLnNR1dAhn3QrE -c8CHrqF58uegW540UWfqDXcvGPUYgLPPweK/j2eyQmhUL9jab0Fnwo1ALgB3VZe8 -vP6sleQLlswhl5kpJgyGFUsf14zzRA7WW0UggFZVTbvO81ymgz81uGQl760FfqtC -5dczp4Y5FeUFlfgFj/nK1y39ijYrxWM+otCvRvxSKbi0JT/FB/w9B+rHaIlwIU32 -Xg8KJKwHI2ykGw7tTTfcdiSYP24rO0Trc97fJLHggJqeC69htjwtNCgrMnkDJAsc -N5BKXB/zUXnbCrjE0NIhrbcDlYgRaD0+VAyxdbUvLtAy6pWhf4f0bkAg1b8+V7fE -Fam9iHxC8NcKmwtrt3D5/me4CX4x9j7YFvrjVY0iJefqemI0jxD/Nm3v28xiH+/E -ys7PaEyKdVZrTVvKzt3H0p9HJqwwp9ETBOdqE3Zeu24eqs59qbMnAoFaW2iG6h1T -XaAAkJyBH8ax0KVdmuc3igL+uf1Oy+F3eoPX8Ds2svre/KABfgeObAr93NW9e3t9 -u8rL0yKEzG8crx+W0KCLDKtghIumWgSrB8ezjpNrqhiOzYyE0Or65WfXpUpG1Qhj -CsUJlXESSOSAGotLPYClXYaJqZG7ttGJ26drRtlyWxg= +ZW50NDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAN+d8++JRl5MBHNZ ++q/zNBUtV014aFRjGCKiuxzwgThBi3sshsBn0kO5UX09cUpIoswRhtDaHfXEU9Qg +HsDkiegY1X5EHBhI5Z1WJuNpw+jzHfYhSY/WOWrT3hTb76/sBThV/hPBUbvLvWhL +uzCI/igqqKgbpBYRYCWF/LDG0UTnyzj2gqtHEgvlaHeKkfN/liE60OZ/Pvwgt/bi +2P6CoACnVZ5n3+feVMu+gVXWM7ordSESsCcaiFjEa3zSPrM5koXY9AB8IJsBBgmk +zXDkMqrwS8VuqgebXNRcleC2L+IjX7a6KjUL5VGyeFKla+eRYb2d/AtQUOLFoKOh +Ukvwertd3JcKUcw4dx/LTjE089Yp1LYOtWOPadhvJN5DVR/kFH0TOMbwpIRcyAK4 +Eoifv2FCm6JbUhuStvKXoJJPEUTLGkCvIPjUnCiru//HXURhPOOeAKsT6cwEoo/n +YzNL8c4GnegQeHvoiFXY++zUowMVQH4PRiIzRxtQ2fO4yy4LZEjdo2mLAUQUZ5np +7JRBg7cDtUnlq884nFXz/RkXYjCycApML+T+YQs8KwgeXlD4ZrRh5uCvHlwwHfjp +faSjMkgu/ROf1t+9sHW2jtX3AFCr6R4XEWZ0rrDQCZCxiWApFdoM01pWnl+Kxqoz +VbODEpjbjE/mtCGn2XmYr6bcXYC5AgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +2JVul/xcJ+YlepOHxJ9dczIcXEjjMOBxWuyK+G9/6wASgHg9e2SB+WS1VSeUARC6 +VkID3Jlwr1gEw4gR3lW2h5I21kdCaBfCHshUoOr9rV5uE76r9kxgyEsMUZMvSClC +eQd8VK4fUT9JEKigIJeCFT9IE9PyxrdH1xpp89jOLy40t3PkubDi8WR8dvPckg3N +juLU/6EtbrtgFMnCqB2TmH4mc6YSCeENUTvt+nSiBKZUblGDuIxu/edX3SscS0Yv +qPM5LPcNHEGeeMC5ZSfotaSzRP+x3OlV9VJNROG4brbaI+3kECtegBgFvKIiK+JY +m7dkt8oIpQc8CKZkM8Kk6e3JXHzKf4vAiWHf0Wyag3gqCukxdas/PMx/3ROi7iDm +XQN713lxhIjtqfXQZjZcRmYQwdnkaSY+H7hgAyhnavqkBmPLeMU5hffdBswrjH+0 +fD0FOIDOWNM9e2Q/qdtHxtglNUmox0ETvl/3gYRkN1I56zNan6FNzGMubilntt2z +xXQwxP4Jn+RoAwb5U9mIlaLJ73FDbl6KAvFSJHlZl34R/o1nKOOAiFSv4V+RcTMd +x49P7cyAcW+eSsIgDzqabhx1OcrFEFRtBy342w5m5Qdq62TpFmeALgRYhAarA9UZ +YY/XOglN88K/3iR+A5LO7Hdv0Q/wShokghcSdAE3JOo= -----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client5-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client5-cert.pem index 890ff5b209d..b17baa62262 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client5-cert.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client5-cert.pem @@ -1,33 +1,33 @@ -----BEGIN CERTIFICATE----- -MIIFojCCA4qgAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJH0wDQYJKoZIhvcNAQEL +MIIFojCCA4qgAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJHEwDQYJKoZIhvcNAQEL BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwODEy -MTAzOTQ0WhcNMzQwODEwMTAzOTQ0WjBXMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwOTAy +MTYwODI0WhcNMzQwODMxMTYwODI0WjBXMQswCQYDVQQGEwJSVTETMBEGA1UECAwK U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRAw DgYDVQQDDAdjbGllbnQ1MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA -t7MEWyM5qCSWFfKIxMAL/Z6GvQtoBY2GmNBDwx9FSBHAXm38jRm1fxIm3xVoujse -zxxZwya1enE8Z/qKKVmElRQypnX0K/TqV4VM2qomfDsv6/f+ueMWgAnPrbK0+Ggb -O1Yw2VB6b3qsGYgETZ9TJjKdHbuqy/hLosdLvZu/+qd6Rl8iCqgIa7jnLu1HFiMc -ZiCN4qAJ39virYvog3t57jS3awsF072kO+HagDmOoFxNhLq+Q7m1DNs6jBiU3Nii -0bhKDEZHER2Fvz5oE9re+gGde/B9mZk2sktZY3Uiw4eT0JagruGgduInvqKAtTo4 -pU23C+zKzSVHXyuhzNjaK/RtgSCTmOvFgXN4uWthcjd3tZOTMY7iapRqRe3JM0Ck -gHEAomR79a3KkYSfuFBdIew3sR04+r3lm/kB+++xSL1ypDFznbI0l2ur0e6+2r9f -whR0d2zA4pNIyl6itUA0FuECVd8Hs0cV/6IdqbCmRz+qWgyDeBhMW3miqy2KXSQm -00HT2Yqf/XWq5bfmQ1sq0oY6+DvsYTW9haVirFb1lsvFKlcAPHke23jKY3pMSkqj -oz+mJd1XpIxqzprpLjd5ayNu9FtTWSt2U8Ys693IVTne9Axh0knmGnLdIwhYTfJC -8lM3lNCqqYQ3eLy12qTPUclObh6dZtjaiSP3ohdXL0ECAwEAAaNrMGkwJwYDVR0R -BCAwHoYcc3BpZmZlOi8vYmFyLmNvbS9mb28vYm9vL2ZhcjAdBgNVHQ4EFgQUwRwX -0H/tt7jtTS0knaDOucaxGKQwHwYDVR0jBBgwFoAURoygeHMdCVTX0B/KOH3lsIbu -jUQwDQYJKoZIhvcNAQELBQADggIBACettwt0CjSG4u1QpjZhgpVI0EIJ97QKld7I -ahSaM4G8SHmU1ROf8kzam1lHVMEZU3YOIkqhyPgZPtcMlg9/CFvgpgHsPTkQu15l -e01Inf7lb3OIj/+PdX3e+P8wu0dRGjH4u3891V+yjkUUQJ8EbUw1DMI5iRiNTGuY -oLi1Oyizpgzu+dsI66QM9ntQPVrpMY27gDHgOSGqW5Mx1pTFL3/Rr9AKtMeh9i01 -YtTJ6LlmjhTIvzVmbSLVDgnR39Oq5LOw0kwK56AErEsubIeds0V+9T4M6wBjrReK -5ubolfDo/Y8Q2ZZONmIPPCh0M7gNdn14tfzgIKXoGWPU0m62PXZv4Syt+McK5twr -V7kXAcMp6MVSZCMqP/FXsUO13NLQLdU/a6BRakzCnrL9ulkB+RWmPKWQYvA4wSEO -9U00a2HVTfNSogMFcABxoRAkJEb951X+rn8L1EKJ1b4HTASbVEE7iYyktl9wWH1j -JXjqRTSgCNRKeezbEkXjJ90PzvCVfQmNxTipKuJjxVzLZpklZzb5uEV0Nx99bOde -QSexYccwADvCcptm6K77BQNcK9jhqfDxpFi4y61dHurFNIQQOQotHg9cKM6obmIG -2WzBBy7e3zs4yI6/jyx1Zh+dZbIi2+m8c8xSnZZDgVTKszAdIDFs6kH+j8zkLwlv -qvounDzC +zivZ5IrYyoJeDX0Zbl/cl8rKE0LbmtD+QKZvQXHD+gstXWxPOEFZVxO3BuqmvBZ1 +MaYNyPYA9JyyP+pO9cE8RrTk3w9xMhv8dYWJQK7na9W9RTSXs8xhPwiEm4XuVgqv +GfK/EvdxbFMhgzExOR333TkmXPxrMm5xPWwV3RsTCjNVW7nmdPHXmchuTO7lQtww +6fETqc1Zqv8TO2x/uGZiwAzNYbueWHyzL4Y1UJ7D0mnNNgJvvxtcrzBTlGLLhJ5o +0+zVQLaOnac0WVk0wDhVaxd/gR4bYN3ixvuFbbOaTroFVTMVme196G2FkJI/05Pn +D68r1yUpvuNDjkBbuBO43PlsFKLhPU5twfu4TesEb2WZ0VsNxn8Hc0Ne02WwHsHa +Fi1N0oXvVIeGvvzevuTkjPbh2aCaQX9qbhLXjhgXNFAXQq+qI8ECCWC4LruPL1Es ++ZM2thQAg4k8GY4U9Q8zX55Ut7t9T771QbHFOAupuTgZJSz0jC8JId0m7NOOqtvu +4w/GqefCt9bp+wlQDXVOgi+S3GqKM1OuIbs5XQJtNfi6l684ptAFHSHMHXvfGLHI +MlKdobjNqEFVK9qO/3YvnTUyRyaUA61aHxraXWBwvcUByXwXdgbuz/M5XAi6fr0V +Trw9iUSviDvu4HwDo8PCw9i8xYFtkr2k1kurWEsFZMcCAwEAAaNrMGkwJwYDVR0R +BCAwHoYcc3BpZmZlOi8vYmFyLmNvbS9mb28vYm9vL2ZhcjAdBgNVHQ4EFgQUENIS +25yehLQrlL8vC+DpkNE7urowHwYDVR0jBBgwFoAUiSo9XUmDdI1vjLtMRUK6G2RZ +kKUwDQYJKoZIhvcNAQELBQADggIBAADH/LINEwB1WUQ2Q/aHiNv1ZyJ+ifgs9ewp +/t0uwS+53ctcmJ6Sqeni3/UIdLYjpcdBbl1XpqTYuGF28C2YZrWKFB0HaOiH6D8B +zcGGAkIKFqnhcJxyl37je+scZ8Wk9b04L+X+miN6cgIWm6rQezDF3rs1xvAVBqTM +YPIk6sBIgHNJy4D3S5KdhqNV0/8KY6T65nGFdEq064qOk8HvS6DyYOs22AitCD+L +gcWGJHJ3BfNASbRrT25zb1HLUIFFbFIGaPFd9GbiU5hGb9MgUzX44q+WdXoEa59a +6y9ZcidjEqAGP/FMz16D831YpqRBherZ09ztWXeTfv4NxauisLuoqpOr7CmpQ+Ct +O5t0cUHILeNBFR7rdMOmDawpEcOSGqcJHdPH4SjP/LtgQODWiNys19Yp5afbM5Lz +IjLjq1wAHVtSvPHjRhnZSq0SiU1XlDmu1Em3HbFe5RmqL/lcLe7/U10ddngADG7E +XgPE0jcvl7rYASqYuTbKd6Q53QYx0K7xc1n8mIRJuAofPwl6Yns/ytvw0+E9TBS1 +oGb7j6V/k+Xd77dfJ6fckJXPg7Fm3GPO1ax7FNU51sCrvAHsMZhiWQa6pZzBEORM +4yI+DSFyskyWXCPth9r3UqHQXzX86LRkyDWg9l6v3NWRSI1j/e7dZds/U/sg2maq +css4A+kM -----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client5-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client5-key.pem index 4abf9d1d980..aa65de6e26c 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client5-key.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client5-key.pem @@ -1,52 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC3swRbIzmoJJYV -8ojEwAv9noa9C2gFjYaY0EPDH0VIEcBebfyNGbV/EibfFWi6Ox7PHFnDJrV6cTxn -+oopWYSVFDKmdfQr9OpXhUzaqiZ8Oy/r9/654xaACc+tsrT4aBs7VjDZUHpveqwZ -iARNn1MmMp0du6rL+Euix0u9m7/6p3pGXyIKqAhruOcu7UcWIxxmII3ioAnf2+Kt -i+iDe3nuNLdrCwXTvaQ74dqAOY6gXE2Eur5DubUM2zqMGJTc2KLRuEoMRkcRHYW/ -PmgT2t76AZ178H2ZmTayS1ljdSLDh5PQlqCu4aB24ie+ooC1OjilTbcL7MrNJUdf -K6HM2Nor9G2BIJOY68WBc3i5a2FyN3e1k5MxjuJqlGpF7ckzQKSAcQCiZHv1rcqR -hJ+4UF0h7DexHTj6veWb+QH777FIvXKkMXOdsjSXa6vR7r7av1/CFHR3bMDik0jK -XqK1QDQW4QJV3wezRxX/oh2psKZHP6paDIN4GExbeaKrLYpdJCbTQdPZip/9darl -t+ZDWyrShjr4O+xhNb2FpWKsVvWWy8UqVwA8eR7beMpjekxKSqOjP6Yl3VekjGrO -mukuN3lrI270W1NZK3ZTxizr3chVOd70DGHSSeYact0jCFhN8kLyUzeU0KqphDd4 -vLXapM9RyU5uHp1m2NqJI/eiF1cvQQIDAQABAoICABA9VC+WZUd3DImeKkxydIeU -QccT62XqRVwxtSaFF4PaheNoh8DAOr+kDtRT49vq1BDCfzD1tWoV+Ugb6aY5fV4M -g2Rx3bHGcge7XLhL/cunvoHMJ7RZM1OttQgJr16dHXiDvcfIXzqHDBB/FMT5RnQv -BJ1sn28ZYTO1Df4SIcj6kKBd/U1siL9f1kLbbm2yPEAkX/Fel+DhYyBX06ij/CXR -z+aM+pPlxPoXlPsW8Nk+WmAB05t7HwVRmtMTF+iPgj1WCb3HYgr3otZC9xfM6+oK -xdhUwgBtKgezCjufpFYddd96dslfCueCRg81SyQ6TxMAqXj1aFb1v7crIp+ltWEH -rpOrp7Z+VH1vWJHZQHRdlNKeM+JEnzwegsf40+L92jItTrNzQqtZwHjdwC+LvtNu -0iXPg5qGku/0rRJpIp6lBeinuFivnm1YFow2Z/fZT3kbai7aokQyavJixTwIH+Ze -hjkBOhMnv37Mn8EhtrlJYPO2m8VqaISFZOmbx+lGb4eQ2i5dVm/rZ0j2y37bMLEK -xe6NcXprMTlVTChVCBGSEJ1J7M0RFOi4I6YIw0A6mcZcEtS1MX7PonMLzhQjlyQt -WawoVhtYSIhpsa+0AqetkmBcjYHpu2vXyTGP32c6DiZDSUOz46jpP/XdcbghfZyy -SyFgjTCyrnrHCwo1odwNAoIBAQDhIlEqPsmN07/6IlwSxvEChEAe6ePsQ7qDC1xQ -Xh42e6E0GaiTeb7px75E0KiumDvqQ55m7QFcOWFJiVn3tBpekplN2T6aBIY1MwYw -2aCBO+SetdNtfeuaFjkO/RfNNgaKN43jfNfwpHeSZFIQf0v9l3xz55+pHkDDYTRv -Z1sAWTJniuyZdLtXgXpWAm31UMF+lNxje8gtg96hQfAQhfQohJ7d8m5IRzcyYbJZ -bguTNLWFH8d5RHEO6AfyOAzv//ExoewrqE8HcdydqJ0b6a2PqdOh/VOcmqDCsZvv -QTt2v81OK/TxTEaRCXNwh6ueTi/6lEr6+JIQWGm/ZzEH6WQTAoIBAQDQ4m+QuvY/ -uMQWlL82Lw+SuHJcjsVQML/PMvDR35+1G5IYliuHEK4URpRZBal9wXVzmqROR2gP -YK1KwUVcLCVjG3jxnCaCHJy634e5/8Hlj+HAPpeHrHldtvkLFjXNT9/zfo0mopjr -P58A5av8XEjjgTWLycD6ba049DntN1xpYCoT/QxdbxKcKDIKOBFfVQ5MnCNEU48P -hFkXxN+/j+im9KHOpz/qnen0WotvI4O6KHBOM48J8OIMpdgwi08hLqKeR+u3Fksm -mbIT77ivWsmmFMF9+Qr8pzW8Qr1ThUDfV3l/yTpXM0jLsp5L47ReueNn9St5muhg -y6STis3Z24HbAoIBABwagZFJdzWaKkvaQC4j4XAT/EEfp1EV/1FeeK6boR/h/PSZ -A18tgnSCd7D1bykkXc5XlI16poHRs+BPIgj21ZfaqwJ5zQPC7tHzRaGReSTIyjg5 -sGPppSB+B69kTrVSOizE1JSCQB6NGVQP6+KdUWodn+6ECn09Bo6dMsN5XnXDrLQa -NoDNfplk01WrRfiqlQztHjnY+NsdrAvrREF7x0fLjl1cZwHkyfIjGfVp9E3y9Xnh -xq/PqGFY+6zz7EGFbKFqeiXVA9CFGSohI0OZvw3BAGcnTWmuL8U+EJ4kQX0IYmOQ -vynkAKcYS+3PMdA4KGsaWeXMY8dgkgrPoeI/8zsCggEBAIXgCmb1HKv8bFiYnvBI -Oy/q3QjrTBS4GcbfPC8WptI5SGpPnLgk/ZzrG1ru4RFDM1aTTykIZO4uJQKbWivX -82JQkjQvwxJy6hRu/Rs9ivTpQWOn+hAvf12LzpPHeVYMElmJy0duwJlNnkfB63eD -YgfFoR7lgLRAJNwy0wlxCN5UjsNe1FTxwjipHmEkTCYQami5TXUWJMNqLHuqZ0js -H80ZTrcWxEVHTlGY0nK5jUx7bHcCXnOdmhNHG4cJUf7PV9Qf/jfKpwGnUdWrN2/T -vRefh9+IBP2m4S62i4cDCW/bLXiz7JtEuCwtX3z+BHUNqGzRWjrHCJY5DlKtjH9L -Sy8CggEAO7GVsRaSB8awfFBK0qI33WQvW84GGeMqhEIUeLsUplEIcw05rsV+TxFD -m8YC21yNEUUHICpEA3w5ZPXb4e7ZX7WRZwWpkF97sklX5tJBqwa9i/x/zhB3Jz5z -dQeAZmLotF1OFczw77lAwxqGTMEsuQ3iZdXMNug1tx/Uv2DehdB2b0kLl9/G2MXf -KhfsH2+08kdg1pkXmQqO27pUzX6CNNHe4g2qM/t9qpPj1cJEyIgL5Wk9GNrWhJLI -gwBOf97eEQ9qxTIJEWxLUVbGMz6ULo23hmZi63+wdlQXXQbrqQiUw2iUuWDvgEX0 -GmrseRP8qnzBQ6OdHEn48aOT5XMXGg== +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDOK9nkitjKgl4N +fRluX9yXysoTQtua0P5Apm9BccP6Cy1dbE84QVlXE7cG6qa8FnUxpg3I9gD0nLI/ +6k71wTxGtOTfD3EyG/x1hYlArudr1b1FNJezzGE/CISbhe5WCq8Z8r8S93FsUyGD +MTE5HffdOSZc/GsybnE9bBXdGxMKM1VbueZ08deZyG5M7uVC3DDp8ROpzVmq/xM7 +bH+4ZmLADM1hu55YfLMvhjVQnsPSac02Am+/G1yvMFOUYsuEnmjT7NVAto6dpzRZ +WTTAOFVrF3+BHhtg3eLG+4Vts5pOugVVMxWZ7X3obYWQkj/Tk+cPryvXJSm+40OO +QFu4E7jc+WwUouE9Tm3B+7hN6wRvZZnRWw3GfwdzQ17TZbAewdoWLU3She9Uh4a+ +/N6+5OSM9uHZoJpBf2puEteOGBc0UBdCr6ojwQIJYLguu48vUSz5kza2FACDiTwZ +jhT1DzNfnlS3u31PvvVBscU4C6m5OBklLPSMLwkh3Sbs046q2+7jD8ap58K31un7 +CVANdU6CL5LcaoozU64huzldAm01+LqXrzim0AUdIcwde98YscgyUp2huM2oQVUr +2o7/di+dNTJHJpQDrVofGtpdYHC9xQHJfBd2Bu7P8zlcCLp+vRVOvD2JRK+IO+7g +fAOjw8LD2LzFgW2SvaTWS6tYSwVkxwIDAQABAoICAAxYkuU9LUs/MEulfKLf6bFL +SvuJSuZD8p3DebEVE3PDPJW2GS5Yt3Vf8FthLNfhjmYJOBUbKiZ7xa7PJKhRyH7B +cV0sKw6hn2YYN4ZgkuohtAJTamwxZndkVrCLfiC35hFrOGb121xtw/l7tiNh+IhO +Vk5GLVMpSu2vLcX+S48WUNu7z6GcI9mJJ5XCzobI8cLolVFbQy3atXefASz1MJ+o +hJoAJrTMztwfMM0hnL9aZ5f+4Fs008GH5tFhcyd/Zu6d5Y1JAVROgXOCRIboOLE/ +iHqeZ2xPDTf2MuDFWw2w1QXrl7UOhDYpbNCh+WF9n14QktMA3jWYnWCE2Rk2USEK +2QhsScNgA7dY3S3DbSK1ZiyZEgbvdq6EYy5I46jNXqRrfKfuD7J04dtO1pmBWZy3 +a765FMiejbHrC1xrKtSnvGj1oJ23P3TRiEL6tx3QF4bAulWE62ULBfDgqr56pPbt +KOfPinEFa/14+B4a21iFzsoA6SWPt+i/k00RyRHw4X7acTd8lsavHcT8PGY4wMLX +cNEombZn4h/0oCqAmBtoXYwctcyQwzhJNzAp1GFMG2gdBPhXUsWcwwjNNzeaewCq +BO0YO/DFgj1qTHZwswTfppJMF+BTE/sol4eaqOV1TYV48OfYTdo+6dqfH2JkaJyy +PVJAoZtKKPfDRlrHMrE9AoIBAQD1oZMIi4Nm4TpdOGsU5O2keDW6DXdoxpOI6u9H +YqWZlSZ57q2u+8JkoVJhH/jq4l1QuXtUObpQ9DYOpPwPkt+ZblUut/ZrmMo+gs81 +RtIv4EGmgFmtrXgmkYGsoNrIu4/ayBPDwGz7Z+gEta+gxhYbxSZgb2ZOZzH/c2my +3CvfgFiYyv/cfVvynjprRZoxowQvJoMCEEA9REZdO5T51lshQFcrpFU2ucQNKV60 +DELV6uJzhL0EDvg9IqP1CxZhdCsTifE/SNVepNWXFegFGVsD/vxmXVxJYevSDQZY +SvGWVcgyuQA8Gdze6y3jOUlzCQ3v7D63PgAPS+yrGXLd0Vz9AoIBAQDW39oRb1n2 +ves41cEz7qPNiGmc+zhzyZE30HFnl/RxREPfhzvifTUk3YTUPdJGpcgf+sMoPQ/R +plx/ZrFPTyV7U+svf/eoPlpNnVnZ1m+C/nN7AjVwq04GRHKYD3lTD7KpoYQ0ICGO +z9xFiGbK4VRNIiNpyGc3IZBmrwB2wsGn1L4yfFZly3Th4qiDmNpR29rgteQYWSNJ +hSqsCFcqogJfppP+QKRGf00uHDa1AGQhB2eFykyAniQw52FcdpUdkGqYiR1vdir7 +5XckNixq7yyRflo8CftHybyHipTBZVXBzdvDpY5mxANPcJlSPhJxqruxK9FzNYv6 +OiDlFnFPwNYTAoIBAQDLByRbWcXhEaWGTA3mlVsAKQRvppXemtRzxYzGOnmaure0 +7V3OVvZU6ysGcMD0lOqAwO95rMWeCF1uOVzHGqWLUrFCDni2cvIoPUM8TC+LtxdM +oqZ9cfbwGjFPGR398Vp0ghAkKzdpjncu/WYPw+ueRJT1/N5ZS979wM3LM0VoVdPl +Z1WZUFzh63tqE3viL1ZUCscau4f9nCN1CLyYzV9B2sayogB1XZL5Ngq7U68i1iUb +SspZQm5ZSfqvntx+7OB2I/yuTGtg8WpCma7QGCcs5GSHz/9qAHBFjNKDxF5v3rO9 +iUDybIYuE8I4IK/fT8qfV2x3Vd4CxsT2n/Bb/KOZAoIBAQCyIwrqgWIYCQNFLxDp +OOJbtse3+R2oAGd/2jLqKzPf4zRR0A95iUvRsEQ6oLX/E9mBiabZaSYALOdqR3yf +v4HXaI8F5hUvSeIbjmO7EOeJteGPDRm4uowI7h4CqnFuxwjbUKgFfs3TU8fNbXOq +pnv5JmAthpLrRcwtFNBRpMxfkyPLPwFxiubvjbUexE3ap2Yh/SmIdf2nKdtim9eH +5KALJFJ06qpGN6uImqNQE27vYvAUHs6lonVmhaxVt4mP5PY6VxIsIc8o3eeUAcV5 +MafFRvcP50aHEVxXEjCY1KXv8fZLkKkp0T7dUQALCqLH0T+hdi5CURYm8KHsylpO +QBQ5AoIBADXNlx5bXEU2lW21wyWG4v3gqvmnDgQG66RXaHC0Rgpn6BW7pose40fv +I82yWviabk7wlvaQgXllzZ5vGL43/8pp+wFpRUl5nX8P1ZA8pRkRArm1vLxQRjkM +90j0M/XhTnC8eC4qKsQx8dPUH0SPkHJAd2tWcP9Q3635zETcINkpsNbsB/QNGBZ1 +JefHA/GNZZBrjFMOnvzEbEbqBdhwlZNRlZxVq/svvNzzv12jFQpM7HVy95bATiZ8 +x7SQlDxCWOWApd9VpM83JnPWCfu0Yi/HHPJd6YdueMhRSvUpLs1sD8fs9bFQ7+OL +erqwgB4b3yDlXijY75pPDxdPf/0qA3Q= -----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client5-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client5-req.pem index a4a3c9d6531..6ad15243bac 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client5-req.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client5-req.pem @@ -1,27 +1,27 @@ -----BEGIN CERTIFICATE REQUEST----- MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp -ZW50NTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALezBFsjOagklhXy -iMTAC/2ehr0LaAWNhpjQQ8MfRUgRwF5t/I0ZtX8SJt8VaLo7Hs8cWcMmtXpxPGf6 -iilZhJUUMqZ19Cv06leFTNqqJnw7L+v3/rnjFoAJz62ytPhoGztWMNlQem96rBmI -BE2fUyYynR27qsv4S6LHS72bv/qnekZfIgqoCGu45y7tRxYjHGYgjeKgCd/b4q2L -6IN7ee40t2sLBdO9pDvh2oA5jqBcTYS6vkO5tQzbOowYlNzYotG4SgxGRxEdhb8+ -aBPa3voBnXvwfZmZNrJLWWN1IsOHk9CWoK7hoHbiJ76igLU6OKVNtwvsys0lR18r -oczY2iv0bYEgk5jrxYFzeLlrYXI3d7WTkzGO4mqUakXtyTNApIBxAKJke/WtypGE -n7hQXSHsN7EdOPq95Zv5AfvvsUi9cqQxc52yNJdrq9Huvtq/X8IUdHdswOKTSMpe -orVANBbhAlXfB7NHFf+iHamwpkc/qloMg3gYTFt5oqstil0kJtNB09mKn/11quW3 -5kNbKtKGOvg77GE1vYWlYqxW9ZbLxSpXADx5Htt4ymN6TEpKo6M/piXdV6SMas6a -6S43eWsjbvRbU1krdlPGLOvdyFU53vQMYdJJ5hpy3SMIWE3yQvJTN5TQqqmEN3i8 -tdqkz1HJTm4enWbY2okj96IXVy9BAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA -nMHx8YWaXuz3XKbouq27h4/2PWl07evoXfKMcG2F5hon6zhrk/scEbKIlLOlSZSE -L1zVZ0jQsWRFsezHVGo4e9ZEmNflWO1XSgHOquQW1NTkyHPL3D2duJTlVLus/4Yo -gqLGgyfhU3JWsFGD9hnlku9z95HdnT8BFXLCikb1KsaC8JOFwYAgxMITb9aKzP1h -eX2hVtXz+O5TZNYSTIXHs+9AtRjvX1Mhj0L4gCYQueAjS/f5jU1/RhqOf10ZLe+B -SbCirZgwvlErULM9KI6sT88OZlEgfoLaoxN2X/9YGGQTD8JvByFPryc7tCTpV6t4 -pKpUQ3vTS3LA6KUQYEHFAXg7u6/r11dk1k2Ni4q/NhPpjhSsooE9qTUDh0jUNW62 -BuOhkxyDFIQVgJqgkD2Uq6GswkgL+GX6fUBwYNYZbdC7tv7ssWX7yGcn0jYZEAIU -8SPhkW+IFqQrG4zhJKh9ohasAn7wV3AfSQBE2P+cgOCasQyTYQU2++1FBVe319lu -IoKDLRAU0shc2vuwnTbnWOlshBBD3gKfSsF4YxV+5zOe550HW+19v8Mlj4mdRaQ3 -Uh3SWGPBGVy+bJgBaSOjoOoUSDPPmzIPo9bGCZ8SOut+/jsZ3PJ8Mn1b+geBugt4 -wmJiTGQXbRunGYV0pft038caNzVJFp/uSd2O3PegwLs= +ZW50NTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAM4r2eSK2MqCXg19 +GW5f3JfKyhNC25rQ/kCmb0Fxw/oLLV1sTzhBWVcTtwbqprwWdTGmDcj2APScsj/q +TvXBPEa05N8PcTIb/HWFiUCu52vVvUU0l7PMYT8IhJuF7lYKrxnyvxL3cWxTIYMx +MTkd9905Jlz8azJucT1sFd0bEwozVVu55nTx15nIbkzu5ULcMOnxE6nNWar/Ezts +f7hmYsAMzWG7nlh8sy+GNVCew9JpzTYCb78bXK8wU5Riy4SeaNPs1UC2jp2nNFlZ +NMA4VWsXf4EeG2Dd4sb7hW2zmk66BVUzFZntfehthZCSP9OT5w+vK9clKb7jQ45A +W7gTuNz5bBSi4T1ObcH7uE3rBG9lmdFbDcZ/B3NDXtNlsB7B2hYtTdKF71SHhr78 +3r7k5Iz24dmgmkF/am4S144YFzRQF0KvqiPBAglguC67jy9RLPmTNrYUAIOJPBmO +FPUPM1+eVLe7fU++9UGxxTgLqbk4GSUs9IwvCSHdJuzTjqrb7uMPxqnnwrfW6fsJ +UA11ToIvktxqijNTriG7OV0CbTX4upevOKbQBR0hzB173xixyDJSnaG4zahBVSva +jv92L501MkcmlAOtWh8a2l1gcL3FAcl8F3YG7s/zOVwIun69FU68PYlEr4g77uB8 +A6PDwsPYvMWBbZK9pNZLq1hLBWTHAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +SGJaGaSkfsUBSqPbkbtyE2ndHIY8QMPvJAMB7pPbA/MlX39oO2U1AraFatN/Xzpd +0uQA2m0UsXF2+ScrG4rd14YkU4wpcT2lIplvFFzUkwfYLATRYrFV0FCru1n28+Ve +TyUG9/tlf/jnNZwWtB5AEbCWmzXSjXUuwXdn2oQ0z+3dxdSnpOrDkzyYeocIMnMJ +pG0qdBd+hJrK6snJNli6EfsmmykyCSLorFpnkm2uKUwdTAD2/MlxYdutjk7tqKw5 +wpYi6Wqt/euyQ94Ri4aICZMBFk5+zDdaEOKVQZ0aUT8RWt/aD2ksbTrclDICOe84 +iG+Nf/CyNqSNqsUugCSCxN3UUZgLKEyJb8Pz4N/nFFRBb1aiZ/5YVRTYtZ+k4tLb +s2exxqVpDg3M5G5bk0iJ8DBEjO/yKwobc7HAtnTAEIs0HGord3yzg3F0Y+5ecQAg +9ESOptz33EBkTHxpBmt0D0ACa4CTghrf/Id8imNtdGU9i7MeEba+iUVAUP8VfhtL +FJPRR8aVaKaLc9uCAiiHuRc+J0EHAwTOKKTK3Z1mkYO00kMCiRcru8/H6ibkrkV7 +kRL6NvAc7CsEzSDPkFKAZhQ4p6AcfNC5yRiNLG9JB/wQgBg8v23Uwtk74gOXIaQN +WUjwzdYOljdcFOzmoDMzyfIFwSXcO3dmmjqOwh2HNw0= -----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client6-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client6-cert.pem index 22e57c3f2b4..b83c380a3bc 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client6-cert.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client6-cert.pem @@ -1,33 +1,33 @@ -----BEGIN CERTIFICATE----- -MIIFojCCA4qgAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJH4wDQYJKoZIhvcNAQEL +MIIFojCCA4qgAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJHIwDQYJKoZIhvcNAQEL BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwODEy -MTAzOTQ0WhcNMzQwODEwMTAzOTQ0WjBXMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwOTAy +MTYwODM0WhcNMzQwODMxMTYwODM0WjBXMQswCQYDVQQGEwJSVTETMBEGA1UECAwK U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRAw DgYDVQQDDAdjbGllbnQ2MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA -ld8aBatdDBGEVARor//8lYQWW75VzNsTaNXzPFjitA0G4Dx4F1pcV2mBxoR/1aqv -4pdwku7FqWxqT7DfEzVMoeBFzMi6Vlrn8YFM7mT1fJ7r/jI+JzDc5yoOF7S963Sz -yuwWHFKrkyXftaxhorv8upz3qTNFvui+3NkCD2N6FNb++Qz2ircB/KQFq3dIluP8 -wL6zEMgyKcgYhr1MLPfe6JSqU4PtGUnKjEpFN6Fi3Ubvd6NjKqxIwQd46xJyFi9V -XLbMOhCtKXXcaO0Qi3rzzS+7seCINX06bAnUHMTMlWNmYw7b09+w9e+8yjICsKZA -hNyyNxA+CikhlcZpP3oMn3T/slIVekuRYszlxD0KLMbZ+lOC/LmEPIrAcfhCc45M -ys7s0ijaXO58NqsGstGHTSz8WW9YP6VWvfcd6OA2sXDAiLDkkgk4Sh6hGGOkRfc2 -RmMmKPgMdLQOlBC5HZ+m+EOsTTQRKDPs6HsxKksxe+mOkAnPExmpYgquSWEHXE0g -eo/BAfLMbYsob72XAe0fMXjRIGKQlJXv9hFIKhcqwnGaOs/n1d4uBkdvdof9RjUt -p+JQxkKIU72J960hSWj3YmVTzdN8HtHK9f78eenr7+HmB4sUFr1qzwA5h+Y9+6Ou -FUNM7+WfXWfqGHaMNbEVZA7saaEYjyR7FjM7+lQikMMCAwEAAaNrMGkwJwYDVR0R -BCAwHoYcc3BpZmZlOi8vYmFyLmNvbS9mb28vYmF6L2ZhcjAdBgNVHQ4EFgQUmRBH -PAQ50KDlkzSbJg9KYGQsbJUwHwYDVR0jBBgwFoAURoygeHMdCVTX0B/KOH3lsIbu -jUQwDQYJKoZIhvcNAQELBQADggIBABzYUj4S02LGetmZJbMs+g1BmD3mwerE44Ji -X6cqOlnIRtjnSjRZqDufAMh9qQvwn1g9d2GyFkkk3FDM0KRubW4w26CV1g4CtduV -CiIibpoVrUCkoHTbJ3WEygzWby1AQ4crgQPFbJwvyzsnjBnLO9EORyL+T9QAx5/j -vOOB/WSOwpu+LEjw3PHlCR5TuDfN8eYoO7thfqlCQ9A6aTVFizYLxfaQi5zluLyh -1UFgBJ5ZEzlkem4chvhNBDYSDnqmCw3X553MVGTa5pE5Q/vv0FzPyPUG/iml7DiT -xsGFLH3tumWw47HTKQit+9spMg29xiiakRcr+TlXy5DTmnR7ERYoV5jHg8EpJqid -dcacvjEVRYblmfb1lRrZaULqXE9+w7JqP4V0r6vQeiftGQ1Pn2MPqcxMHqehr730 -pp7o4L4c9IpyuGs7SQAf1LCQMhb3H+iQEeVI1kExeWcPZZJs6Bhi3fEb4/m61W6f -6vJd2WJBv0uxgMHKrv3k/5biPbVwz6yA1oCdwEGiCHyCFmyYNC76qrbU+A6F3zqs -fmTRlS8PLZ7iJJw6g6URh+Dvez/+PMq0KlyHKP8CFjyWr0i5rpeNBTTgnfCrKiJQ -TQdEC2+KATwYlzB8DXeC8jcj+L4K8Cp6XRBIQnALnOmwJcRsTSVggVHxF3zuxVf6 -SXfWywZU +uz+3VyFhBD6K68mkDM0LJfRfGbuD8tSmHhtO6+XQzhIUiANW+A1WdyD+zWky0QsS +vl/2Ds1PmjU7659pkSD8Sidjdz3/TP0eO49nHinLjRQa2Oawk6PCjLIcpJ+A5eGb +Hno/oiQBMCAPpVh3sex+5yPiUQN62cKRWryv2JJqJmEgvpjC92SaIjf9M3mCsxMf +an0CZj6hNcorxXQNnF4JZRPQ4mMgBqgFS5Oz1YujHiBUN9ZoIGmS3HZ9LYl5lL7o +GxNXluyIrIw1kiyQrx+wJFdYwTBeyCqw8wmf993tHRmvpJ2ZFVXTbYqpj2Qkom+J +EpCRBqCmK+/uSgLOAicS/wR2eiYXkw2rYDIQ6yCyeW7nnaFVWNAFNSdAW1Jdr+Z2 +fKMolhYAcpeMQAYN5wTqJBF9dDwMxbOcdSh1wPZcenuO2q5cMJcn3qt3+SigNKcQ +BvqtZ54GNGq9h24f11+cqco80v4WYxrmXu8bXQfgtX07UVvqkjbG7O4HtbGDvOlw +KO7d1kOj4XUJdZbz9g5kaGufN4hlBs9JzYNOZdywNwBiPxHHE71Ht2ihfbBKSl9x +4Zse5YhPqToacWd5FRW+If5EpHkxxW+f4e61S2/8fnn5yHpMX22HXNBmOUR3vBrl +xfkje22ZuEf5NfB95aEaRZABmXQaHKdAVQeaAT9TvPMCAwEAAaNrMGkwJwYDVR0R +BCAwHoYcc3BpZmZlOi8vYmFyLmNvbS9mb28vYmF6L2ZhcjAdBgNVHQ4EFgQU0ieN +0CTYiMz7HJs9OH2U4imSzN0wHwYDVR0jBBgwFoAUiSo9XUmDdI1vjLtMRUK6G2RZ +kKUwDQYJKoZIhvcNAQELBQADggIBAELD77yWem26DIWG2Mi6q9KvLb/d8bOFN4mg +SYYekdEryyYCFhbb0P9T7GKj4KxNxkZaGgwXl+PM8SH7FhqemRCwCtHMVQiWvC1x +XLCrGiep7Dx+vvKHoCdapQp3uRfsZMHSZbsHJgQRFRt/jgP3jFcrLAa9tD8Yza4y +lBhh8pUROC0edlqom+BkLYA6A7FWA89x/NZL6wiyPKqRh2SuACszrlQdj4oqdgIF +pAILjDy4fLaCGGUgu9kHonV0xTjqSdFXiSeImSkePOvGWrh2B5GqRbanPHDSR5VD +pIhKZPpnvUBNhzoAMv1RFbgNs9m3sYqYChxb8IOGiY/3EG4rgygVez2yu16v5WMU +PIWwSv3N+oceG085dHxhn4TcDamdFrQHssp7GJBbyGEINHQRiZ4cu3Vz2cVNYcKD +iFJZ8vVwU7IZOEQeww6DT+gL+wqSgTyDvEXQNbYupFYPZXSBIaygH4eHa+PqPnNL +DJTpPAlwNRB2+eL3bZxvNAfwqIY6xgwnLBr1QrosmfMKgkswkg8gVoIpIRGACzk2 +iY818Jn+IG/M/aPF3p5dTsOqH3bQmz4ZpoLB2dytqkNDGKSAPPGOpe4MDyx0prCH +GWDjEjn4xT9AjAJVpTWJCENPzFzwA7byApuZwLTy/5ZBbNJf1K/JwsqXUhHKENb2 +NzMKvQCT -----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client6-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client6-key.pem index 9bf833a3a79..e56fed5dddf 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client6-key.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client6-key.pem @@ -1,52 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCV3xoFq10MEYRU -BGiv//yVhBZbvlXM2xNo1fM8WOK0DQbgPHgXWlxXaYHGhH/Vqq/il3CS7sWpbGpP -sN8TNUyh4EXMyLpWWufxgUzuZPV8nuv+Mj4nMNznKg4XtL3rdLPK7BYcUquTJd+1 -rGGiu/y6nPepM0W+6L7c2QIPY3oU1v75DPaKtwH8pAWrd0iW4/zAvrMQyDIpyBiG -vUws997olKpTg+0ZScqMSkU3oWLdRu93o2MqrEjBB3jrEnIWL1Vctsw6EK0pddxo -7RCLevPNL7ux4Ig1fTpsCdQcxMyVY2ZjDtvT37D177zKMgKwpkCE3LI3ED4KKSGV -xmk/egyfdP+yUhV6S5FizOXEPQosxtn6U4L8uYQ8isBx+EJzjkzKzuzSKNpc7nw2 -qway0YdNLPxZb1g/pVa99x3o4DaxcMCIsOSSCThKHqEYY6RF9zZGYyYo+Ax0tA6U -ELkdn6b4Q6xNNBEoM+zoezEqSzF76Y6QCc8TGaliCq5JYQdcTSB6j8EB8sxtiyhv -vZcB7R8xeNEgYpCUle/2EUgqFyrCcZo6z+fV3i4GR292h/1GNS2n4lDGQohTvYn3 -rSFJaPdiZVPN03we0cr1/vx56evv4eYHixQWvWrPADmH5j37o64VQ0zv5Z9dZ+oY -dow1sRVkDuxpoRiPJHsWMzv6VCKQwwIDAQABAoICABApNaok12zEustAinVoGjmZ -O7v640+q1AX7BDaesxuNSA+IFb8W7Fw0sVzRzuMge8ToZahCfSgO28vqG7P7jF7s -SzqQ7p7/QIGnWr9ePaaJkJMhodgiCcAXpdtijxRDylt4Z834EC5W6C+z/fdJZDBP -WtlxAPVcLbCs6e704CY9JNOAVVmR8HHo0f+yZi53OTsjlTWbEOW2gQaOhRdYRz43 -QcKtuDSA44mLPLmJ+po32vzArqHcYCAIiT54B7tisUbFI3Kjd7i/z9u0rJMN2Bpp -Kkk9GBMEhPc/fBL6GcGGLXVDFetb9L4gcvAuwmGAG/b3wXP0mWwJEWkQMys4IwIo -oJ+FRCGtkUq7cdwtwX+mJyNKA1g1xO9YfLMhmKpUaxcLvsMBMb2ZO33/sIH0GNSL -csuIVAiVzwODNo9TbbgKylkKUjbVC7XnQwdGOUrl7wGQdI/jTjbfwHR67S7S6Zkw -Ctobr9+lxz9pUcR7U9QMAKwCheNcx+vVbmeU0QesL2pVWKjCgWJLI4hmK9epGrTQ -ssKmuIV4k8fDQ+4yOL6uk8SbGRLKE/tjG5IArloh+0TWlF537xhFu1U48CT9i60I -GN4FYes+P4LkJToJP4w1Dh83VuHLJBR6xoK+tb352SaQHONv0nKTnxm9PeRsADwV -Fu1YNitG7P86cpkvVgMxAoIBAQDHPgalQK/ld/aNbLj9Nx+M4Ok/eukNnQYjGstW -ScW4OPHhNDDS90X9vAA3sKK6qwpt2B0BTBoSb9zo34G2Zx4IJGOgzVlnRKb202aM -I1tSdt8ULWwjCXyN4jH078FAn9RKtA0T4V/Ps1R177oB3P5ohOCAWAFatK2ASd9y -bt7n5THi8xcjNikJiMqyHXzVMIB6owe9AXpoY89KRJm8c2J/tgypYSJ73JbMTkQ/ -sOQz7G6BpaZMc0IcjqsyhY7o4kcM+G2VYvoLvvstAJkLgWShZkHEFARmOfgVhNw6 -Fps+0VpmCCqoEIsMXA1D+Cr/XVDJ/O8xQLOQP6jhVeV2ySGpAoIBAQDAkKc5Nuzl -SQwnQNY68h4n8NPjpgtxwdxVwBvYWiBNo9wol+RjltgXb37ks3/SE9eoBJOLuzAM -agA8pR3hk8B0Jshpe+ucBwT2k5to2lCSWVcr+VdMjKcm4s1ts+zCacbaEZDsbBW1 -J5qXxy/78V2rLWcd4aWEu4U35mM48V4W6RuC7cqHWYargXM12U3aTQX/ZA71EONw -VjNYgOJ4YvBAoKaaPVBEjLftWQPCwuFua+dC/i+1QYJUhoxVUd5ySvz/P3FCVrP2 -vd/PHi02mim2lGkrl71bSeu3DLbJFGf60/LbsdK5/nOp66vqCJa+KBQ8UEqK9Pb2 -IzktLFWjBTqLAoIBAFY3+Rg5zeQ27XJ1wfeRiw2jpkvaHE/Py4AVcDh+5Bi8S69w -rlAcwSTz3gQ7Y6/zpCe0hZiyHfYsgiQj1DXimZRcauCC/FU72CSnhpDOHA6rcg2B -OnJPAJ9Faujbd31HrM5G7AHWXWe76qi+fHeh8lW4ao4fhzaBSTNixFb4s20WOWhE -WZbbMRb3Iv/A0uxOeMqZEhgwM5BI4ML6vKxYIh+x3/jMF+gRpQ/0LoSBP79YaNmh -nT0oA0voUR0jvpv4j1aFiqZCloHOu1LWF2RrMejGH1+CagTywPBO9h4M+lFMtxnA -HvVrc0B99R+cnL602ukNk1R08z9QzPv1975Xe0ECggEBAJ+A8lLtEQrXF+8HwKvw -01PIIuslJcOjjaulxirOdIV7HiitsCthzjqTSyuLF2xerTpqGAEdGy6dOvcA2iB4 -r2hGm6jsUXvbQJwyJf+THjwrLo6pKUuqEeae5QRijdF3ppQJCt8apFx6oo3oGvH4 -utrIb+qLdvvcC/wCpNuM6p/VlMk9yI2WdXtobZMEHX2eYUJrkgwiYhIyBLPhhjWr -1k/iAj4uXWd6m/tIyVmw7OP3Ewcl30SnL5puHJ2rg7NuM+QFm/4ULVtLabB40YCx -761hfz/xn5KXR11HdbxXX82fdEHQKLmRcCMmqC7h0GNQdXqEE0rIoCu/f2PQnlq7 -QQMCggEAZBCnrS1NtlP4KN+tbSfUssxvnU/Lriw+VzkGclsw5QMRjeTTvcEV0Bh+ -ylvf7xwX0nansg+hRbB0k9wYmkasqFPSOhQYTQ1H+N8oHgoOEDjZz7xkP5j7h9qb -K51uS6qoRE0qroI+BDWSom+9MPcnk1eyIzfNm2hk5ymbOz6lxSoSI3HdoxtAFoz8 -D3h/7o9dBMZIjBSawh1M/YLKYemZW/FyV3EDQqi5wbWV+C8vZkAiIohr4N55MM1A -BWZG9+PdP5kCiacwZqUzX5vKyLTG9d8Fv2GQ3eCzMb2aRMKX61Z224+XuLy5/rsF -jbMZsV6A1MRCzhDoVIaUcb/2RUtChA== +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQC7P7dXIWEEPorr +yaQMzQsl9F8Zu4Py1KYeG07r5dDOEhSIA1b4DVZ3IP7NaTLRCxK+X/YOzU+aNTvr +n2mRIPxKJ2N3Pf9M/R47j2ceKcuNFBrY5rCTo8KMshykn4Dl4Zseej+iJAEwIA+l +WHex7H7nI+JRA3rZwpFavK/YkmomYSC+mML3ZJoiN/0zeYKzEx9qfQJmPqE1yivF +dA2cXgllE9DiYyAGqAVLk7PVi6MeIFQ31mggaZLcdn0tiXmUvugbE1eW7IisjDWS +LJCvH7AkV1jBMF7IKrDzCZ/33e0dGa+knZkVVdNtiqmPZCSib4kSkJEGoKYr7+5K +As4CJxL/BHZ6JheTDatgMhDrILJ5buedoVVY0AU1J0BbUl2v5nZ8oyiWFgByl4xA +Bg3nBOokEX10PAzFs5x1KHXA9lx6e47arlwwlyfeq3f5KKA0pxAG+q1nngY0ar2H +bh/XX5ypyjzS/hZjGuZe7xtdB+C1fTtRW+qSNsbs7ge1sYO86XAo7t3WQ6PhdQl1 +lvP2DmRoa583iGUGz0nNg05l3LA3AGI/EccTvUe3aKF9sEpKX3Hhmx7liE+pOhpx +Z3kVFb4h/kSkeTHFb5/h7rVLb/x+efnIekxfbYdc0GY5RHe8GuXF+SN7bZm4R/k1 +8H3loRpFkAGZdBocp0BVB5oBP1O88wIDAQABAoICACymDcaV6/dCHogIpbhzsAPV +2FNfdiAR+qZVJqVzQC3S+9hOy48MRyDS2k8KlZZpCIgig56V8DQ6G1acxWRYtC/O +YpZNTzIBbRMQp6r2llXGhHxRzar2sm4wDkpmyiqGeCF1TvUPlsTt8C0iAjCHzt64 +nL9qkAGatmQnd9qxVuRd5pvr+xlYgLRGG3cJs1OV7LjMpCTTLEeSNIu5l4FAnwbe +CcHhlwJfUBvsVUZHLJcDaHGEC4InCmDNVB3mmPoR53CFVS5kqlZSfHelbr6DVNHl +jgFK0l7yZw0cr2tAHMkClfIvrg/7ThXhqXrKiz28ULf/hsVIyzbQ2EYHky3KOWny +04O7/NnOkVHs+XUyNC4sv9nkcd9ntKkvvUPPK0U6vbW7IasC3jCh5LMyZjHYwgmK +hzxEBZSyutKWn3RWncarwQ/1Vbq3HjbkeVTipXAa7Bny17wiAeZMZ2GqQZ9VcNQ3 +YJWDgxS5shwcEo+71sC4o2HjmWKcPujmt84XcWc6yphPbCpbwwswaQD5MaZxeDgZ +OUhF9LfslzNrwwoZTFz/Qhy3TOshF7BIbUdQnWLMNdAb9ccby/0WgOmfD6V4t99N +ksb2nWgvvK3isycs6HHVP/fgv+yM9cKGs66JoH2Jm+SInUtpR5Gv1aGeV97/9WFd +JuiHtHQIty+8n6GDTscJAoIBAQDuHCBeZ+pVTyW6wxjd4OD2keuDjM3Z7X/UgCoJ +kR87Dwjd8SHEw8QaH8vvGYBNktFu3KbQ1TV2OR8gAdnwlHeI5V2/nIVX0UBjQM9X +GC3cmzsMOBAem0nuYXZG9yvawwPUdZ18fQc2wAs4GqL4uKaOeuCefNyK5wKfvX7M +sA49D45gvLUhpfkTeM8HK9UQwMfMg2fFBzZifqTIG4OGkkAeEY+rkJTUxnvTuuFU +dkXXF8Qe+pSPkbQVQYYRRO9Wk0i16R6VaYrl3vvi72w2gEw7iQya0A1bHZe3s7vv +jQuz8h954kcgLYCqsOm/mj3t654jrjW1Z5yRjznTUJKrKMh3AoIBAQDJUVCp2Frm +NgzrZXD1QrkJ1qCRBHyVu7FikXqNszc9lLD5y8YWRnhDtGruOQ3DYjpuD/DMrO2P ++iBTambM3mJt6FE8TkXHyMzLoJ/I8SMLMbLNdDpsj8D8RlftwIESiNu9DQfMle5l +8jxZ7R7usio8HysVm5u6fsSmYVUZF+sWLLAUqote4HQxdvDup9A1q7onVZUYfKnK +mCVKqfdqFDqMKTSHOngxA5wzQWdMqdgLxiYKPYbkNsJ3dhXQwJjfbyDQq4X/foec +0wG91/WqsLiMOZLsQBiGMgOq85IqGBByl51QnkT8drPEJsXX6UCHjQ7AYHe0U+pe +JTa6nMfk2AplAoIBAQDemJa+CuFrvSEldowBqOBGQeXtRi2PBNNTAjnKVcvfd0+v +xGPwQZ9LWNxaevmSC6YUwNyGMDvZeM8bKf/nQ7R32w0IOzgA/9L0trrezfDZ4piR +9LtFEaFM4/ohn6J00Yj8DrQak/uxeFlEqsoeQOkcG81u/IVkqU+vrAQlMJUvCiLt +VpzyhunSBUdtidGW5tIh49qXvAVgkMpVdDtCC+k68unf1rr8K03Jg1RxlFv4F/S1 +jUZi7TBwCqBd9pbU1b3MqdF4loFOImZSIceFL+2UXqvU8pj5zDFwf+s6qB3/rGM2 +m44oi8FUUS1EfNpWWMWuz4bQPruE7GN/pDxpHChDAoIBAGZF5yLCBTvazalwuXf/ +051J6fyCOQCgf7dBjO8b0r54IYhlm1aJqmNK7t/jIDjYWDK96mkwmOeB/JYkAeGm +QH7xfQOUCCM8wb3Y9hPRXSo8r0ds+plYVoRTACyKlYfi+y8JxaKLsLcd3scYjZRZ +8tbkRrENgom2CRU1cVP0MLvtK+7zzSYABUdz02aK3l3FxiZhdgMgoemIbbmGq2i6 +qhu2ezcP3yuXV+06Fs59ooIowf6Fz1d08kpaNS7+CSvJevRHjyWDFEX5dHMLmYSD +jt2+CgP3c/4IvpBEeUblPsXYfFUOcqGHdD/8KppLpzq10H6vA2EbGH+HjEw1hd+e +WXkCggEBAIq+MHXpbipKp7lsxsp/moeNXUojLfiBcQLU+zauWBdP3kTJmDd7D3Vy +MoN4zpvhboue70RP1ZZG3yp8s9yR62XrO+44DrKKIhVitpa1IPSavFovFVl/vx1H +F6PuZvACBF7mRZa23h9XSizvdfycEDf3rZidLG/Y9IarrLF3HYjClBktJTtm0YaU +QnmTSUdNvRBCs6/BGQNXY76VB5AHNjeLjjuLLbxYF5XsHlsdacgdMv2ShmIcibrT +nSMK3RdRdRt1Nu1k6v7MkGcaSky7zJIeu6+VftA/7bVKBcnvb+iGZSUSk2QRTCGW +nT+c65hmPp61jcBOgCF3CntcIw4eEDc= -----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client6-req.pem b/tests/integration/test_ssl_cert_authentication/certs/client6-req.pem index a83e6508dbf..1723c8c7273 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/client6-req.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/client6-req.pem @@ -1,27 +1,27 @@ -----BEGIN CERTIFICATE REQUEST----- MIIEnDCCAoQCAQAwVzELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEQMA4GA1UEAwwHY2xp -ZW50NjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAJXfGgWrXQwRhFQE -aK///JWEFlu+VczbE2jV8zxY4rQNBuA8eBdaXFdpgcaEf9Wqr+KXcJLuxalsak+w -3xM1TKHgRczIulZa5/GBTO5k9Xye6/4yPicw3OcqDhe0vet0s8rsFhxSq5Ml37Ws -YaK7/Lqc96kzRb7ovtzZAg9jehTW/vkM9oq3AfykBat3SJbj/MC+sxDIMinIGIa9 -TCz33uiUqlOD7RlJyoxKRTehYt1G73ejYyqsSMEHeOsSchYvVVy2zDoQrSl13Gjt -EIt6880vu7HgiDV9OmwJ1BzEzJVjZmMO29PfsPXvvMoyArCmQITcsjcQPgopIZXG -aT96DJ90/7JSFXpLkWLM5cQ9CizG2fpTgvy5hDyKwHH4QnOOTMrO7NIo2lzufDar -BrLRh00s/FlvWD+lVr33HejgNrFwwIiw5JIJOEoeoRhjpEX3NkZjJij4DHS0DpQQ -uR2fpvhDrE00ESgz7Oh7MSpLMXvpjpAJzxMZqWIKrklhB1xNIHqPwQHyzG2LKG+9 -lwHtHzF40SBikJSV7/YRSCoXKsJxmjrP59XeLgZHb3aH/UY1LafiUMZCiFO9ifet -IUlo92JlU83TfB7RyvX+/Hnp6+/h5geLFBa9as8AOYfmPfujrhVDTO/ln11n6hh2 -jDWxFWQO7GmhGI8kexYzO/pUIpDDAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA -aQr207af+Rbh3xeTB6o7xA8okzQnya3E0sBHqXGTWGvDOYg+yMlkR7/06YWcM1uu -O//iyRcu2lfd/+NmduxG2RtBT+Nn91WSF9UZH37uIKZIBsj5zaYhNFMnCLjtdpgO -lL0HFPU0IVR41Fq+vSGuSiOLLjnNG42/irGJYcvfEQ2yNRYQ2/y6GV/puGjvH/oN -np+ucn7ae/sLlpQYMT5OZ8Nwy/2b5hcEC4ekuubGeYtnWx4is9hxz8IeJZAxRHJI -xxIDkhP82elJpTMv4dsZYTHYHv++M/WbWb73HiAlD9g0BKYuzolbCu1TYZzbEz+a -4/lnbkR6FkKxFlP0JrTJSceQW3d3iEuER+n71Si7thwk8Bh/np7GjbHSs0ZnT+NB -ua59bE2YBWpZEoA5IgfRK+4i8AtzyJqDPnQNM8DEBN/BLqppGLitw/nMcEC2nWT+ -b+PIB907kkikBUbyZmjF6f9Fhpm35WQuCubVz86f+BbGZLwfGyVnl7TLxXiVANI6 -DXqptFt3ppv/pPRzg473pUZutjzFsUbcJNal2FWtCH4tV5lTWaBeypC3PIzahdkA -PwF1IxmpBqmuWIHXIqId60g0LLah32sT9w4BaC0rpUNeuG2yTeXz3RRjG7a/x8xl -6m751kWb28j9L3ujZI1fr6E1RKVIhe0OC9wU3D/5J/Q= +ZW50NjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALs/t1chYQQ+iuvJ +pAzNCyX0Xxm7g/LUph4bTuvl0M4SFIgDVvgNVncg/s1pMtELEr5f9g7NT5o1O+uf +aZEg/EonY3c9/0z9HjuPZx4py40UGtjmsJOjwoyyHKSfgOXhmx56P6IkATAgD6VY +d7Hsfucj4lEDetnCkVq8r9iSaiZhIL6YwvdkmiI3/TN5grMTH2p9AmY+oTXKK8V0 +DZxeCWUT0OJjIAaoBUuTs9WLox4gVDfWaCBpktx2fS2JeZS+6BsTV5bsiKyMNZIs +kK8fsCRXWMEwXsgqsPMJn/fd7R0Zr6SdmRVV022KqY9kJKJviRKQkQagpivv7koC +zgInEv8EdnomF5MNq2AyEOsgsnlu552hVVjQBTUnQFtSXa/mdnyjKJYWAHKXjEAG +DecE6iQRfXQ8DMWznHUodcD2XHp7jtquXDCXJ96rd/kooDSnEAb6rWeeBjRqvYdu +H9dfnKnKPNL+FmMa5l7vG10H4LV9O1Fb6pI2xuzuB7Wxg7zpcCju3dZDo+F1CXWW +8/YOZGhrnzeIZQbPSc2DTmXcsDcAYj8RxxO9R7dooX2wSkpfceGbHuWIT6k6GnFn +eRUVviH+RKR5McVvn+HutUtv/H55+ch6TF9th1zQZjlEd7wa5cX5I3ttmbhH+TXw +feWhGkWQAZl0GhynQFUHmgE/U7zzAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAgEA +Gewd4gSxvJJ1LIKLVTLeMdEdJfzsS52Kh3BCerku/C4ZKcIyT49dTbi6l6d40bHJ +Cs32Hwps8/qufhwwWn0r/wyww1Mgfr6ccMgKmrz1VbgWmD9owDzlL014ygiDk8yi +LwfSLe43NFrFC/FcAJUd/P8UEe0/8GcHjzhU/zqh3VAL7RzSL3k73CsTFiDaxtQL +8qISpA0lYTldMx5RaN9COWi0rPFv7mJAYRXtE/Cb+T2hb53khOiiRrJYIEJjjAhj +g8p9FOzlbXdpfoChVk7NA90CbCbxrQ2BiUqQAVYnGhntzHMwR9YxOYjvjnuiHuHX +7+4Vheda88TciMJlj0TC2e1mXBo182n/qfETeI26MsEOs3DidLT+ygM3woFQyIrX +3x2kDlvmILKg1mPGhqaRwwzCmk5a1TVdDTRo9VkOvR5/tbfG3vHlgpvBtUFCkEjX +HOyRh0A3TquofUbtx638lMWscBLqM5g6VO+Hytk6zBmq+8caJFNTOeTHZur04ZLM +SWfkIwl0B863owNuq4KxXI3NvpCc5LtGc9UrwVoHSH/pv6tbKEX15Y0ERL5/e33M +GT0D00cPnWAzmYQpYzHQ3Dj29XKlIxWBkn1QvrIFyb/T+dld1efZ3HlQxZEQvOsR +McY90r+HmVt8uCioYnC4DmchWlSX1MJe/h72udVbAXk= -----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/server-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/server-cert.pem index 1ad3712d547..073c6485bd2 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/server-cert.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/server-cert.pem @@ -1,33 +1,31 @@ -----BEGIN CERTIFICATE----- -MIIFpTCCA42gAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJHgwDQYJKoZIhvcNAQEL +MIIFZTCCA02gAwIBAgIUBfEMZ1Z/4weV13ryVA9qyNTPJGwwDQYJKoZIhvcNAQEL BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM -GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwODEy -MTAzOTMyWhcNMzQwODEwMTAzOTMyWjBWMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwNjI2 +MTAyNTAxWhcNMzQwNjI0MTAyNTAxWjBWMQswCQYDVQQGEwJSVTETMBEGA1UECAwK U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQ8w -DQYDVQQDDAZzZXJ2ZXIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC1 -l+TO19J5ZBVs+ZVzT62wM3JRh+CzO9/ogc5A0ERwVsfBFQYz1r6pGHHTxM2w3auu -ehyALbiW95eVvkpu0QyOd+CaH4K0DI/reotYyvAX5+Qv5tnHGAwFKNeDnNG4hpCR -MO/BlohZEh3Ppcm7jvMVpPhSM8ojZwSvHsEMBVb4QinvpvROLIDrA/zPQSlW0S0A -ZfvV+cjxIoDjTsg/M0Jn+6VNNRrAemx8aKv4TyOh2b+zYNa8wnlhdspTgjCseMeC -ORzDOKMP784p0xWFwQzaoS0HYrxLoijiwIw6/I72ADQxkupW4OY22W6qjheDe67M -33/KZVZ9lssc7QKGBuvkxgsMq63M+VI8umY3gsIRLJKEwXkqviIXNCqw+5zMjsRB -L3UUBjHibD1cHpc170xf4IOMa0Nk5svNy0tQ8OdTC90t2ni99O5E+MOHNTn3mr4G -Chv/+PVlWjf+2PWksmhNdTBkE64JXSrf3fb4OScbkhwD06YWVPtak5xv61QNgL9q -srARcm8f5jrAlzqU6rYHnCkjB2A011DLh3idwxZfOtxT1jfu+NMP2N8xFgIjtp8y -iVRQopXMA/HT/3J/INO6SLo3t5nBRjphDsaOcp3tOlAUYa++2QXM87+QKBlRBgC2 -Mr2iUonqPJFmAa62TudYGoBKqeVfz5yFmclUGZlb/QIDAQABo28wbTArBgNVHREE -JDAigiBpbnRlZ3JhdGlvbi10ZXN0cy5jbGlja2hvdXNlLmNvbTAdBgNVHQ4EFgQU -q1dxD9JZn+JeqZOIKHi85hq3J20wHwYDVR0jBBgwFoAURoygeHMdCVTX0B/KOH3l -sIbujUQwDQYJKoZIhvcNAQELBQADggIBALywV91PI6aClAL7LmZjVq+IF0AeSH64 -gqx4WJsOr8sRSu7SXea85JW5qHrZByjdH4oqxVrugji0hET1M5tqExsiQSBBH94W -kpPtr1UbJHWdKPKRFe8iqnFfSaTUZwA3gNJkmdv01gPGrySb5O8AhvGqCzhEQ5Iu -PH9Fu+xclKwF7UoZgfTWsQgw1GaNpSg7+86J3F9vfOzphsth2vKxGYVxnsTxbKtl -pwiy6jOwhUeix6z9+OgVOfmIxj6OvNRsGHRvlOumo3NyVMGrnABnIjBLj+k8rsn8 -wzKdck8K6ttj2fXQOdTywwipN6BrkMK9AJS5a+0ylKsyWG/scjbt0cMY1uvxLXJs -/KOw7FUy/FfdnD0BnYhmiIy65guWSjtyw+n5QK6z9OXTfcgeYZyKhgyK/HLxfMtX -PNs/DghZrI8vqX+oGW+HTBpitRThxCZ0gBra+OZRidF0Rqd4nBKwFJGaYk6B7uF8 -9oek3IHaPEBV5bqknHH8/DjzXnDpFZ/hphkBgmzU8wquGiX2sfYiQUQlcS5b1t05 -VYtcnOtIhbBe/kixcDtyqRCSCPRdrgg4vOXduOpUbeejmGpjSPe/KeVhMfP6JHWt -rxfwkTdN5xtlM6JMrbN/NBWtIrMsHyM5uwXqY9LnO19mHVYYzEZqSCaot5yOMvzO -BMSyrrc+OTy6 +DQYDVQQDDAZzZXJ2ZXIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDd +l4bW8vca9/iFDASgmGxyCQvEK5roWwPzE/DXdS+mMJM/fffyqB7Cldm1KW2ILDYU +6JGnlYdxvWLOEa7n4cIBrQ2za7vfDU4LzGJTBN/9C2TYGFP13FTfZi2gZOuipT+M +pkFCfJG1zMWR13yKNxiS6Ixf76c6ADUyt7GmRcuKSctKzM7hUARfeko5iBN8aHpD +nxYa6xSGlHXHDzHHEqEpsk+iQ/eK6o3EeElg7v74sZSzqleIkEfFLiRIPNvuAwEE +IRIQg+k/2S673kdANlyfszXXbUGCYU9lOOPYgAEFgb5ljSXZJBTW3uFlyIONK5I4 +EHgELy9Hs0qnqttXCx8d5I/eI/+Zx9GPF3x8gkMi5wQePiBpqde20kRYDvP2eGeZ +eFTnKYpqCshVAJ4+1Kg96f4S9+GZ1wfzsgS1lA7+9qr3qTvI+XnKhC2h3bqyWS+C +BQikgADbURgZT4EzhXvq7fSCFHFTA6xZzXNEVO7DkRrYbvkq06bIl1Ov9vcbCN4I +zOuJJMxlk2Dv3C3mUox3HmcOdO+vvtWnZt3HbFY7ZPCV68ObSf77YD3O5on5RFQu +hk+AinrsDIL6NiiVzALXBL+e8flkqZDzRk1mGphVXGcRP6nn4VtrfN+Bmbm8pu3m +6aYuqSX6vQXb7EHW1tAbvlfbxIlP/Hp5GoV8zqI/tQIDAQABoy8wLTArBgNVHREE +JDAigiBpbnRlZ3JhdGlvbi10ZXN0cy5jbGlja2hvdXNlLmNvbTANBgkqhkiG9w0B +AQsFAAOCAgEANvZ7QDkHIKRq/g4GPkuiU7DN44/TW4bOFe7rDC5S4z5sh/i/Tvur +JYW7m97vLui5PJf6Vbd7xyl5MFIiz2KzoLi26rlvYcI/BT8mIG8jMg7pJjp/wJGa +QdCxdO99a4SIwg7x8pvWChFAOij5e6RhrIvsEB0LN5kKRSnQ0sW5khfsdFbn+3Iy +VwyvvE+nsCqE+KK358EMHicn8FVD3Ze+YzckX0am9DbshL5+eVQ9nOhUV2B8PcbG +SGzqJF07wOBwCdcn3eY+V98SQqrpGC9tCXmv8qErfkq7pkUGWq15d+miF/gaUz+Y +yDPwgi1pephBJ34IhLUUk0IPZJ23uVv/zfB+SpZ9/5pjsmnapR3Zf725jWrhjeT8 +44i5kNeVCvZPzQO9cTOsLXJbWb0vqRzKsvuSvffDQZql4bMMvhPjMibqCiRuSHO/ +yPlWiJjhkZz52DPJX6+LOeP2pFfUe2TR6IqcFPfUs/bV6aD2L/s5UZfZXWS5i5FR +I8uvcKOWL7NBbdY+NVE5aT7DqfhaRurjp61Aym18FgXLHpDHYV9IpkU34+A1MBUi +bzHZRWhxZMRxYezC7jE4zsZ5CQtSq1miDPcDaeK5vMd/Vdys5MIekqCfKUh+Cd5Q +gfC2QgNgodcWRF2plNgA+3E0dUULR3+1s83gGWGC8/UFW+9dtYV4nv8= -----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/server-key.pem b/tests/integration/test_ssl_cert_authentication/certs/server-key.pem index 8097d371e0a..067adf4e1fc 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/server-key.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/server-key.pem @@ -1,52 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQC1l+TO19J5ZBVs -+ZVzT62wM3JRh+CzO9/ogc5A0ERwVsfBFQYz1r6pGHHTxM2w3auuehyALbiW95eV -vkpu0QyOd+CaH4K0DI/reotYyvAX5+Qv5tnHGAwFKNeDnNG4hpCRMO/BlohZEh3P -pcm7jvMVpPhSM8ojZwSvHsEMBVb4QinvpvROLIDrA/zPQSlW0S0AZfvV+cjxIoDj -Tsg/M0Jn+6VNNRrAemx8aKv4TyOh2b+zYNa8wnlhdspTgjCseMeCORzDOKMP784p -0xWFwQzaoS0HYrxLoijiwIw6/I72ADQxkupW4OY22W6qjheDe67M33/KZVZ9lssc -7QKGBuvkxgsMq63M+VI8umY3gsIRLJKEwXkqviIXNCqw+5zMjsRBL3UUBjHibD1c -Hpc170xf4IOMa0Nk5svNy0tQ8OdTC90t2ni99O5E+MOHNTn3mr4GChv/+PVlWjf+ -2PWksmhNdTBkE64JXSrf3fb4OScbkhwD06YWVPtak5xv61QNgL9qsrARcm8f5jrA -lzqU6rYHnCkjB2A011DLh3idwxZfOtxT1jfu+NMP2N8xFgIjtp8yiVRQopXMA/HT -/3J/INO6SLo3t5nBRjphDsaOcp3tOlAUYa++2QXM87+QKBlRBgC2Mr2iUonqPJFm -Aa62TudYGoBKqeVfz5yFmclUGZlb/QIDAQABAoICAAE49rAsDa+JOAFZN/pg4lqw -4nzhxUWeZjazwj4/kBOETfFZG65gYPvBRBeg20WkmaCyLbTWsO8bqWa+CK1gT3mc -uslCh94kIanBH4XA0NJ2U8hNcN3WpeMySWxstnzVqaVj3eYPkARMkd9fW7cQH6NI -qle3OhsFhj54sj/17duGDoHBx6X87VPSYOwfc5cnV8Mc5hZv8mPHP0BFON3K02Wz -CzTYtBZMt8T4l7vkjLSUNinX1WNIZJpkKrkzTXV00IxCyJSBafO4NOHm8alarbFl -4AXbwd9GUzNe4QqY0xZHztcRkp73xvG0j4r2c+0zN64RqpwJapA8f9RGdYjWvMhD -8gB39KWCKjuVsFMnDrepmGjb2D5LgOlY8UTaG/9bA7U8QwJeCv7c8Z1GO5LxUIT4 -OcMoIkC+ooEOIrnty2AdWGVa8yyDXNaqejE1v76ZLKhM7lZxcmAJ4YU04gGhBp6k -wGxQHFPdfe4A4d+pfNsz8WnBKGihZJaC5EEef8loKuu84UezPVbup/YZqgfYWmQd -FoIxiY3aTre8jqEzSDiff8oKmjkSB+ab+ypCXxzzTDO4QV7BItdaV+T1ok5fdGqm -+k+M8pe9Fj35HTuBItwsvjPlvow5pt+qIe3YNjwgBOdrX+AeO5PDscZbOscAenr1 -ulCE/49ZIvT5kfNX9h0dAoIBAQDqI9hDGigVYChG4js7auJFMMouo6k19BbcUCBH -2jasHUI9MZrUiMZRY574YydPMA9jIzoNE8KuJUT5uvjIROYb3HbT2TxgOZhVCYr2 -Zr59AUtg4uIiXVqCTioLkoxc1PxnC+UR0kL162yS67w0xy4cEL/1RHSN3Q4NG1rZ -IC1Fzb5miSVReRbWLhdfxakcVdgYH3hwyRSgQo4H1Hjmi6oLAYGYDDgGwVPKmolC -hr0NBLkoKzXi015BTM5rcbBbm6vbrrcchWg8Q79fpc5tfASNdtoe1UHUFoypX3km -7dpyOfK225p/Ko4NqcpqsyKA1WcEswjwd4LL4muXLCmEu/6LAoIBAQDGjCLsErpG -XN5Ehl5iIuMVlGgMYoKG3qQcq3pJAsupQxic7p49JNPVqQHHhUbWow9p+4bfOGwp -WIg9NiX0917Jh8yiyNIR9w6LdD3j2EX826McRhTBCZb58Q4/wxYZS63XDnguX202 -WE5i9s3RGmPt7YlX6V1dyqrp6zpsU56tC0mnsuWjWwtXa3vdFpV4uMtCitWphvr7 -b+bEFtAf9fxzFfVwd7D3XhZXqB9ADS3UXNc68nufEd/arWaWHESTmuam/DL5FStq -nrMT4RYvlxGQ8TIjj3K99JKAOvBQtE5OBycP7FKgl+HKa6W4Caa2/2rqqTajM0/l -v/zuVZSXYaiXAoIBAQCX2CM6cU4pfrDgN3t+bzTK9ndIg+8LBH+G5ZfpwUTPnjIF -CZSF3SXwRpEkOPmXS23PN1tAe943ngf7hRwulTA+RV3yyeFz7iYyj/vFDKDg5OQD -s2BANRarhxGRa7sHwUDnezlVs2ylm8ZQCf2GpN3saZhz8a89CW1BZsomIc36hRHD -4ZGIIuWChWW3QLMJo0p/anlre5yayk7eGRdHPLBMbu+isdnr15kFve7ibAtS1AtY -V52cusNXyf3chGCBKJXt5ILwRjxxSOZlm1wieNli/P88G8WTTARxG2+wpXSTu60C -lay+Z9S0W7bgN8hakQs8aveTK45xtydbZNKQOHM9AoIBAAlJGoUB5zZNN5Pq2NUs -HoSu2SLjJImcNf6lQmXRvKag3bDRNBNV+pY+fjfPkt2M6LajLc6Yu+4/FrgOJT5M -p2Ezea2PSMvQSxzYheB3B+boertix6uzgB0WWf0/aXQrZujs40//5IKrJJdaRYvm -+Q9ykX7MCjLNvKqN9sCENKKRKZOvOTAnnOMswrE3UixAMDlfdtij2G8T763yBy/H -eFQFjeIBpwQaoV7eHQAI3cDVyrcWChQJaPe4LlkM32Qr1Wev2c5uYAZvf56JY1k8 -bnTh4t9o4QvjOUrH9t8/X34ktX34JnEeSVAHMsvln6dlUKHC4ixFxRHQpcqbtARU -a+cCggEBANs4GG0pXMXyxhqHANzvhukWQ6Q3shwGXf5iCInW4Ti5JcxjzkIzeLE4 -ou+EE/3c7YJ5xXCPPyhfIpxA8ff4xpiarlazm0qfnGu7F6oAdJVG0qdmN1S5tYXu -b4NIf6WCIWWI+qxhW5At3/91TC7ykxT/7qkI3uwIErzJK/1wttP+nxgw6xEOyW87 -AsPwVa/c3Ym41Mg5yqzoOXda/V90hcm0Xbi44hZBgVMP+pLhAOyfnGG56JF3ILit -VkEGVALXop6m7+/aGMr9z+xcCb1mp35r8aPP/GTKXvzGfN+yNx1di+kbXu1vIHd0 -0oKmfeNXirYgQwbe3yjrS9cCOW0+ERE= +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDdl4bW8vca9/iF +DASgmGxyCQvEK5roWwPzE/DXdS+mMJM/fffyqB7Cldm1KW2ILDYU6JGnlYdxvWLO +Ea7n4cIBrQ2za7vfDU4LzGJTBN/9C2TYGFP13FTfZi2gZOuipT+MpkFCfJG1zMWR +13yKNxiS6Ixf76c6ADUyt7GmRcuKSctKzM7hUARfeko5iBN8aHpDnxYa6xSGlHXH +DzHHEqEpsk+iQ/eK6o3EeElg7v74sZSzqleIkEfFLiRIPNvuAwEEIRIQg+k/2S67 +3kdANlyfszXXbUGCYU9lOOPYgAEFgb5ljSXZJBTW3uFlyIONK5I4EHgELy9Hs0qn +qttXCx8d5I/eI/+Zx9GPF3x8gkMi5wQePiBpqde20kRYDvP2eGeZeFTnKYpqCshV +AJ4+1Kg96f4S9+GZ1wfzsgS1lA7+9qr3qTvI+XnKhC2h3bqyWS+CBQikgADbURgZ +T4EzhXvq7fSCFHFTA6xZzXNEVO7DkRrYbvkq06bIl1Ov9vcbCN4IzOuJJMxlk2Dv +3C3mUox3HmcOdO+vvtWnZt3HbFY7ZPCV68ObSf77YD3O5on5RFQuhk+AinrsDIL6 +NiiVzALXBL+e8flkqZDzRk1mGphVXGcRP6nn4VtrfN+Bmbm8pu3m6aYuqSX6vQXb +7EHW1tAbvlfbxIlP/Hp5GoV8zqI/tQIDAQABAoICAQDaRKlTDRwN+ndXRlFAhyM6 +6GIopvL9MLmhM+EluY5n2q0P+1rCMIusC8LYSahUW4gh7DucoRM7G9s5M/3e9mcN +E5LNSq9RtF9OC9JGCCVBsXlxyfTZ1l/bdWA3/3CDUtZYCmN5xA4az0tErsdDtaWE +/39V+E/2N8Iu5PYd293zp2CRm0+kbBcCnQiDxt+6yYa1GPzDIw+iyJWCsBrOBjGt +SrBaGyy4LvXZsspEquWHvhPFLWLvZ37qYNroNNpFhbv4f0K19dlJRPpdn0L7oxB1 +VicQvdOrQ4LbJ8B2vw9Ch1wt12ySiJHmXMAUa//4jBSJGN++72NY8uf0Y72N7ayF +Bck5QE0we4i7hhuN0WL+IftYD/O5NgOnprjMWifLOhQ8OECZfhOKgbRU+i3aJl8D ++raQBW7/GM0uL7xIoMcEZSwMs/sQR4loNCJ0UsIeWTdWNrhXrEuzDQGXoWcB8K/r +VVayDO5Qqx8R77HB82/pRdqEWNaNQd7DhPgnWiixISpIGm6zMvT3S0hzEkxu7FNb +uciq9i82BrBkKhg1kiF70FqG13VeMFqTJUuqtoRs1QEQgumvWB47n6FiVwHdDbbH +sUeKZYwbrY22Cn4rrfXH+0KKM9TDR0CiCv+CkSGmG57l5tW7aSUWun46qP8vh7sc +ztzb4LzyUt6XEBIWIqBIQQKCAQEA9+f4TyGo88qzTKaQoko5OAYhvAGr1UGAS6qh +sJpd7rHd9g6DnXyOInpQNglToiJ94mNdIK8f/oOjUXZh2E4CWuxeK291BNiaqCxe +s3LS3XjkdHQIHvqJUw/r4YJ+zfoGznthNbDwDkBob9x3h9rknTbGdLcgaTGi/0PZ +cFnyWDPNPskbcZ3Dxr41oDHiVsOx8n4d4HtspXzh+EPiQiJz5HVfwGNEyzhmFWIa +EzQaxnHL+WF1Pqe1wrzOwZia0Jss8lxbHcTnJupaV5WDvbxY0E4ynofShicv1U76 +B41xDKP/8hFWck9LiMDXk9vrbQIHvGAcsYr5N/jzIrDiEXxvGQKCAQEA5NOegb6m +Ak0mXg+XKhZnSdR1KaWT4/hbVWqYjwxAeAJfMaxjD4MXA8qeEAJo3l3ETkdFCHp/ +lr/BZzNYXnmHU6uvDn2Xq8gPO04ruSV0WWthikXb5gszXdkUH+6fryHn6L0kYJQH +NARQzOgdEcmTP7sy/5GDKzydWbT5qRNOnESUWgnJi9ePjGB9zWxn4jx9AfOYtozh +UmEgofSDGbFlamQic8HGnSJFgOxIZ0AfurPIRSR51gvXx2D5TcsPjLlDrY07IcF3 +DjqfJl0gC1XN5BXdpPvjvNrns+ZK/SRoGlgb8Q4tZLQevox9W110amvMeZj4yMTK +9mgGOSYCzZ6U/QKCAQEA1mBZ4Qwpj1DNRk6PqlfnLSRYTb1gO9UdvdE7a33CFuTn +HZ2lgS2xt+zvqhrcoMuU8o2cfeQTFcP+Gjb2G9gxvzDBqmwC1IL/Esjzx9hWssCV +RoMEds2OrS6Ke4OeZj59XldhU83DeX+HEJylHO1UXwN8EHg/5dfPrVCeGsMdh9qb +9VxxiAm2wAnCU9pvcTpfimQ3L+VrqZvZyRfi8+/ZKkm52KO/XMFTvdAM3mhjcxH7 +Ipd9jQX4bwNZBB8UWaqm7pqhDJg2j/d+0lhwCUZzwwasTV0E14/RlHNsUdWlWhoD +/e+yQr2BgyvIAIvgBW8JA4RVq86S/y0gC/LMO/TQGQKCAQBB2rlaY7DJJsTs+xWp +EiuFrvRNGQ734+j9KyFewcrn/t7An/keZL7B45UbzGW74UZ2tMIkT4TasLMLbVZ4 +UgdlSBqoU/LLiFcB3Vxt+16BwYqfzb0cdorA7pGBIx6nu11PuOd4OAHesYNDhWWg +Ud/jzo89x/X1AovSXmgfhaPxCzeatghgC5iPcNGjxhgbnwbnAeEoYGEUYUmP8pus +UEZ8mPblU5ZCcLOKB/ZKaMT46Xawl2/M7zmZcsos3kzKViMpFmU3MMN/v9U/qDtp +p7cKdlSEf82p82INfzCDq++d7U+VT1w3CDN06V/GZJ31ZrLBKAopVaGHyqZH2i2i +WYpNAoIBACmfr9BoJh1/mbLdd/WpOyORKTbnwstlMgoUcARRJn86iPZuyI4QoSSb +TePZqrWRVmO/K5M65hFjhUpqTWKJGJy5LYIZ4yuIbonJAPNUhjA0bkar9cULBFzy +rb0xmW6sRlBnqhv4aDlOkhHkkR9lB9rTIUW+ankuvVBiGWo4eE8DvZYo30frltku +2K/kqd3NppTl7dN4EnGTo8ROZvr3EMwSu6nE+wUr4G7YuCLdPxwb8gAB8dbmaUsn +AXocUh96kYqTwRxo8FO9SqgQYMf81/ovPUfv+7mwO40oygzy/YkGmB1shFIbQuzU +lJvRfdXyyC9DbllQkxWfdvaanLS3r1w= -----END PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/server-req.pem b/tests/integration/test_ssl_cert_authentication/certs/server-req.pem index 53408abf254..bd8e2e1fb7f 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/server-req.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/server-req.pem @@ -1,27 +1,27 @@ -----BEGIN CERTIFICATE REQUEST----- MIIEmzCCAoMCAQAwVjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGc2Vy -dmVyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtZfkztfSeWQVbPmV -c0+tsDNyUYfgszvf6IHOQNBEcFbHwRUGM9a+qRhx08TNsN2rrnocgC24lveXlb5K -btEMjnfgmh+CtAyP63qLWMrwF+fkL+bZxxgMBSjXg5zRuIaQkTDvwZaIWRIdz6XJ -u47zFaT4UjPKI2cErx7BDAVW+EIp76b0TiyA6wP8z0EpVtEtAGX71fnI8SKA407I -PzNCZ/ulTTUawHpsfGir+E8jodm/s2DWvMJ5YXbKU4IwrHjHgjkcwzijD+/OKdMV -hcEM2qEtB2K8S6Io4sCMOvyO9gA0MZLqVuDmNtluqo4Xg3uuzN9/ymVWfZbLHO0C -hgbr5MYLDKutzPlSPLpmN4LCESyShMF5Kr4iFzQqsPuczI7EQS91FAYx4mw9XB6X -Ne9MX+CDjGtDZObLzctLUPDnUwvdLdp4vfTuRPjDhzU595q+Bgob//j1ZVo3/tj1 -pLJoTXUwZBOuCV0q3932+DknG5IcA9OmFlT7WpOcb+tUDYC/arKwEXJvH+Y6wJc6 -lOq2B5wpIwdgNNdQy4d4ncMWXzrcU9Y37vjTD9jfMRYCI7afMolUUKKVzAPx0/9y -fyDTuki6N7eZwUY6YQ7GjnKd7TpQFGGvvtkFzPO/kCgZUQYAtjK9olKJ6jyRZgGu -tk7nWBqASqnlX8+chZnJVBmZW/0CAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4ICAQBv -CpHPJBbcSRdDqO/iwJeySMSwhcM85CvyJmJNmqHLjXCl+cKQ1uJkjXFsz6HlPK1L -kXY5/ubRwQ3XuJIHi4PSshjEY4AXYTRJwTq8k2PGbDI4kBC36uEDQRI5eZBEH83E -KEvNggWXp2h1M0QBHJuG3qkrRnKjzLCUJ8Y2NjEq9+xNPFRR86WGy91uTMMct8xs -6bPPCLB6ySu9jsaenK5mTWEtYIAkxHJTK32STlFSOsost9vyYdvh7wd+mFDleD1K -vkZ4C0E8CWJISDQHHKIZEFaLFV0fEbOqvV9R18QTzAuV3uVzpP11pJxtciFHiuHg -/MCLJHWVcFNX6dSF4m/RmW8MKvdaI+fcuBgao1h8x6PMGPkPHYDQvupWwD0w5hrl -axqkXLpe73IXP5XMGeICVD8Nt77KYKW6LbtRNKIU28MfmhLn/zXqF0tf9EYtBn7s -rDwzzPN5ywhV+YXfQYSpCzheVMyfxjM3X060V5hyRWc9hMS7aC56aMNmEDI72IWl -ZAegSLnU5YF2iXaYRfftTPYiLL1n+TffK56qPI5QSn7fxuvG3N7cvVC3zeli8PGJ -gWWw4zPwRFvHk/NSepN2ZfiLZpMExZW4l5jvUrfWWjdQdh7B0a6JD6YfDS+Ot9nZ -eZkLUSJnrGqHq63BGjHDBNJo1A3GcegeaYrVkaGDzg== +dmVyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA3ZeG1vL3Gvf4hQwE +oJhscgkLxCua6FsD8xPw13UvpjCTP3338qgewpXZtSltiCw2FOiRp5WHcb1izhGu +5+HCAa0Ns2u73w1OC8xiUwTf/Qtk2BhT9dxU32YtoGTroqU/jKZBQnyRtczFkdd8 +ijcYkuiMX++nOgA1MrexpkXLiknLSszO4VAEX3pKOYgTfGh6Q58WGusUhpR1xw8x +xxKhKbJPokP3iuqNxHhJYO7++LGUs6pXiJBHxS4kSDzb7gMBBCESEIPpP9kuu95H +QDZcn7M1121BgmFPZTjj2IABBYG+ZY0l2SQU1t7hZciDjSuSOBB4BC8vR7NKp6rb +VwsfHeSP3iP/mcfRjxd8fIJDIucEHj4gaanXttJEWA7z9nhnmXhU5ymKagrIVQCe +PtSoPen+EvfhmdcH87IEtZQO/vaq96k7yPl5yoQtod26slkvggUIpIAA21EYGU+B +M4V76u30ghRxUwOsWc1zRFTuw5Ea2G75KtOmyJdTr/b3GwjeCMzriSTMZZNg79wt +5lKMdx5nDnTvr77Vp2bdx2xWO2TwlevDm0n++2A9zuaJ+URULoZPgIp67AyC+jYo +lcwC1wS/nvH5ZKmQ80ZNZhqYVVxnET+p5+Fba3zfgZm5vKbt5ummLqkl+r0F2+xB +1tbQG75X28SJT/x6eRqFfM6iP7UCAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4ICAQBc +TPShRdB7ZIL4xQNAxWLGoEbshbY/UpJQZdjojxn27roVwEhwP6B1/KgiVKV2X6bE +a36LUnaWYllIMAh4oOHJkIm2gZ3xitdEK1anCf5NJga7TnkwGfD4jTZA91fWCynt +a/64s0KQggKsUVY12TTJVQvOH/l9RMrXOq+jgIh4OURwCBtHTCS6oOp3eF04pEu7 +w54dMAsxp/N9AQaLh14IZUZAQ2v5kdipL99EEQH/G6llU08XrJ4LfGgwDkUjJSsA +TzrX2uk9MLHJVwjpZ99ktjNBs8Gyr4fGOmstT5TXEOO6bVhqZDC6kEL7vrmFSnDZ +g/9lrd4wLUT/STt+E4Qukedi0n/419IpkIAE5C1HOXRnQaOUdcnrLixUB21mBHt/ +n7gkwdY7Cu77dYvBIShzeCnxiJED0+XrBPD5yPokxEjE3MmiIK6meHHCuIwqLjX8 +I78ysv4COeH0svwSjvJInveS9QRCAWBpdvskxPJR8dpoysAff+jiyp3ZvkwcIiUO +Vbsusorpp8pFJXZxvPDBXCy5TOlFnIG/9itjPj98pFRIl5bxzNwDf4wrkwHEpR3i +jpM6y+/RWZn69BpTeAG0vZHhGk3SuXdU56cRzatys4X3biCImgDqeJMUcAoxlrIZ +vgbJVTorwqQmPz5kt28b8ddnUVobbZEPlRITcqjwFg== -----END CERTIFICATE REQUEST----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/wrong-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/wrong-cert.pem index 6096794a1d8..b56a10f8a92 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/wrong-cert.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/wrong-cert.pem @@ -1,32 +1,32 @@ -----BEGIN CERTIFICATE----- -MIIFjTCCA3WgAwIBAgIUF+tVv6V0/vm8r0Gtvaro922XodswDQYJKoZIhvcNAQEL +MIIFjTCCA3WgAwIBAgIUMcX2R8I2H8vNtASHi0EoufIgWEUwDQYJKoZIhvcNAQEL BQAwVjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xpZW50MB4XDTI0 -MDgxMjEwMzk0NVoXDTM0MDgxMDEwMzk0NVowVjELMAkGA1UEBhMCUlUxEzARBgNV +MDYyNjEwMjUwNVoXDTM0MDYyNDEwMjUwNVowVjELMAkGA1UEBhMCUlUxEzARBgNV BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0 ZDEPMA0GA1UEAwwGY2xpZW50MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC -AgEAvKjjhLJvnrHpaxX/JgSqymisQ7IvMoiEapAf3fArNt1dg5b6E13w33ZuwtJ8 -Dem8BYceFWmu5YeOWQP038iJRDr5uofVTJAqyVzt4aM5YJVo7CaXVSlRRLhDN1zu -WsOnsC28uB9+0zn90729iJyU0DbQ2Tym/C6emzBS7OcD8cy84MvMdvyn6F5O8pjQ -A6V7l0dS2fD32GbAg8tpRABORUGnYDDHUIklwHQxQtZWQmJneYKGl0M3BdGaCA9g -AtRlRLwpNBQYwk5APh8TKNfAsEg6565IQ8sV12xYzphF9XMTFkRiLtyqMES+zpqu -rUu+xZ9gKFBL1rr+Jm36cA5zfDg0xdyr0u+j5zq2adM7lTctycaCw0pn4l1rGZHq -A33LV3Qgd4woCui2asgPvEiJU27qEcK4RM8x+Mf9vpm6jGhYjxLCAxoMHDaAoKKL -t3xd4PwqG8VBK3FbObU9qTXYnJW0tGvoMyo4u6onIGyQtqLul/Va8Op1QM9rOr2+ -3zUiNT5Zocg8EVb1gT+mJP/axSjCDAhk0+lh2HenEAR036Hicefp9O6SZkBdarTJ -ffqdIRNgbq5uIBqQ/Q6/A9lt5lR+lz1q+ZIJiwBD4ysE9dqwqUQboHoQg/yR80bu -+mlLow4E82rWW+qjD1FODLXukthibPh3vOLJyNl/UOKrj8UCAwEAAaNTMFEwHQYD -VR0OBBYEFGzWyPpUSrjEFc4yrS8QHu2wpVFVMB8GA1UdIwQYMBaAFGzWyPpUSrjE -Fc4yrS8QHu2wpVFVMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB -AGfn+XugP3raB31fEwqwN/wshe6jmcq9q2XYW+dHdrBq++ChE6Cw6BpjCFKLWWzr -udIo4opiDTkAauG3BOfQfUwJgyL8s81tAfOPxrB8O9NcvDNK92RugmT1FWGkvBV9 -lPRExxkjPbhaNzlcOIcqVX/O1eVuv+Oli+ZZBN2mwNySXR0VZiHjOV3eh6Kf7c49 -BWjgClWlGe93ZNTyaUJ4RXasBNQ4551Kvqw+pEMaaVLU3uVgJk2QQT+lSOJQrwqc -DHC7WqYS/uCm22CGOt9vP3dK5Xo8AMPYLvB3O8JcWrNdwJ/SoK6lTmKNLhqYer0r -UkP6noRRAJcXbTsH5iQvj9Daw6qjHEs2RuV2iRFxvCzAzf5bURAwXkjnCsZ24SJa -oc4yG4jfjzFUmO+xQ6+ykgMOHrLPHXcJOfgRhA0XW0BiTEKaZfI/M64W8kfU9w+M -Zo7n8CAWp75VDcAYXDfO8en6YLphpCkfhGrBqO4DfPpsHuNIETRv4wMexiaAc5xK -dud1EmuX46vreEMPtjlsO8BZTtVT5JF2P7y4wS8vd4s9vVHixvfEqVNcixVDNrQh -YZrUiTL14kPDrJy9UfzylWeEc4cQfTwzId2QKRfX7+u9Xe6LPfdsGVN5gGXdKtkY -z2aV9XUEX0VmDmKdKcvhMu+ReOsfM66vFheT1ZEAKNnG +AgEApABgxjCrAiDY529xEjboRE/PVwgVStD0dPE2vSOOqxfVYYOqP7VF7pG2PeA7 +Ek9Qqv2cqaDWbDWn5vT3AiN+aslfHtXJgGjPBrpOQ3Me6RSu2aUHPIguabUjz+kJ +VSkoj0pkbAqqIxKse1hZtUNHVwOmg/PthkpGPNxofNX1kCfBWoNJyuKFOoGeNgmY +6joY10zTaiHpq8hhA3b7WY35QdNGD7SwkYBvTjGGzBr/hu26fhX/DnceZ9kf9n6q +899gB2kZH5T1eTlShh12TI1sHa+BGz1YwR0HqM88zXDyAf7bWl7Hy5f8e9keZdx7 +Fx/ws93Bb3GusA5zwUm1iUe1OIwTLFlL+Kdkr0qzofDcQ0ZnNwrME4oAhCJFwSnJ +OWnrCmUou2XXj86Xl481uBana30zzJ9TniHM/hA54ttHsva/yB8tyoXcI4FASwk3 +GdihsOBbRS6KvmeKpEXQpsvBQ9GejSL/UUWuKg+O0ysHE9+QX/+OznFp66h/x7PP +Q7g6ipwAjgwuG5jm/Czz+dw4j3Qp5N5f7Dn3QhDzmXkKgzRzirKh9XVQqUFRwlLn +8VuzAhD5SjRN0JDE2jlt0Hin90zx/nkOV2b5hTYu9NVgmrfCye6uB/qsK7PQBh69 +9i4+8tBGXrcS2Nenm+Hm12fFhNum96A0ahj134H2ks4JcKcCAwEAAaNTMFEwHQYD +VR0OBBYEFIZYdI/00qzj+5JqEzEJfpj93AphMB8GA1UdIwQYMBaAFIZYdI/00qzj ++5JqEzEJfpj93AphMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB +AE+9y6Ea3pjIegOeffXzI725vXO0OSuv76vRoswUVQDKL7j7QcvqdcKCK/2hmeMW +MBNwKZneG5iumH2Yp5RQ14arihS9+SYEpgftvfNOZrzZ37RttJhSjBQ7GUorH7pD +uQ4s1unqKXv981JCtlKOQh5ZKthHAMsP/sfYuWg2ksVn1hvFTJZirewVUOgR8zPB +Djl6PdjZLDu1FlglQQ5YUNgsJbAqkBPcA0hEwGU0j5QEncvdspn4LiH2/mHhnzM9 +3QEvsXUxgJo989Am6+E77XNX0wcALj2zUmPPDWYgHgLKO/ZcSAOQ9JaauVhUre2D +7jPwnN47yLak5obVcyCpaDPKYk6sEUZkiWRaONvugoIbjYivmB/BJc0njfVA0kzT +FDwpUTtSddZgHzdTXe0p5C7IGmYkp/vgKlSpSYY6+aCiVApJSdJjL6FZKoOXqDnr +OgoQGSOJif4mDeipKOdrb2JtYwJkRl0c1S+tgOi8PU+ROvZxQGWI9/i20H58M7j0 +r/WhbudhcAqWglk5WOpCodbJhXffCrbUm5NjoFr7AKswxLJVz39WIe/duHPEGV7v +jLd/zj7eJRv5ycDyt91rbGxQ9NKzEx+by/5WIZTi+z+2PG75tdpQUwgEIh1c/XOt +6uXtS0sNnnjHVmXPBC+Myz+1NolYWjZMcBQ2xGIORvm8 -----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/wrong-key.pem b/tests/integration/test_ssl_cert_authentication/certs/wrong-key.pem index 7a7bfe60326..3924eac91c2 100644 --- a/tests/integration/test_ssl_cert_authentication/certs/wrong-key.pem +++ b/tests/integration/test_ssl_cert_authentication/certs/wrong-key.pem @@ -1,52 +1,52 @@ -----BEGIN PRIVATE KEY----- -MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQC8qOOEsm+eselr -Ff8mBKrKaKxDsi8yiIRqkB/d8Cs23V2DlvoTXfDfdm7C0nwN6bwFhx4Vaa7lh45Z -A/TfyIlEOvm6h9VMkCrJXO3hozlglWjsJpdVKVFEuEM3XO5aw6ewLby4H37TOf3T -vb2InJTQNtDZPKb8Lp6bMFLs5wPxzLzgy8x2/KfoXk7ymNADpXuXR1LZ8PfYZsCD -y2lEAE5FQadgMMdQiSXAdDFC1lZCYmd5goaXQzcF0ZoID2AC1GVEvCk0FBjCTkA+ -HxMo18CwSDrnrkhDyxXXbFjOmEX1cxMWRGIu3KowRL7Omq6tS77Fn2AoUEvWuv4m -bfpwDnN8ODTF3KvS76PnOrZp0zuVNy3JxoLDSmfiXWsZkeoDfctXdCB3jCgK6LZq -yA+8SIlTbuoRwrhEzzH4x/2+mbqMaFiPEsIDGgwcNoCgoou3fF3g/CobxUErcVs5 -tT2pNdiclbS0a+gzKji7qicgbJC2ou6X9Vrw6nVAz2s6vb7fNSI1PlmhyDwRVvWB -P6Yk/9rFKMIMCGTT6WHYd6cQBHTfoeJx5+n07pJmQF1qtMl9+p0hE2Burm4gGpD9 -Dr8D2W3mVH6XPWr5kgmLAEPjKwT12rCpRBugehCD/JHzRu76aUujDgTzatZb6qMP -UU4Mte6S2GJs+He84snI2X9Q4quPxQIDAQABAoICAApGgpncJCM3GnkIKiz2bRz4 -JYXMYzCz6c5qCipK5fenh+veYcGDSNbK+w5ma3ZQiDqe2N8esfVzdCfaBNDZecx7 -D9X+hvoUEhiElLpI6xudF6lhErYDOZduF88goyTTakM4woIeyQgVLQOG3pddu+c5 -TRe/63Jp0Z6vO50Gmhrl5VWzE/BZI4YO+OrSsuW38irTqioPq1gghJTJE/MttxWj -lUuybHCw/5rjWTmENg+Ij405ND5x3UHWYDbXK4oL6nYbb30UKSMQIwSfKap8UdTo -IjkzL2Ft4sModg/OkGTlfyEj2VsnDqfxXpkfKKtsqDfYTeL0OKU56xTJwa2vw89k -gtCP0t/UQzv2MRPvoCOKEIj8AiUj5X9uLPY6ijA4jD396OVnuZh9+KgpLwB7piT+ -WNrbm/Nrfe617kK41NGAVpVIMo1y8fSbplZqMOWSSqLO0eyHkEJK5yK1WtFfzVAE -UKH5jRjS52XcLrLa7VxAcsywv45UNiSiYRSxBY7MOg1mL94BlcghW/YHg3h3vKZC -NVEBoaDHNd8Q3V5sQgpENUjXmWFaE/1Su9i/oooy/GAwRBySHnUkKPvBP6Unus/S -Pvc2vQPMHpfMbPc+L9YxxbbgVHBembYJn7qIejoqfedtaZoqB240s4AowxDiK88f -QN1L8TiaDiGJBIuvfhtRAoIBAQD/2TAXiLZ4fkbcIbpxrU8nmQNrqxZPlqVLKLDj -4eq4wBLMywOsW1RGoaipooWmEWICgrh11YtoE3GjgPYh2x+oXPOpUn5P4Gbx88o7 -8rKMDu//yRKnj/e0GNVy+ugXqVZj3+o2Mdyv2nlAqLEeZsuzFi7nXoaNJjOsumyq -e3eP/iM9jV2CakhRPnZH9B5/6dgFjwR4Vw9DU2cCaQZtIEOGANCEfP3UWaP1GW0Y -eMtDYiGAxkYqpCDLwbQT2jbgCZUUBtIbkZhkVgXDrE6K2Ey9M7HroLo7Tl5NGBMA -V7lZxpVa/4T2R7RFlXZFChDNul9lngQEFHlrbITJG8iOLfpdAoIBAQC8xYIlIyLW -q/mzUErH7cqL76ORcm8ejymD+ii+ERQpqwTQJljBf5mnadfTQPN1kD7YLAXSgufq -oEnVNFYR/k/M2aztTWfsUNGW0MAhS4tTewFgcia8macfhOccP3mItZ3KOQgpXxnW -+h8DXLlq6FFvybK00msribpxNQybS7y4DK5VMRbg0aBZAn+6O4Lz5uLR+PU02pFG -FgaLVh1E0h2mp0ib42BaRyVJI0CY402XKrSm/BDnHhGfoLW5kK53gvk2g6RzIDwy -qF/DJ0HHlAQL9Z9OxoMy4haHXcBGP9rGJX59eyb6adinsSBl7wXStdjvTsmlNYD7 -KqltgoAdkaSJAoIBAHWZs01d/eGsyY1tw3F1JCkjYDshTQQQTrIZZhWZRnbdcsbH -mkyjPj2pGQnANoZ6/v4AcCFZotaX+WgaYwh03DxbXXS7AmxczXTxhke/6uoOA6sj -FXwH5OfXcmWDhyM5JwiJZ/K5QKNkXM+nuqIqxf7vd2fKPzaqFJ6UolZKok6Bllk4 -nX7Qs6UEfQHd6BcLucv0TS2zdsSPlY26EMYgSmlR/oannVT6Ty7eHRNekq/Kb3Pt -r1ryTlDaHJfzeb3JKckmyXT6m32jPMsQbJnNiph9Jo8UNgYEo7v0EOfbasslSImn -YcqCcw55AQAC/G5T+H2RAG+PqbADFZYLO0h/QdECggEAAJxS83PNpQwhXqcf1s26 -HgHEcbABFAQ6iibBAvxjKE9ZUKMPHnEfOh9ph1FqeDLjQSfDTQ8HWQbztjDZJm3A -LFV37byWXXlVdDtwo6ru5HI9auZzoaCNndh8NuctqXeM7x6IHDomhz9/4i7mmqXt -vYLVhSg5GIb1h3A4fjgcgCvqVHQ4Mrn63s7XQu72WXuhuDQp9uXOGn/vvXul1Jcp -aWSZI4f0w9X/FOF8UAJMOfT1aKTgGR9Lx0xpyhPhvJk73SVH3ud3ymIpDSSPXeno -qXE99q9FtWBt2jo/aPrD5mgwpI9FbQHypXg5NpszVZ8o+H00wcgOhsF4ktYdO/tA -oQKCAQB7TZVBjOsOY/S8eWbTZo0w5K0THL15k++gfZAOCcQamhGe+Cqar85kyJ50 -aAE+p04i7ZuNAtxAetPU8GvdYK/DNdDa+yeUTvBjl7Lpi6FjsPwdCSDQJNQk4qCW -rPHnBfCK7wHKpOKsQuwKXH4Ndo3En6y4EmICHMdfu9OWZlNhKc2o00SdkdXF9+Wt -7iKNb3JDCG1ZOEISMkBdzbRjNfzzxoleQFr+2C3nUogMkGdgwMtC4JmgO+K5dsSp -/vNHeNUOYxLj8D7H0y75iMPR+LqhlKvFj6P3BlslD0Lb/yg80U55v821mYBRs8pg -AVHPFI5a8A+VunzBvaL+SEP/rwrI +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCkAGDGMKsCINjn +b3ESNuhET89XCBVK0PR08Ta9I46rF9Vhg6o/tUXukbY94DsST1Cq/ZypoNZsNafm +9PcCI35qyV8e1cmAaM8Guk5Dcx7pFK7ZpQc8iC5ptSPP6QlVKSiPSmRsCqojEqx7 +WFm1Q0dXA6aD8+2GSkY83Gh81fWQJ8Fag0nK4oU6gZ42CZjqOhjXTNNqIemryGED +dvtZjflB00YPtLCRgG9OMYbMGv+G7bp+Ff8Odx5n2R/2fqrz32AHaRkflPV5OVKG +HXZMjWwdr4EbPVjBHQeozzzNcPIB/ttaXsfLl/x72R5l3HsXH/Cz3cFvca6wDnPB +SbWJR7U4jBMsWUv4p2SvSrOh8NxDRmc3CswTigCEIkXBKck5aesKZSi7ZdePzpeX +jzW4FqdrfTPMn1OeIcz+EDni20ey9r/IHy3KhdwjgUBLCTcZ2KGw4FtFLoq+Z4qk +RdCmy8FD0Z6NIv9RRa4qD47TKwcT35Bf/47OcWnrqH/Hs89DuDqKnACODC4bmOb8 +LPP53DiPdCnk3l/sOfdCEPOZeQqDNHOKsqH1dVCpQVHCUufxW7MCEPlKNE3QkMTa +OW3QeKf3TPH+eQ5XZvmFNi701WCat8LJ7q4H+qwrs9AGHr32Lj7y0EZetxLY16eb +4ebXZ8WE26b3oDRqGPXfgfaSzglwpwIDAQABAoICACiTq1399qGtLN1+NjSyfx8/ +u+Ylqtb7AjDY6Zk8bfUpDXN2Fy5yFF5lkPiYPSVXmHbmDtftYoAdenBrVZ4i2All +z3IapSNvSyG4ANsxZYl3w5c3/KVecFVZKwYq+1MlvtJNLrGIpfXNjf1qq69seP8v +eQiW1sLuJ5ixU+znJz3GiFFzwFNBXoNORK3MDBiPzUufx4Mv5tfI2S/5RVEwDmbZ +9jC2nSUy6Nco69geKfDhas39dUDH+i7pir37MyLptqG+wCePPHkE1MU4Duf76a8i +mEf8ErSdESMUO0/9TPNvcihW4Qofjam624mKVq4vCegGyvBe6UDIIp3FNfREWLLC +ilit37ZVOHbq79qV+DobLADYXZGXrptzr6VqVZvvQUEwOjftD2B8GJzxbNxRl77F +BbeOhYA/IDr8YR7Qos0HjqDDekRavHKAa5kcf8yFVJaQycirHJZpjr3oNFktqg6H +9eb537SdPI5nHtgTSQSosxst+iBsjMCJ7rU7aowy9gKG75s9eME06hiQsukNcOI3 +hmBqQBeX+yLWh7Z6A2Y7MepHuYHWKereBGISR58bvTmNyI4mLWYwJZzjON2tot3a +MJM86gw83TuX1Qmp3+NjduQtdtMjDSXLN2yBbK4CufQYaTxK1xdHUoK/uvG9kIq3 +tP+/fiTHZHyuTSSgwOypAoIBAQDT2Vj2uahOypboLv33XtFr2kuDe1sogpLaAVu4 +Dv4vO835gto4fx79rK3u2fBxiVx0yv7kwitvpcwaoMdkUEoSBtOugYB9UezW+SX5 +91bpY0mBH0ToSmosPFVc6PEki6LRV+VGZ1gFXU7uZ4Wia9opENfT7d8cjBQ4NZ/M +sCyqHR2KYB82DHx6Lrtrs0eWn33N8BVgsy4xSbOi2YrgQCnJvfPWVYtcXjRbplj4 +jCVGnPlac0Z/bv3Kb7Q7EOS+d+RFi1ZpsFYPbW5KRhGshzOxGw5d/nCjkEXCV0ht +uK6KndjFOvCGfikZW7WVpw7bkCe0W2Ko/JSX99ccJBDyau1NAoIBAQDGLj/wVxss +dllwswBOAV3oSL9xbUPs+Xbr/FK4XKcL7wcY6cfdZBlsQLQCoAzyTZ8t+g756Hlt +a8qmW2/Wvdo+m63Z2ecnbI9pJsVyYeT8pVumx4biHuXbRBYO/0ZZPtB5kTT6Hzhv +ZHxoUj4jb7L/5kwEdEPFIZX4rVEXY20LJL5wtv2zEQylQk9kunjUgrP1L/GtcNh+ +QRzLXiJWAoC4HWcXmdxb/Hc0BU5geSwZL4bbl3YL3lwWvkd3aY17T90EjWy4uA6G +tsHCxbxauul1q8OkmCcLEfhYnDh95YoVddR97XhC33S0v4dYjX/iS46g8fJ0HhRo +9YGWsD+tRevDAoIBAFCp/5/iTV3C8fbyfa1FI0SH2Bz2SV2Bal0sCzpoKwzdHq6U +znaYoLpCl+/MeCsi/FtUN/3umQ9n9/FjqshdcfavNsbJdJ1DJoUsVPN65FL1hTVv +LJOuUgMJ7g70e21I5fQEHb7S9scEIlvQeye/HVBpo2SEvGFoTQKiGHid1EPp1ies +NfYkhvkW9jIqD2Yg0IwrkFhDoaEOySGG58Q/ainw8/l2lRvUmucSzenFoyPh/Wgd +YIiBQI1mPyAGbLLBf9+jEIIprHsvVcFeMLiaumoDPVM44LbG5mj7Rw7QNVV+iN2A +dbkgLJIFQ3z6IUQk/ZlE+qoRkprSuctzSCil4jkCggEAdiYWilNz6NL5yX193gNk +l9nfAGFS0JF8+31nV3AtSqkLAyhEtlE58ta0Oqhub3olPwTILucQlVJg80Kp700q +Mo8fWzRUYaWP7fFmXyXLnW97r3dei6o+ALWbrP81UnlnUkJmYgOA4q/2lz8Iupma +DoOeqD0kNf8q6KFzKc1lsfIK8ym1IC826cMZkAS3ioINhUw6+dq/xq1M3FVXhQ1i +7eDhmClrPQ/LhSDwtAUpbC5waLPodXTwU8LG2oL8DRr0ugUSXyGjz15fL54xB6pN +CpEHRzZKeIgTFci0ySGya87eiuCrBLsxWZyhtQJOznubIYp8sAtKwbQzuMGEhOmd +fwKCAQEAlZoi1SzHstg6PfpwrIHJV3imLa550k9hyAu610CKjMsg6IsFsgu9/J0b +9hkhlafeW+p9zhKSwjl3aCuWUMNE53R+zYmMIJJrBzejC+1H0SKW0Zix9+ghixOX +da1jRaUxUqApJYvvxUC8FbnATM/Eq0ofhGkG3o575SlO+54twJO+bXGAUf/C6xMY +AQUQh90pTbZ96Q3Wdm2Qmrhd/GUaC6k1vAHVHHU8WQHiLmo1fF4gL/TqRv5KEPUM +un6ld7h8BEWtMClhSIiL2h5nvSYGcB6Lai6rPO0UUbGkWBQFpGaeglUmoYi0ciC5 +lMRrRHGUiWHW9C4/siOKYrHBeH5oNQ== -----END PRIVATE KEY----- From d281333db221915e9ee3e01b90d609d1a4cee791 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 2 Sep 2024 20:11:00 +0000 Subject: [PATCH 126/215] Better process of object storage arguments --- .../ObjectStorage/Azure/Configuration.cpp | 19 +- .../ObjectStorage/Azure/Configuration.h | 26 +++ .../ObjectStorage/HDFS/Configuration.cpp | 21 +- .../ObjectStorage/HDFS/Configuration.h | 17 ++ .../ObjectStorage/Local/Configuration.cpp | 10 +- .../ObjectStorage/Local/Configuration.h | 17 ++ .../ObjectStorage/S3/Configuration.cpp | 106 +++++++--- src/Storages/ObjectStorage/S3/Configuration.h | 43 ++++ .../ObjectStorage/StorageObjectStorage.h | 4 + src/TableFunctions/ITableFunctionCluster.h | 7 +- src/TableFunctions/ITableFunctionFileLike.cpp | 2 +- src/TableFunctions/ITableFunctionFileLike.h | 3 +- .../TableFunctionObjectStorage.h | 46 ----- .../TableFunctionObjectStorageCluster.h | 22 +- tests/integration/test_s3_cluster/test.py | 15 +- .../0_stateless/01801_s3_cluster.reference | 192 ++++++++++++++++++ .../queries/0_stateless/01801_s3_cluster.sql | 20 +- 17 files changed, 437 insertions(+), 133 deletions(-) diff --git a/src/Storages/ObjectStorage/Azure/Configuration.cpp b/src/Storages/ObjectStorage/Azure/Configuration.cpp index fa5346e9288..8121f389a8d 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.cpp +++ b/src/Storages/ObjectStorage/Azure/Configuration.cpp @@ -24,6 +24,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int LOGICAL_ERROR; } const std::unordered_set required_configuration_keys = { @@ -146,14 +147,13 @@ void StorageAzureConfiguration::fromNamedCollection(const NamedCollection & coll void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context, bool with_structure) { - if (engine_args.size() < 3 || engine_args.size() > (with_structure ? 8 : 7)) + if (engine_args.size() < 3 || engine_args.size() > getMaxNumberOfArguments(with_structure)) { throw Exception( ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Storage AzureBlobStorage requires 3 to {} arguments: " - "AzureBlobStorage(connection_string|storage_account_url, container_name, blobpath, " - "[account_name, account_key, format, compression, structure)])", - (with_structure ? 8 : 7)); + "Storage AzureBlobStorage requires 1 to {} arguments. All supported signatures:\n{}", + getMaxNumberOfArguments(with_structure), + getSignatures(with_structure)); } for (auto & engine_arg : engine_args) @@ -294,13 +294,8 @@ void StorageAzureConfiguration::addStructureAndFormatToArgsIfNeeded( } else { - if (args.size() < 3 || args.size() > 8) - { - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Storage Azure requires 3 to 7 arguments: " - "StorageObjectStorage(connection_string|storage_account_url, container_name, " - "blobpath, [account_name, account_key, format, compression, structure])"); - } + if (args.size() < 3 || args.size() > getMaxNumberOfArguments()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 3 to {} arguments in table function azureBlobStorage, got {}", getMaxNumberOfArguments(), args.size()); for (auto & arg : args) arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); diff --git a/src/Storages/ObjectStorage/Azure/Configuration.h b/src/Storages/ObjectStorage/Azure/Configuration.h index 8cf583c32db..2e4719ea24d 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.h +++ b/src/Storages/ObjectStorage/Azure/Configuration.h @@ -22,6 +22,29 @@ public: static constexpr auto type_name = "azure"; static constexpr auto engine_name = "Azure"; + /// All possible signatures for Azure engine with structure argument (for example for azureBlobStorage table function). + static constexpr auto max_number_of_arguments_with_structure = 4; + static constexpr auto signatures_with_structure = + " - connection_string, container_name, blobpath\n" + " - connection_string, container_name, blobpath, structure \n" + " - connection_string, container_name, blobpath, format \n" + " - connection_string, container_name, blobpath, format, compression \n" + " - connection_string, container_name, blobpath, format, compression, structure \n" + " - storage_account_url, container_name, blobpath, account_name, account_key\n" + " - storage_account_url, container_name, blobpath, account_name, account_key, structure\n" + " - storage_account_url, container_name, blobpath, account_name, account_key, format\n" + " - storage_account_url, container_name, blobpath, account_name, account_key, format, compression\n" + " - storage_account_url, container_name, blobpath, account_name, account_key, format, compression, structure\n"; + + /// All possible signatures for Azure engine without structure argument (for example for AzureBlobStorage table engine). + static constexpr auto max_number_of_arguments_without_structure = 3; + static constexpr auto signatures_without_structure = + " - connection_string, container_name, blobpath\n" + " - connection_string, container_name, blobpath, format \n" + " - connection_string, container_name, blobpath, format, compression \n" + " - storage_account_url, container_name, blobpath, account_name, account_key\n" + " - storage_account_url, container_name, blobpath, account_name, account_key, format\n" + " - storage_account_url, container_name, blobpath, account_name, account_key, format, compression\n"; StorageAzureConfiguration() = default; StorageAzureConfiguration(const StorageAzureConfiguration & other); @@ -29,6 +52,9 @@ public: std::string getTypeName() const override { return type_name; } std::string getEngineName() const override { return engine_name; } + std::string getSignatures(bool with_structure = true) const override { return with_structure ? signatures_with_structure : signatures_without_structure; } + size_t getMaxNumberOfArguments(bool with_structure = true) const override { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; } + Path getPath() const override { return blob_path; } void setPath(const Path & path) override { blob_path = path; } diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.cpp b/src/Storages/ObjectStorage/HDFS/Configuration.cpp index 8abc51a7526..9b5bbdeacc1 100644 --- a/src/Storages/ObjectStorage/HDFS/Configuration.cpp +++ b/src/Storages/ObjectStorage/HDFS/Configuration.cpp @@ -24,6 +24,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int LOGICAL_ERROR; } StorageHDFSConfiguration::StorageHDFSConfiguration(const StorageHDFSConfiguration & other) @@ -83,12 +84,13 @@ StorageObjectStorage::QuerySettings StorageHDFSConfiguration::getQuerySettings(c void StorageHDFSConfiguration::fromAST(ASTs & args, ContextPtr context, bool with_structure) { - const size_t max_args_num = with_structure ? 4 : 3; - if (args.empty() || args.size() > max_args_num) - { - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Expected not more than {} arguments", max_args_num); - } + if (args.empty() || args.size() > getMaxNumberOfArguments(with_structure)) + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Storage HDFS requires 1 to {} arguments. All supported signatures:\n{}", + getMaxNumberOfArguments(with_structure), + getSignatures(with_structure)); + std::string url_str; url_str = checkAndGetLiteralArgument(args[0], "url"); @@ -184,11 +186,8 @@ void StorageHDFSConfiguration::addStructureAndFormatToArgsIfNeeded( else { size_t count = args.size(); - if (count == 0 || count > 4) - { - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Expected 1 to 4 arguments in table function, got {}", count); - } + if (count == 0 || count > getMaxNumberOfArguments()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to {} arguments in table function hdfs, got {}", getMaxNumberOfArguments(), count); auto format_literal = std::make_shared(format_); auto structure_literal = std::make_shared(structure_); diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.h b/src/Storages/ObjectStorage/HDFS/Configuration.h index 85f11aa6a98..86373512fa6 100644 --- a/src/Storages/ObjectStorage/HDFS/Configuration.h +++ b/src/Storages/ObjectStorage/HDFS/Configuration.h @@ -16,6 +16,20 @@ public: static constexpr auto type_name = "hdfs"; static constexpr auto engine_name = "HDFS"; + /// All possible signatures for HDFS engine with structure argument (for example for hdfs table function). + static constexpr auto max_number_of_arguments_with_structure = 4; + static constexpr auto signatures_with_structure = + " - uri\n" + " - uri, format\n" + " - uri, format, structure\n" + " - uri, format, structure, compression_method\n"; + + /// All possible signatures for HDFS engine without structure argument (for example for HS table engine). + static constexpr auto max_number_of_arguments_without_structure = 3; + static constexpr auto signatures_without_structure = + " - uri\n" + " - uri, format\n" + " - uri, format, compression_method\n"; StorageHDFSConfiguration() = default; StorageHDFSConfiguration(const StorageHDFSConfiguration & other); @@ -23,6 +37,9 @@ public: std::string getTypeName() const override { return type_name; } std::string getEngineName() const override { return engine_name; } + std::string getSignatures(bool with_structure = true) const override { return with_structure ? signatures_with_structure : signatures_without_structure; } + size_t getMaxNumberOfArguments(bool with_structure = true) const override { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; } + Path getPath() const override { return path; } void setPath(const Path & path_) override { path = path_; } diff --git a/src/Storages/ObjectStorage/Local/Configuration.cpp b/src/Storages/ObjectStorage/Local/Configuration.cpp index a0cf70e6212..0554b9c317c 100644 --- a/src/Storages/ObjectStorage/Local/Configuration.cpp +++ b/src/Storages/ObjectStorage/Local/Configuration.cpp @@ -26,11 +26,11 @@ void StorageLocalConfiguration::fromNamedCollection(const NamedCollection & coll void StorageLocalConfiguration::fromAST(ASTs & args, ContextPtr context, bool with_structure) { - const size_t max_args_num = with_structure ? 4 : 3; - if (args.empty() || args.size() > max_args_num) - { - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Expected not more than {} arguments", max_args_num); - } + if (args.empty() || args.size() > getMaxNumberOfArguments(with_structure)) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Storage Local requires 1 to {} arguments. All supported signatures:\n{}", + getMaxNumberOfArguments(with_structure), + getSignatures(with_structure)); for (auto & arg : args) arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); diff --git a/src/Storages/ObjectStorage/Local/Configuration.h b/src/Storages/ObjectStorage/Local/Configuration.h index c713a81975b..71eec13ca31 100644 --- a/src/Storages/ObjectStorage/Local/Configuration.h +++ b/src/Storages/ObjectStorage/Local/Configuration.h @@ -19,6 +19,20 @@ public: using ConfigurationPtr = StorageObjectStorage::ConfigurationPtr; static constexpr auto type_name = "local"; + /// All possible signatures for Local engine with structure argument (for example for local table function). + static constexpr auto max_number_of_arguments_with_structure = 4; + static constexpr auto signatures_with_structure = + " - path\n" + " - path, format\n" + " - path, format, structure\n" + " - path, format, structure, compression_method\n"; + + /// All possible signatures for S3 engine without structure argument (for example for Local table engine). + static constexpr auto max_number_of_arguments_without_structure = 3; + static constexpr auto signatures_without_structure = + " - path\n" + " - path, format\n" + " - path, format, compression_method\n"; StorageLocalConfiguration() = default; StorageLocalConfiguration(const StorageLocalConfiguration & other) = default; @@ -26,6 +40,9 @@ public: std::string getTypeName() const override { return type_name; } std::string getEngineName() const override { return "Local"; } + std::string getSignatures(bool with_structure = true) const override { return with_structure ? signatures_with_structure : signatures_without_structure; } + size_t getMaxNumberOfArguments(bool with_structure = true) const override { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; } + Path getPath() const override { return path; } void setPath(const Path & path_) override { path = path_; } diff --git a/src/Storages/ObjectStorage/S3/Configuration.cpp b/src/Storages/ObjectStorage/S3/Configuration.cpp index 8bf2669f2af..56bc6ea2f61 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.cpp +++ b/src/Storages/ObjectStorage/S3/Configuration.cpp @@ -170,21 +170,20 @@ void StorageS3Configuration::fromNamedCollection(const NamedCollection & collect void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_structure) { - /// Supported signatures: S3('url') S3('url', 'format') S3('url', 'format', 'compression') S3('url', NOSIGN) S3('url', NOSIGN, 'format') S3('url', NOSIGN, 'format', 'compression') S3('url', 'aws_access_key_id', 'aws_secret_access_key') S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'session_token') S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format') S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'session_token', 'format') S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression') - /// S3('url', 'aws_access_key_id', 'aws_secret_access_key', 'session_token', 'format', 'compression') - /// with optional headers() function - size_t count = StorageURL::evalArgsAndCollectHeaders(args, headers_from_ast, context); - if (count == 0 || count > (with_structure ? 7 : 6)) + if (count == 0 || count > getMaxNumberOfArguments(with_structure)) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Storage S3 requires 1 to 5 arguments: " - "url, [NOSIGN | access_key_id, secret_access_key], name of used format and [compression_method]"); + "Storage S3 requires 1 to {} arguments. All supported signatures:\n{}", + getMaxNumberOfArguments(with_structure), + getSignatures(with_structure)); std::unordered_map engine_args_to_idx; bool no_sign_request = false; - /// For 2 arguments we support 2 possible variants: + /// When adding new arguments in the signature don't forget to update addStructureAndFormatToArgsIfNeeded as well. + + /// For 2 arguments we support: /// - s3(source, format) /// - s3(source, NOSIGN) /// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN or not. @@ -196,10 +195,15 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ else engine_args_to_idx = {{"format", 1}}; } - /// For 3 arguments we support 2 possible variants: + /// For 3 arguments we support: + /// if with_structure == 0: + /// - s3(source, NOSIGN, format) /// - s3(source, format, compression_method) /// - s3(source, access_key_id, secret_access_key) + /// if with_structure == 1: /// - s3(source, NOSIGN, format) + /// - s3(source, format, structure) + /// - s3(source, access_key_id, secret_access_key) /// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN or format name. else if (count == 3) { @@ -219,7 +223,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ else engine_args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}}; } - /// For 4 arguments we support 3 possible variants: + /// For 4 arguments we support: /// if with_structure == 0: /// - s3(source, access_key_id, secret_access_key, session_token) /// - s3(source, access_key_id, secret_access_key, format) @@ -229,7 +233,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ /// - s3(source, access_key_id, secret_access_key, format), /// - s3(source, access_key_id, secret_access_key, session_token) /// - s3(source, NOSIGN, format, structure) - /// We can distinguish them by looking at the 2-nd argument: check if it's a NOSIGN or not. + /// We can distinguish them by looking at the 2-nd argument: check if it's a NOSIGN, format name of something else. else if (count == 4) { auto second_arg = checkAndGetLiteralArgument(args[1], "access_key_id/NOSIGN"); @@ -258,7 +262,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ } } } - /// For 5 arguments we support 2 possible variants: + /// For 5 arguments we support: /// if with_structure == 0: /// - s3(source, access_key_id, secret_access_key, session_token, format) /// - s3(source, access_key_id, secret_access_key, format, compression) @@ -302,13 +306,16 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ } } } + /// For 6 arguments we support: + /// if with_structure == 0: + /// - s3(source, access_key_id, secret_access_key, session_token, format, compression_method) + /// if with_structure == 1: + /// - s3(source, access_key_id, secret_access_key, format, structure, compression_method) + /// - s3(source, access_key_id, secret_access_key, session_token, format, structure) else if (count == 6) { if (with_structure) { - /// - s3(source, access_key_id, secret_access_key, format, structure, compression_method) - /// - s3(source, access_key_id, secret_access_key, session_token, format, structure) - /// We can distinguish them by looking at the 4-th argument: check if it's a format name or not auto fourth_arg = checkAndGetLiteralArgument(args[3], "format/session_token"); if (fourth_arg == "auto" || FormatFactory::instance().exists(fourth_arg)) { @@ -324,6 +331,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_ engine_args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"session_token", 3}, {"format", 4}, {"compression_method", 5}}; } } + /// s3(source, access_key_id, secret_access_key, session_token, format, structure, compression_method) else if (with_structure && count == 7) { engine_args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"session_token", 3}, {"format", 4}, {"structure", 5}, {"compression_method", 6}}; @@ -390,8 +398,8 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( HTTPHeaderEntries tmp_headers; size_t count = StorageURL::evalArgsAndCollectHeaders(args, tmp_headers, context); - if (count == 0 || count > 6) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to 6 arguments in table function, got {}", count); + if (count == 0 || count > getMaxNumberOfArguments()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected 1 to {} arguments in table function s3, got {}", getMaxNumberOfArguments(), count); auto format_literal = std::make_shared(format_); auto structure_literal = std::make_shared(structure_); @@ -403,7 +411,8 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( args.push_back(std::make_shared("auto")); args.push_back(structure_literal); } - /// s3(s3_url, format) or s3(s3_url, NOSIGN) + /// s3(s3_url, format) or + /// s3(s3_url, NOSIGN) /// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN or not. else if (count == 2) { @@ -445,6 +454,7 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( } /// s3(source, format, structure, compression_method) or /// s3(source, access_key_id, secret_access_key, format) or + /// s3(source, access_key_id, secret_access_key, session_token) or /// s3(source, NOSIGN, format, structure) /// We can distinguish them by looking at the 2-nd argument: check if it's NOSIGN, format name or neither. else if (count == 4) @@ -466,18 +476,28 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( } else { - if (checkAndGetLiteralArgument(args[3], "format") == "auto") - args[3] = format_literal; - args.push_back(structure_literal); + auto fourth_arg = checkAndGetLiteralArgument(args[3], "format/session_token"); + if (fourth_arg == "auto" || FormatFactory::instance().exists(fourth_arg)) + { + if (checkAndGetLiteralArgument(args[3], "format") == "auto") + args[3] = format_literal; + args.push_back(structure_literal); + } + else + { + args.push_back(format_literal); + args.push_back(structure_literal); + } } } /// s3(source, access_key_id, secret_access_key, format, structure) or + /// s3(source, access_key_id, secret_access_key, session_token, format) or /// s3(source, NOSIGN, format, structure, compression_method) /// We can distinguish them by looking at the 2-nd argument: check if it's a NOSIGN keyword name or not. else if (count == 5) { auto second_arg = checkAndGetLiteralArgument(args[1], "format/NOSIGN"); - if (boost::iequals(sedond_arg, "NOSIGN")) + if (boost::iequals(second_arg, "NOSIGN")) { if (checkAndGetLiteralArgument(args[2], "format") == "auto") args[2] = format_literal; @@ -485,20 +505,50 @@ void StorageS3Configuration::addStructureAndFormatToArgsIfNeeded( args[3] = structure_literal; } else + { + auto fourth_arg = checkAndGetLiteralArgument(args[3], "format/session_token"); + if (fourth_arg == "auto" || FormatFactory::instance().exists(fourth_arg)) + { + if (checkAndGetLiteralArgument(args[3], "format") == "auto") + args[3] = format_literal; + if (checkAndGetLiteralArgument(args[4], "structure") == "auto") + args[4] = structure_literal; + } + else + { + if (checkAndGetLiteralArgument(args[4], "format") == "auto") + args[4] = format_literal; + args.push_back(structure_literal); + } + } + } + /// s3(source, access_key_id, secret_access_key, format, structure, compression) or + /// s3(source, access_key_id, secret_access_key, session_token, format, structure) + else if (count == 6) + { + auto fourth_arg = checkAndGetLiteralArgument(args[3], "format/session_token"); + if (fourth_arg == "auto" || FormatFactory::instance().exists(fourth_arg)) { if (checkAndGetLiteralArgument(args[3], "format") == "auto") args[3] = format_literal; if (checkAndGetLiteralArgument(args[4], "structure") == "auto") args[4] = structure_literal; } + else + { + if (checkAndGetLiteralArgument(args[4], "format") == "auto") + args[4] = format_literal; + if (checkAndGetLiteralArgument(args[5], "format") == "auto") + args[5] = structure_literal; + } } - /// s3(source, access_key_id, secret_access_key, format, structure, compression) - else if (count == 6) + /// s3(source, access_key_id, secret_access_key, session_token, format, structure, compression_method) + else if (count == 7) { - if (checkAndGetLiteralArgument(args[3], "format") == "auto") - args[3] = format_literal; - if (checkAndGetLiteralArgument(args[4], "structure") == "auto") - args[4] = structure_literal; + if (checkAndGetLiteralArgument(args[4], "format") == "auto") + args[4] = format_literal; + if (checkAndGetLiteralArgument(args[5], "format") == "auto") + args[5] = structure_literal; } } } diff --git a/src/Storages/ObjectStorage/S3/Configuration.h b/src/Storages/ObjectStorage/S3/Configuration.h index 0aed0f366a6..6eec216f3fe 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.h +++ b/src/Storages/ObjectStorage/S3/Configuration.h @@ -14,8 +14,48 @@ class StorageS3Configuration : public StorageObjectStorage::Configuration public: using ConfigurationPtr = StorageObjectStorage::ConfigurationPtr; + static constexpr auto xx = + 42; + static constexpr auto type_name = "s3"; static constexpr auto namespace_name = "bucket"; + /// All possible signatures for S3 storage with structure argument (for example for s3 table function). + static constexpr auto max_number_of_arguments_with_structure = 7; + static constexpr auto signatures_with_structure = + " - url\n" + " - url, NOSIGN\n" + " - url, format\n" + " - url, NOSIGN, format\n" + " - url, format, structure\n" + " - url, NOSIGN, format, structure\n" + " - url, format, structure, compression_method\n" + " - url, NOSIGN, format, structure, compression_method\n" + " - url, access_key_id, secret_access_key\n" + " - url, access_key_id, secret_access_key, session_token\n" + " - url, access_key_id, secret_access_key, format\n" + " - url, access_key_id, secret_access_key, session_token, format\n" + " - url, access_key_id, secret_access_key, format, structure\n" + " - url, access_key_id, secret_access_key, session_token, format, structure\n" + " - url, access_key_id, secret_access_key, format, structure, compression_method\n" + " - url, access_key_id, secret_access_key, session_token, format, structure, compression_method\n" + "All signatures supports optional headers (specified as `headers('name'='value', 'name2'='value2')`)"; + + /// All possible signatures for S3 storage without structure argument (for example for S3 table engine). + static constexpr auto max_number_of_arguments_without_structure = 6; + static constexpr auto signatures_without_structure = + " - url\n" + " - url, NOSIGN\n" + " - url, format\n" + " - url, NOSIGN, format\n" + " - url, format, compression_method\n" + " - url, NOSIGN, format, compression_method\n" + " - url, access_key_id, secret_access_key\n" + " - url, access_key_id, secret_access_key, session_token\n" + " - url, access_key_id, secret_access_key, format\n" + " - url, access_key_id, secret_access_key, session_token, format\n" + " - url, access_key_id, secret_access_key, format, compression_method\n" + " - url, access_key_id, secret_access_key, session_token, format, compression_method\n" + "All signatures supports optional headers (specified as `headers('name'='value', 'name2'='value2')`)"; StorageS3Configuration() = default; StorageS3Configuration(const StorageS3Configuration & other); @@ -24,6 +64,9 @@ public: std::string getEngineName() const override { return url.storage_name; } std::string getNamespaceType() const override { return namespace_name; } + std::string getSignatures(bool with_structure = true) const override { return with_structure ? signatures_with_structure : signatures_without_structure; } + size_t getMaxNumberOfArguments(bool with_structure = true) const override { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; } + Path getPath() const override { return url.key; } void setPath(const Path & path) override { url.key = path; } diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index f39586c23b4..ba7fd350bae 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -170,6 +170,10 @@ public: /// buckets in S3. If object storage doesn't have any namepaces return empty string. virtual std::string getNamespaceType() const { return "namespace"; } + /// Return the string containing all supported signatures for this storage arguments. + virtual std::string getSignatures(bool with_structure = true) const = 0; + virtual size_t getMaxNumberOfArguments(bool with_structure = true) const = 0; + virtual Path getPath() const = 0; virtual void setPath(const Path & path) = 0; diff --git a/src/TableFunctions/ITableFunctionCluster.h b/src/TableFunctions/ITableFunctionCluster.h index 28dc43f350b..744d7139d16 100644 --- a/src/TableFunctions/ITableFunctionCluster.h +++ b/src/TableFunctions/ITableFunctionCluster.h @@ -23,7 +23,6 @@ class ITableFunctionCluster : public Base { public: String getName() const override = 0; - String getSignature() const override = 0; static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure_, const String & format_, const ContextPtr & context) { @@ -46,7 +45,11 @@ protected: void parseArgumentsImpl(ASTs & args, const ContextPtr & context) override { if (args.empty()) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "The signature of table function {} shall be the following:\n{}", getName(), getSignature()); + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "The function {} should have arguments. The first argument must be the cluster name and the rest are the arguments of " + "corresponding table function", + getName()); /// Evaluate only first argument, everything else will be done Base class args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(args[0], context); diff --git a/src/TableFunctions/ITableFunctionFileLike.cpp b/src/TableFunctions/ITableFunctionFileLike.cpp index 1a58be4f75b..23e59494f61 100644 --- a/src/TableFunctions/ITableFunctionFileLike.cpp +++ b/src/TableFunctions/ITableFunctionFileLike.cpp @@ -57,7 +57,7 @@ void ITableFunctionFileLike::parseArguments(const ASTPtr & ast_function, Context void ITableFunctionFileLike::parseArgumentsImpl(ASTs & args, const ContextPtr & context) { - if (args.empty() || args.size() > 4) + if (args.empty() || args.size() > getMaxNumberOfArguments()) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "The signature of table function {} shall be the following:\n{}", getName(), getSignature()); for (auto & arg : args) diff --git a/src/TableFunctions/ITableFunctionFileLike.h b/src/TableFunctions/ITableFunctionFileLike.h index ba1b7d2bb3f..4c97507b8d1 100644 --- a/src/TableFunctions/ITableFunctionFileLike.h +++ b/src/TableFunctions/ITableFunctionFileLike.h @@ -15,6 +15,7 @@ class Context; class ITableFunctionFileLike : public ITableFunction { public: + static constexpr auto max_number_of_arguments = 4; static constexpr auto signature = " - filename\n" " - filename, format\n" " - filename, format, structure\n" @@ -32,7 +33,7 @@ public: NameSet getVirtualsToCheckBeforeUsingStructureHint() const override; - static size_t getMaxNumberOfArguments() { return 4; } + static size_t getMaxNumberOfArguments() { return max_number_of_arguments; } static void updateStructureAndFormatArgumentsIfNeeded(ASTs & args, const String & structure, const String & format, const ContextPtr &); diff --git a/src/TableFunctions/TableFunctionObjectStorage.h b/src/TableFunctions/TableFunctionObjectStorage.h index 6fa10face82..6b923f93e75 100644 --- a/src/TableFunctions/TableFunctionObjectStorage.h +++ b/src/TableFunctions/TableFunctionObjectStorage.h @@ -23,83 +23,42 @@ struct AzureDefinition { static constexpr auto name = "azureBlobStorage"; static constexpr auto storage_type_name = "Azure"; - static constexpr auto signature = " - connection_string, container_name, blobpath\n" - " - connection_string, container_name, blobpath, structure \n" - " - connection_string, container_name, blobpath, format \n" - " - connection_string, container_name, blobpath, format, compression \n" - " - connection_string, container_name, blobpath, format, compression, structure \n" - " - storage_account_url, container_name, blobpath, account_name, account_key\n" - " - storage_account_url, container_name, blobpath, account_name, account_key, structure\n" - " - storage_account_url, container_name, blobpath, account_name, account_key, format\n" - " - storage_account_url, container_name, blobpath, account_name, account_key, format, compression\n" - " - storage_account_url, container_name, blobpath, account_name, account_key, format, compression, structure\n"; - static constexpr auto max_number_of_arguments = 8; }; struct S3Definition { static constexpr auto name = "s3"; static constexpr auto storage_type_name = "S3"; - static constexpr auto signature = " - url\n" - " - url, format\n" - " - url, format, structure\n" - " - url, format, structure, compression_method\n" - " - url, access_key_id, secret_access_key\n" - " - url, access_key_id, secret_access_key, session_token\n" - " - url, access_key_id, secret_access_key, format\n" - " - url, access_key_id, secret_access_key, session_token, format\n" - " - url, access_key_id, secret_access_key, format, structure\n" - " - url, access_key_id, secret_access_key, session_token, format, structure\n" - " - url, access_key_id, secret_access_key, format, structure, compression_method\n" - " - url, access_key_id, secret_access_key, session_token, format, structure, compression_method\n" - "All signatures supports optional headers (specified as `headers('name'='value', 'name2'='value2')`)"; - static constexpr auto max_number_of_arguments = 8; }; struct GCSDefinition { static constexpr auto name = "gcs"; static constexpr auto storage_type_name = "GCS"; - static constexpr auto signature = S3Definition::signature; - static constexpr auto max_number_of_arguments = S3Definition::max_number_of_arguments; }; struct COSNDefinition { static constexpr auto name = "cosn"; static constexpr auto storage_type_name = "COSN"; - static constexpr auto signature = S3Definition::signature; - static constexpr auto max_number_of_arguments = S3Definition::max_number_of_arguments; }; struct OSSDefinition { static constexpr auto name = "oss"; static constexpr auto storage_type_name = "OSS"; - static constexpr auto signature = S3Definition::signature; - static constexpr auto max_number_of_arguments = S3Definition::max_number_of_arguments; }; struct HDFSDefinition { static constexpr auto name = "hdfs"; static constexpr auto storage_type_name = "HDFS"; - static constexpr auto signature = " - uri\n" - " - uri, format\n" - " - uri, format, structure\n" - " - uri, format, structure, compression_method\n"; - static constexpr auto max_number_of_arguments = 4; }; struct LocalDefinition { static constexpr auto name = "local"; static constexpr auto storage_type_name = "Local"; - static constexpr auto signature = " - path\n" - " - path, format\n" - " - path, format, structure\n" - " - path, format, structure, compression_method\n"; - static constexpr auto max_number_of_arguments = 4; }; template @@ -107,14 +66,9 @@ class TableFunctionObjectStorage : public ITableFunction { public: static constexpr auto name = Definition::name; - static constexpr auto signature = Definition::signature; - - static size_t getMaxNumberOfArguments() { return Definition::max_number_of_arguments; } String getName() const override { return name; } - virtual String getSignature() const { return signature; } - bool hasStaticStructure() const override { return configuration->structure != "auto"; } bool needStructureHint() const override { return configuration->structure == "auto"; } diff --git a/src/TableFunctions/TableFunctionObjectStorageCluster.h b/src/TableFunctions/TableFunctionObjectStorageCluster.h index 296791b8bda..11e6c1fde82 100644 --- a/src/TableFunctions/TableFunctionObjectStorageCluster.h +++ b/src/TableFunctions/TableFunctionObjectStorageCluster.h @@ -19,40 +19,22 @@ struct AzureClusterDefinition { static constexpr auto name = "azureBlobStorageCluster"; static constexpr auto storage_type_name = "AzureBlobStorageCluster"; - static constexpr auto signature = " - cluster, connection_string|storage_account_url, container_name, blobpath, [account_name, account_key, format, compression, structure]"; - static constexpr auto max_number_of_arguments = AzureDefinition::max_number_of_arguments + 1; }; struct S3ClusterDefinition { static constexpr auto name = "s3Cluster"; static constexpr auto storage_type_name = "S3Cluster"; - static constexpr auto signature = " - cluster, url\n" - " - cluster, url, format\n" - " - cluster, url, format, structure\n" - " - cluster, url, access_key_id, secret_access_key\n" - " - cluster, url, format, structure, compression_method\n" - " - cluster, url, access_key_id, secret_access_key, format\n" - " - cluster, url, access_key_id, secret_access_key, format, structure\n" - " - cluster, url, access_key_id, secret_access_key, format, structure, compression_method\n" - " - cluster, url, access_key_id, secret_access_key, session_token, format, structure, compression_method\n" - "All signatures supports optional headers (specified as `headers('name'='value', 'name2'='value2')`)"; - static constexpr auto max_number_of_arguments = S3Definition::max_number_of_arguments + 1; }; struct HDFSClusterDefinition { static constexpr auto name = "hdfsCluster"; static constexpr auto storage_type_name = "HDFSCluster"; - static constexpr auto signature = " - cluster_name, uri\n" - " - cluster_name, uri, format\n" - " - cluster_name, uri, format, structure\n" - " - cluster_name, uri, format, structure, compression_method\n"; - static constexpr auto max_number_of_arguments = HDFSDefinition::max_number_of_arguments + 1; }; /** -* Class implementing s3/hdfs/azureBlobStorage)Cluster(...) table functions, +* Class implementing s3/hdfs/azureBlobStorageCluster(...) table functions, * which allow to process many files from S3/HDFS/Azure blob storage on a specific cluster. * On initiator it creates a connection to _all_ nodes in cluster, discloses asterisks * in file path and dispatch each file dynamically. @@ -64,10 +46,8 @@ class TableFunctionObjectStorageCluster : public ITableFunctionCluster; diff --git a/tests/integration/test_s3_cluster/test.py b/tests/integration/test_s3_cluster/test.py index b2954276ca5..c31851fdfe9 100644 --- a/tests/integration/test_s3_cluster/test.py +++ b/tests/integration/test_s3_cluster/test.py @@ -460,12 +460,19 @@ def test_cluster_format_detection(started_cluster): assert result == expected_result + def test_cluster_default_expression(started_cluster): node = started_cluster.instances["s0_0_0"] - node.query("insert into function s3('http://minio1:9001/root/data/data1', 'minio', 'minio123', JSONEachRow) select 1 as id settings s3_truncate_on_insert=1") - node.query("insert into function s3('http://minio1:9001/root/data/data2', 'minio', 'minio123', JSONEachRow) select * from numbers(0) settings s3_truncate_on_insert=1") - node.query("insert into function s3('http://minio1:9001/root/data/data3', 'minio', 'minio123', JSONEachRow) select 2 as id settings s3_truncate_on_insert=1") + node.query( + "insert into function s3('http://minio1:9001/root/data/data1', 'minio', 'minio123', JSONEachRow) select 1 as id settings s3_truncate_on_insert=1" + ) + node.query( + "insert into function s3('http://minio1:9001/root/data/data2', 'minio', 'minio123', JSONEachRow) select * from numbers(0) settings s3_truncate_on_insert=1" + ) + node.query( + "insert into function s3('http://minio1:9001/root/data/data3', 'minio', 'minio123', JSONEachRow) select 2 as id settings s3_truncate_on_insert=1" + ) expected_result = node.query( "SELECT * FROM s3('http://minio1:9001/root/data/data{1,2,3}', 'minio', 'minio123', 'JSONEachRow', 'id UInt32, date Date DEFAULT 18262') order by id" @@ -499,4 +506,4 @@ def test_cluster_default_expression(started_cluster): "SELECT * FROM s3Cluster(cluster_simple, test_s3_with_default) order by id" ) - assert result == expected_result \ No newline at end of file + assert result == expected_result diff --git a/tests/queries/0_stateless/01801_s3_cluster.reference b/tests/queries/0_stateless/01801_s3_cluster.reference index 4166d1718b1..c77baca9f09 100644 --- a/tests/queries/0_stateless/01801_s3_cluster.reference +++ b/tests/queries/0_stateless/01801_s3_cluster.reference @@ -190,3 +190,195 @@ 20 21 22 23 24 25 26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 +0 0 0 +0 0 0 +0 0 0 +1 2 3 +4 5 6 +7 8 9 +10 11 12 +13 14 15 +16 17 18 +20 21 22 +23 24 25 +26 27 28 diff --git a/tests/queries/0_stateless/01801_s3_cluster.sql b/tests/queries/0_stateless/01801_s3_cluster.sql index 68d90ea4be0..f94f1102dc0 100644 --- a/tests/queries/0_stateless/01801_s3_cluster.sql +++ b/tests/queries/0_stateless/01801_s3_cluster.sql @@ -2,21 +2,37 @@ -- Tag no-fasttest: Depends on AWS select * from s3('http://localhost:11111/test/{a,b,c}.tsv') ORDER BY c1, c2, c3; +select * from s3('http://localhost:11111/test/{a,b,c}.tsv', NOSIGN) ORDER BY c1, c2, c3; select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'TSV') ORDER BY c1, c2, c3; +select * from s3('http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV') ORDER BY c1, c2, c3; select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3; -select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest') ORDER BY c1, c2, c3; +select * from s3('http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3; select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3; +select * from s3('http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3; +select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest') ORDER BY c1, c2, c3; +select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '') ORDER BY c1, c2, c3; select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV') ORDER BY c1, c2, c3; +select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV') ORDER BY c1, c2, c3; select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3; +select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3; select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3; +select * from s3('http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3; select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv') ORDER BY c1, c2, c3; +select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', NOSIGN) ORDER BY c1, c2, c3; select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'TSV') ORDER BY c1, c2, c3; +select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV') ORDER BY c1, c2, c3; select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3; -select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest') ORDER BY c1, c2, c3; +select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3; select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3; +select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', NOSIGN, 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3; +select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest') ORDER BY c1, c2, c3; +select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '') ORDER BY c1, c2, c3; select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV') ORDER BY c1, c2, c3; +select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV') ORDER BY c1, c2, c3; select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3; +select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64') ORDER BY c1, c2, c3; select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3; +select * from s3Cluster('test_cluster_two_shards_localhost', 'http://localhost:11111/test/{a,b,c}.tsv', 'test', 'testtest', '', 'TSV', 'c1 UInt64, c2 UInt64, c3 UInt64', 'auto') ORDER BY c1, c2, c3; From b7e1dda2e0555a5e08bda8d0bb5085806214d77f Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 2 Sep 2024 20:13:44 +0000 Subject: [PATCH 127/215] Remove unneded code --- src/Storages/ObjectStorage/S3/Configuration.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Storages/ObjectStorage/S3/Configuration.h b/src/Storages/ObjectStorage/S3/Configuration.h index 6eec216f3fe..63c1d24de62 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.h +++ b/src/Storages/ObjectStorage/S3/Configuration.h @@ -14,9 +14,6 @@ class StorageS3Configuration : public StorageObjectStorage::Configuration public: using ConfigurationPtr = StorageObjectStorage::ConfigurationPtr; - static constexpr auto xx = - 42; - static constexpr auto type_name = "s3"; static constexpr auto namespace_name = "bucket"; /// All possible signatures for S3 storage with structure argument (for example for s3 table function). From 13f4eb3fac6c2c0781351ee5db382383193b2af5 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Mon, 2 Sep 2024 22:24:53 +0200 Subject: [PATCH 128/215] Fix for graphite merge mode --- src/Storages/MergeTree/MergeTask.cpp | 25 ++++++++++++------------- src/Storages/MergeTree/MergeTask.h | 2 +- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 3bee2ecb0d9..fb5bbc4729c 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -1224,7 +1224,8 @@ public: UInt64 merge_block_size_rows_, UInt64 merge_block_size_bytes_, bool blocks_are_granules_size_, - bool cleanup_) + bool cleanup_, + time_t time_of_merge_) : ITransformingStep(input_stream_, input_stream_.header, Traits{}) // TODO proper traits? , sort_description(sort_description_) , partition_key_columns(partition_key_columns_) @@ -1234,6 +1235,7 @@ public: , merge_block_size_bytes(merge_block_size_bytes_) , blocks_are_granules_size(blocks_are_granules_size_) , cleanup(cleanup_) + , time_of_merge(time_of_merge_) {} String getName() const override { return "ApplyMergePolicy"; } @@ -1245,10 +1247,6 @@ public: /// that is going in insertion order. ProcessorPtr merged_transform; -// /// There is no sense to have the block size bigger than one granule for merge operations. -// const UInt64 merge_block_size_rows = data_settings->merge_max_block_size; -// const UInt64 merge_block_size_bytes = data_settings->merge_max_block_size_bytes; - const auto &header = pipeline.getHeader(); const auto input_streams_count = pipeline.getNumStreams(); @@ -1330,15 +1328,15 @@ public: } private: - SortDescription sort_description; - Names partition_key_columns; - MergeTreeData::MergingParams merging_params{}; + const SortDescription sort_description; + const Names partition_key_columns; + const MergeTreeData::MergingParams merging_params{}; WriteBuffer * rows_sources_write_buf; const UInt64 merge_block_size_rows; const UInt64 merge_block_size_bytes; - bool blocks_are_granules_size; - bool cleanup{false}; - time_t time_of_merge{0}; + const bool blocks_are_granules_size; + const bool cleanup{false}; + const time_t time_of_merge{0}; }; @@ -1403,7 +1401,7 @@ private: }; -void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() +void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const { /** Read from all parts, merge and write into a new one. * In passing, we calculate expression for sorting. @@ -1516,7 +1514,8 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() data_settings->merge_max_block_size, data_settings->merge_max_block_size_bytes, ctx->blocks_are_granules_size, - global_ctx->cleanup); + global_ctx->cleanup, + global_ctx->time_of_merge); merge_step->setStepDescription("Merge sorted parts"); merge_parts_query_plan.addStep(std::move(merge_step)); } diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index c80995888d4..a5d7851932c 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -291,7 +291,7 @@ private: bool executeMergeProjections(); MergeAlgorithm chooseMergeAlgorithm() const; - void createMergedStream(); + void createMergedStream() const; void extractMergingAndGatheringColumns() const; void setRuntimeContext(StageRuntimeContextPtr local, StageRuntimeContextPtr global) override From 48cacd6f310c107c1dd0239a7639527adb054b69 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Mon, 2 Sep 2024 22:36:42 +0200 Subject: [PATCH 129/215] Use query plan for column vertical merges --- src/Storages/MergeTree/MergeTask.cpp | 175 ++++++++++++++++++++------- src/Storages/MergeTree/MergeTask.h | 5 +- 2 files changed, 133 insertions(+), 47 deletions(-) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index fb5bbc4729c..75fd61ae4be 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -804,35 +804,106 @@ bool MergeTask::VerticalMergeStage::prepareVerticalMergeForAllColumns() const bool all_parts_on_remote_disks = std::ranges::all_of(global_ctx->future_part->parts, [](const auto & part) { return part->isStoredOnRemoteDisk(); }); ctx->use_prefetch = all_parts_on_remote_disks && global_ctx->data->getSettings()->vertical_merge_remote_filesystem_prefetch; - if (ctx->use_prefetch && ctx->it_name_and_type != global_ctx->gathering_columns.end()) - ctx->prepared_pipe = createPipeForReadingOneColumn(ctx->it_name_and_type->name); +// if (ctx->use_prefetch && ctx->it_name_and_type != global_ctx->gathering_columns.end()) +// ctx->prepared_pipe = createPipeForReadingOneColumn(ctx->it_name_and_type->name); return false; } -Pipe MergeTask::VerticalMergeStage::createPipeForReadingOneColumn(const String & column_name) const +QueryPlan MergeTask::VerticalMergeStage::createPlanForReadingOneColumn(const String & column_name) const { - Pipes pipes; - for (size_t part_num = 0; part_num < global_ctx->future_part->parts.size(); ++part_num) + /// Read from all parts + std::vector plans; + for (const auto & part : global_ctx->future_part->parts) { - Pipe pipe = createMergeTreeSequentialSource( + auto plan_for_part = std::make_unique(); + createReadFromPartStep( MergeTreeSequentialSourceType::Merge, + *plan_for_part, *global_ctx->data, global_ctx->storage_snapshot, - global_ctx->future_part->parts[part_num], + part, Names{column_name}, - /*mark_ranges=*/ {}, global_ctx->input_rows_filtered, /*apply_deleted_mask=*/ true, + std::nullopt, ctx->read_with_direct_io, - ctx->use_prefetch); + ctx->use_prefetch, + global_ctx->context, + getLogger("VerticalMergeStage")); - pipes.emplace_back(std::move(pipe)); + plans.emplace_back(std::move(plan_for_part)); } - return Pipe::unitePipes(std::move(pipes)); + QueryPlan merge_parts_query_plan; + + /// Union of all parts streams + { + DataStreams input_streams; + input_streams.reserve(plans.size()); + for (auto & plan : plans) + input_streams.emplace_back(plan->getCurrentDataStream()); + + auto union_step = std::make_unique(std::move(input_streams)); + merge_parts_query_plan.unitePlans(std::move(union_step), std::move(plans)); + } + + return merge_parts_query_plan; } +/// Gathers values from all parts for one column using rows sources temporary file +class ColumnGathererStep : public ITransformingStep +{ +public: + ColumnGathererStep( + const DataStream & input_stream_, + CompressedReadBufferFromFile * rows_sources_read_buf_, + UInt64 merge_block_size_rows_, + UInt64 merge_block_size_bytes_, + bool is_result_sparse_) + : ITransformingStep(input_stream_, input_stream_.header, Traits{}) // TODO proper traits? + , rows_sources_read_buf(rows_sources_read_buf_) + , merge_block_size_rows(merge_block_size_rows_) + , merge_block_size_bytes(merge_block_size_bytes_) + , is_result_sparse(is_result_sparse_) + {} + + String getName() const override { return "ColumnGatherer"; } + + void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & /*pipelineSettings*/) override + { + const auto &header = pipeline.getHeader(); + const auto input_streams_count = pipeline.getNumStreams(); + + rows_sources_read_buf->seek(0, 0); + + auto transform = std::make_unique( + header, + input_streams_count, + *rows_sources_read_buf, + merge_block_size_rows, + merge_block_size_bytes, + is_result_sparse); + + pipeline.addTransform(std::move(transform)); + } + + void updateOutputStream() override + { + output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); + + /// TODO: is this correct? + output_stream->sort_scope = DataStream::SortScope::None; + } + +private: + MergeTreeData::MergingParams merging_params{}; + CompressedReadBufferFromFile * rows_sources_read_buf; + const UInt64 merge_block_size_rows; + const UInt64 merge_block_size_bytes; + const bool is_result_sparse; +}; + void MergeTask::VerticalMergeStage::prepareVerticalMergeForOneColumn() const { const auto & column_name = ctx->it_name_and_type->name; @@ -840,50 +911,64 @@ void MergeTask::VerticalMergeStage::prepareVerticalMergeForOneColumn() const ctx->progress_before = global_ctx->merge_list_element_ptr->progress.load(std::memory_order_relaxed); global_ctx->column_progress = std::make_unique(ctx->progress_before, ctx->column_sizes->columnWeight(column_name)); - Pipe pipe; - if (ctx->prepared_pipe) - { - pipe = std::move(*ctx->prepared_pipe); +// Pipe pipe; +//// if (ctx->prepared_pipe) +//// { +//// pipe = std::move(*ctx->prepared_pipe); +//// +//// auto next_column_it = std::next(ctx->it_name_and_type); +//// if (next_column_it != global_ctx->gathering_columns.end()) +//// ctx->prepared_pipe = createPipeForReadingOneColumn(next_column_it->name); +//// } +//// else +// { +// pipe = createPipeForReadingOneColumn(column_name); +// } - auto next_column_it = std::next(ctx->it_name_and_type); - if (next_column_it != global_ctx->gathering_columns.end()) - ctx->prepared_pipe = createPipeForReadingOneColumn(next_column_it->name); - } - else + auto merge_column_query_plan = createPlanForReadingOneColumn(column_name); + + /// Add column gatherer step { - pipe = createPipeForReadingOneColumn(column_name); +// ctx->rows_sources_read_buf->seek(0, 0); + bool is_result_sparse = global_ctx->new_data_part->getSerialization(column_name)->getKind() == ISerialization::Kind::SPARSE; + const auto data_settings = global_ctx->data->getSettings(); + auto merge_step = std::make_unique( + merge_column_query_plan.getCurrentDataStream(), + ctx->rows_sources_read_buf.get(), //global_ctx->rows_sources_temporary_file_name, + data_settings->merge_max_block_size, + data_settings->merge_max_block_size_bytes, + is_result_sparse); + merge_step->setStepDescription("Gather column"); + merge_column_query_plan.addStep(std::move(merge_step)); } - ctx->rows_sources_read_buf->seek(0, 0); - bool is_result_sparse = global_ctx->new_data_part->getSerialization(column_name)->getKind() == ISerialization::Kind::SPARSE; - - const auto data_settings = global_ctx->data->getSettings(); - auto transform = std::make_unique( - pipe.getHeader(), - pipe.numOutputPorts(), - *ctx->rows_sources_read_buf, - data_settings->merge_max_block_size, - data_settings->merge_max_block_size_bytes, - is_result_sparse); - - pipe.addTransform(std::move(transform)); - + /// Add expression step for indexes MergeTreeIndices indexes_to_recalc; - auto indexes_it = global_ctx->skip_indexes_by_column.find(column_name); - - if (indexes_it != global_ctx->skip_indexes_by_column.end()) + IndicesDescription indexes_to_recalc_description; { - indexes_to_recalc = MergeTreeIndexFactory::instance().getMany(indexes_it->second); + auto indexes_it = global_ctx->skip_indexes_by_column.find(column_name); - pipe.addTransform(std::make_shared( - pipe.getHeader(), - indexes_it->second.getSingleExpressionForIndices(global_ctx->metadata_snapshot->getColumns(), - global_ctx->data->getContext()))); + if (indexes_it != global_ctx->skip_indexes_by_column.end()) + { + indexes_to_recalc_description = indexes_it->second; + indexes_to_recalc = MergeTreeIndexFactory::instance().getMany(indexes_it->second); - pipe.addTransform(std::make_shared(pipe.getHeader())); + auto indices_expression_dag = indexes_it->second.getSingleExpressionForIndices(global_ctx->metadata_snapshot->getColumns(), global_ctx->data->getContext())->getActionsDAG().clone(); + auto calculate_indices_expression_step = std::make_unique( + merge_column_query_plan.getCurrentDataStream(), + std::move(indices_expression_dag)); + merge_column_query_plan.addStep(std::move(calculate_indices_expression_step)); + } } - ctx->column_parts_pipeline = QueryPipeline(std::move(pipe)); + { + auto pipelineSettings = BuildQueryPipelineSettings::fromContext(global_ctx->context); + auto builder = merge_column_query_plan.buildQueryPipeline( + QueryPlanOptimizationSettings::fromContext(global_ctx->context), + pipelineSettings); + + ctx->column_parts_pipeline = QueryPipelineBuilder::getPipeline(std::move(*builder)); + } /// Dereference unique_ptr ctx->column_parts_pipeline.setProgressCallback(MergeProgressCallback( diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index a5d7851932c..b36f5f832d9 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -333,7 +333,8 @@ private: Float64 progress_before = 0; std::unique_ptr column_to{nullptr}; - std::optional prepared_pipe; +// TODO: is this really needed for prefetch? +// std::optional prepared_pipe; size_t max_delayed_streams = 0; bool use_prefetch = false; std::list> delayed_streams; @@ -378,7 +379,7 @@ private: bool executeVerticalMergeForOneColumn() const; void finalizeVerticalMergeForOneColumn() const; - Pipe createPipeForReadingOneColumn(const String & column_name) const; + QueryPlan createPlanForReadingOneColumn(const String & column_name) const; VerticalMergeRuntimeContextPtr ctx; GlobalRuntimeContextPtr global_ctx; From d28cba981ccd6a58939854a0204d654c6075337d Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Tue, 3 Sep 2024 08:59:01 +0200 Subject: [PATCH 130/215] Fix clang_tidy --- src/Storages/MergeTree/MergeTask.cpp | 6 +++--- src/Storages/MergeTree/MergeTask.h | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 75fd61ae4be..cafc11fc34d 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -195,7 +195,7 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::extractMergingAndGatheringColu } } -bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() +bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const { ProfileEvents::increment(ProfileEvents::Merge); @@ -657,7 +657,7 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::constructTaskForProjectionPart } -bool MergeTask::ExecuteAndFinalizeHorizontalPart::executeMergeProjections() // NOLINT +bool MergeTask::ExecuteAndFinalizeHorizontalPart::executeMergeProjections() const { /// In case if there are no projections we didn't construct a task if (!ctx->merge_projection_parts_task_ptr) @@ -676,7 +676,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::executeMergeProjections() // N return true; } -bool MergeTask::ExecuteAndFinalizeHorizontalPart::executeImpl() +bool MergeTask::ExecuteAndFinalizeHorizontalPart::executeImpl() const { Stopwatch watch(CLOCK_MONOTONIC_COARSE); UInt64 step_time_ms = global_ctx->data->getSettings()->background_task_preferred_step_execution_time_ms.totalMilliseconds(); diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index b36f5f832d9..a30ab4712d5 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -268,12 +268,12 @@ private: { bool execute() override; - bool prepare(); - bool executeImpl(); + bool prepare() const; + bool executeImpl() const; void finalize() const; /// NOTE: Using pointer-to-member instead of std::function and lambda makes stacktraces much more concise and readable - using ExecuteAndFinalizeHorizontalPartSubtasks = std::array; + using ExecuteAndFinalizeHorizontalPartSubtasks = std::array; const ExecuteAndFinalizeHorizontalPartSubtasks subtasks { @@ -288,7 +288,7 @@ private: void calculateProjections(const Block & block) const; void finalizeProjections() const; void constructTaskForProjectionPartsMerge() const; - bool executeMergeProjections(); + bool executeMergeProjections() const; MergeAlgorithm chooseMergeAlgorithm() const; void createMergedStream() const; From 919f51533ab26e23dae8c0f2fb7f8ca65112d3e2 Mon Sep 17 00:00:00 2001 From: marco-vb Date: Tue, 3 Sep 2024 09:37:03 +0000 Subject: [PATCH 131/215] Fixed style issues. --- src/Access/Authentication.cpp | 2 +- .../test_ssl_cert_authentication/configs/ssl_config.xml | 0 .../configs/users_with_ssl_auth.xml | 0 tests/integration/test_ssl_cert_authentication/test.py | 9 +++++++-- 4 files changed, 8 insertions(+), 3 deletions(-) mode change 100755 => 100644 tests/integration/test_ssl_cert_authentication/configs/ssl_config.xml mode change 100755 => 100644 tests/integration/test_ssl_cert_authentication/configs/users_with_ssl_auth.xml diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 3acf8a89b70..90c82b466db 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -248,7 +248,7 @@ bool Authentication::areCredentialsValid( return true; // Wildcard support (1 only) - if (subject.contains('*')) + if (subject.contains('*')) { auto prefix = std::string_view(subject).substr(0, subject.find('*')); auto suffix = std::string_view(subject).substr(subject.find('*') + 1); diff --git a/tests/integration/test_ssl_cert_authentication/configs/ssl_config.xml b/tests/integration/test_ssl_cert_authentication/configs/ssl_config.xml old mode 100755 new mode 100644 diff --git a/tests/integration/test_ssl_cert_authentication/configs/users_with_ssl_auth.xml b/tests/integration/test_ssl_cert_authentication/configs/users_with_ssl_auth.xml old mode 100755 new mode 100644 diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 1adfa4855f1..26ee33f3f6f 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -370,14 +370,19 @@ def test_x509_san_support(): == "CREATE USER jemma IDENTIFIED WITH ssl_certificate SAN \\'URI:spiffe://foo.com/bar\\', \\'URI:spiffe://foo.com/baz\\'\n" ) + def test_x509_san_wildcard_support(): assert ( - execute_query_native(instance, "SELECT currentUser()", user="stewie", cert_name="client5") + execute_query_native( + instance, "SELECT currentUser()", user="stewie", cert_name="client5" + ) == "stewie\n" ) assert ( - instance.query("SELECT name, auth_type, auth_params FROM system.users WHERE name='stewie'") + instance.query( + "SELECT name, auth_type, auth_params FROM system.users WHERE name='stewie'" + ) == 'stewie\tssl_certificate\t{"subject_alt_names":["URI:spiffe:\\\\/\\\\/bar.com\\\\/foo\\\\/*\\\\/far"]}\n' ) From cec5037c4dbf0124a008ce62d05ad0abd330ec2b Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Tue, 3 Sep 2024 12:05:26 +0200 Subject: [PATCH 132/215] Update Configuration.h --- src/Storages/ObjectStorage/Azure/Configuration.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/ObjectStorage/Azure/Configuration.h b/src/Storages/ObjectStorage/Azure/Configuration.h index 2e4719ea24d..311ad76814d 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.h +++ b/src/Storages/ObjectStorage/Azure/Configuration.h @@ -23,7 +23,7 @@ public: static constexpr auto type_name = "azure"; static constexpr auto engine_name = "Azure"; /// All possible signatures for Azure engine with structure argument (for example for azureBlobStorage table function). - static constexpr auto max_number_of_arguments_with_structure = 4; + static constexpr auto max_number_of_arguments_with_structure = 8; static constexpr auto signatures_with_structure = " - connection_string, container_name, blobpath\n" " - connection_string, container_name, blobpath, structure \n" @@ -37,7 +37,7 @@ public: " - storage_account_url, container_name, blobpath, account_name, account_key, format, compression, structure\n"; /// All possible signatures for Azure engine without structure argument (for example for AzureBlobStorage table engine). - static constexpr auto max_number_of_arguments_without_structure = 3; + static constexpr auto max_number_of_arguments_without_structure = 7; static constexpr auto signatures_without_structure = " - connection_string, container_name, blobpath\n" " - connection_string, container_name, blobpath, format \n" From d6ea08e8122c2577b5c486fae4e880217908ace9 Mon Sep 17 00:00:00 2001 From: marco-vb Date: Tue, 3 Sep 2024 12:47:20 +0000 Subject: [PATCH 133/215] Fixed documentation related to wildcard support. --- docs/en/operations/external-authenticators/ssl-x509.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/external-authenticators/ssl-x509.md b/docs/en/operations/external-authenticators/ssl-x509.md index 435dd8227fc..a7514966fa7 100644 --- a/docs/en/operations/external-authenticators/ssl-x509.md +++ b/docs/en/operations/external-authenticators/ssl-x509.md @@ -6,7 +6,7 @@ import SelfManaged from '@site/docs/en/_snippets/_self_managed_only_no_roadmap.m -[SSL 'strict' option](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) enables mandatory certificate validation for the incoming connections. In this case, only connections with trusted certificates can be established. Connections with untrusted certificates will be rejected. Thus, certificate validation allows to uniquely authenticate an incoming connection. `Common Name` or `subjectAltName extension` field of the certificate is used to identify the connected user. `subjectAltName extension` supports the usage of one wildcard '*' in the server configuration: see [test_x509_san_wildcard_support](../../../../tests/integration/test_ssl_cert_authentication/test.py). This allows to associate multiple certificates with the same user. Additionally, reissuing and revoking of the certificates does not affect the ClickHouse configuration. +[SSL 'strict' option](../server-configuration-parameters/settings.md#server_configuration_parameters-openssl) enables mandatory certificate validation for the incoming connections. In this case, only connections with trusted certificates can be established. Connections with untrusted certificates will be rejected. Thus, certificate validation allows to uniquely authenticate an incoming connection. `Common Name` or `subjectAltName extension` field of the certificate is used to identify the connected user. `subjectAltName extension` supports the usage of one wildcard '*' in the server configuration. This allows to associate multiple certificates with the same user. Additionally, reissuing and revoking of the certificates does not affect the ClickHouse configuration. To enable SSL certificate authentication, a list of `Common Name`'s or `Subject Alt Name`'s for each ClickHouse user must be specified in the settings file `users.xml `: From 6a6935cb84a31493def51cf5d65954bec75f587e Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Tue, 3 Sep 2024 13:09:18 +0200 Subject: [PATCH 134/215] Cleanup --- src/Storages/MergeTree/MergeTask.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index cafc11fc34d..6f5b8301d4a 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -1466,7 +1466,7 @@ public: subqueries_for_sets = transform->getSubqueries(); } - String getName() const override { return "Materializing"; } + String getName() const override { return "TTL"; } PreparedSets::Subqueries getSubqueries() { return std::move(subqueries_for_sets); } @@ -1524,9 +1524,8 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const std::vector plans; for (const auto & part : global_ctx->future_part->parts) { - /// TODO: this is just for debugging purposes, remove it later if (part->getMarksCount() == 0) - LOG_DEBUG(ctx->log, "Part {} is empty", part->name); + LOG_TRACE(ctx->log, "Part {} is empty", part->name); auto plan_for_part = std::make_unique(); createReadFromPartStep( @@ -1613,12 +1612,12 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const /// If deduplicate_by_columns is empty, add all columns except virtuals. if (global_ctx->deduplicate_by_columns.empty()) { - for (const auto & column_name : global_ctx->merging_columns.getNames()) + for (const auto & column : global_ctx->merging_columns) { - if (virtuals.tryGet(column_name, VirtualsKind::Persistent)) + if (virtuals.tryGet(column.name, VirtualsKind::Persistent)) continue; - global_ctx->deduplicate_by_columns.emplace_back(column_name); + global_ctx->deduplicate_by_columns.emplace_back(column.name); } } From a1cec53b7c2a6508277280bd8c36f90dfe661560 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Tue, 3 Sep 2024 14:54:05 +0200 Subject: [PATCH 135/215] Fix updateOutputStream and Traits --- src/Storages/MergeTree/MergeTask.cpp | 106 ++++++++++++++++++++------- 1 file changed, 78 insertions(+), 28 deletions(-) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 6f5b8301d4a..1bf1573fc1f 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -861,7 +861,7 @@ public: UInt64 merge_block_size_rows_, UInt64 merge_block_size_bytes_, bool is_result_sparse_) - : ITransformingStep(input_stream_, input_stream_.header, Traits{}) // TODO proper traits? + : ITransformingStep(input_stream_, input_stream_.header, getTraits()) , rows_sources_read_buf(rows_sources_read_buf_) , merge_block_size_rows(merge_block_size_rows_) , merge_block_size_bytes(merge_block_size_bytes_) @@ -891,12 +891,24 @@ public: void updateOutputStream() override { output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); - - /// TODO: is this correct? - output_stream->sort_scope = DataStream::SortScope::None; } private: + static Traits getTraits() + { + return ITransformingStep::Traits + { + { + .returns_single_stream = true, + .preserves_number_of_streams = true, + .preserves_sorting = true, + }, + { + .preserves_number_of_rows = false, + } + }; + } + MergeTreeData::MergingParams merging_params{}; CompressedReadBufferFromFile * rows_sources_read_buf; const UInt64 merge_block_size_rows; @@ -962,10 +974,9 @@ void MergeTask::VerticalMergeStage::prepareVerticalMergeForOneColumn() const } { - auto pipelineSettings = BuildQueryPipelineSettings::fromContext(global_ctx->context); - auto builder = merge_column_query_plan.buildQueryPipeline( - QueryPlanOptimizationSettings::fromContext(global_ctx->context), - pipelineSettings); + auto pipeline_settings = BuildQueryPipelineSettings::fromContext(global_ctx->context); + auto optimization_settings = QueryPlanOptimizationSettings::fromContext(global_ctx->context); + auto builder = merge_column_query_plan.buildQueryPipeline(optimization_settings, pipeline_settings); ctx->column_parts_pipeline = QueryPipelineBuilder::getPipeline(std::move(*builder)); } @@ -1297,7 +1308,7 @@ bool MergeTask::execute() /// Apply merge strategy (Ordinary, Colapsing, Aggregating, etc) to the stream -class ApplyMergeStep : public ITransformingStep /// TODO: is this transformation step? +class ApplyMergeStep : public ITransformingStep { public: ApplyMergeStep( @@ -1311,7 +1322,7 @@ public: bool blocks_are_granules_size_, bool cleanup_, time_t time_of_merge_) - : ITransformingStep(input_stream_, input_stream_.header, Traits{}) // TODO proper traits? + : ITransformingStep(input_stream_, input_stream_.header, getTraits()) , sort_description(sort_description_) , partition_key_columns(partition_key_columns_) , merging_params(merging_params_) @@ -1403,16 +1414,24 @@ public: void updateOutputStream() override { output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); - output_stream->sort_description = sort_description; - - /// TODO: is this correct? -// if (partition_key_columns.empty()) - output_stream->sort_scope = DataStream::SortScope::Global; -// else -// output_stream->sort_scope = DataStream::SortScope::Stream; } private: + static Traits getTraits() + { + return ITransformingStep::Traits + { + { + .returns_single_stream = true, + .preserves_number_of_streams = true, + .preserves_sorting = true, + }, + { + .preserves_number_of_rows = false, + } + }; + } + const SortDescription sort_description; const Names partition_key_columns; const MergeTreeData::MergingParams merging_params{}; @@ -1425,12 +1444,12 @@ private: }; -class MaterializingStep : public ITransformingStep /// TODO: is this transformation step? +class MaterializingStep : public ITransformingStep { public: explicit MaterializingStep( const DataStream & input_stream_) - : ITransformingStep(input_stream_, input_stream_.header, Traits{}) // TODO proper traits? + : ITransformingStep(input_stream_, input_stream_.header, getTraits()) {} String getName() const override { return "Materializing"; } @@ -1442,9 +1461,23 @@ public: void updateOutputStream() override { - /// TODO: can this be simplified? output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); - output_stream->sort_description = input_streams.front().sort_description; + } + +private: + static Traits getTraits() + { + return ITransformingStep::Traits + { + { + .returns_single_stream = true, + .preserves_number_of_streams = true, + .preserves_sorting = true, + }, + { + .preserves_number_of_rows = true, + } + }; } }; @@ -1460,7 +1493,7 @@ public: const MergeTreeData::MutableDataPartPtr & data_part_, time_t current_time, bool force_) - : ITransformingStep(input_stream_, input_stream_.header, Traits{}) // TODO proper traits? + : ITransformingStep(input_stream_, input_stream_.header, getTraits()) { transform = std::make_shared(context_, input_stream_.header, storage_, metadata_snapshot_, data_part_, current_time, force_); subqueries_for_sets = transform->getSubqueries(); @@ -1477,10 +1510,25 @@ public: void updateOutputStream() override { - // TODO: implement? + output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); } private: + static Traits getTraits() + { + return ITransformingStep::Traits + { + { + .returns_single_stream = true, + .preserves_number_of_streams = true, + .preserves_sorting = true, + }, + { + .preserves_number_of_rows = false, + } + }; + } + std::shared_ptr transform; PreparedSets::Subqueries subqueries_for_sets; }; @@ -1658,12 +1706,14 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const if (!subqueries.empty()) addCreatingSetsStep(merge_parts_query_plan, std::move(subqueries), global_ctx->context); - auto pipelineSettings = BuildQueryPipelineSettings::fromContext(global_ctx->context); - auto builder = merge_parts_query_plan.buildQueryPipeline( - QueryPlanOptimizationSettings::fromContext(global_ctx->context), - pipelineSettings); + { + auto pipeline_settings = BuildQueryPipelineSettings::fromContext(global_ctx->context); + auto optimization_settings = QueryPlanOptimizationSettings::fromContext(global_ctx->context); + auto builder = merge_parts_query_plan.buildQueryPipeline(optimization_settings, pipeline_settings); + + global_ctx->merged_pipeline = QueryPipelineBuilder::getPipeline(std::move(*builder)); + } - global_ctx->merged_pipeline = QueryPipelineBuilder::getPipeline(std::move(*builder)); /// Dereference unique_ptr and pass horizontal_stage_progress by reference global_ctx->merged_pipeline.setProgressCallback(MergeProgressCallback(global_ctx->merge_list_element_ptr, global_ctx->watch_prev_elapsed, *global_ctx->horizontal_stage_progress)); /// Is calculated inside MergeProgressCallback. From 8361724539408d95f9757e00047919d70ea50bbd Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Tue, 3 Sep 2024 17:02:25 +0200 Subject: [PATCH 136/215] Build pipeline for next column for prefetching --- src/Storages/MergeTree/MergeTask.cpp | 140 +++++++++++++-------------- src/Storages/MergeTree/MergeTask.h | 14 ++- 2 files changed, 81 insertions(+), 73 deletions(-) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 1bf1573fc1f..a4104672de7 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -804,53 +804,12 @@ bool MergeTask::VerticalMergeStage::prepareVerticalMergeForAllColumns() const bool all_parts_on_remote_disks = std::ranges::all_of(global_ctx->future_part->parts, [](const auto & part) { return part->isStoredOnRemoteDisk(); }); ctx->use_prefetch = all_parts_on_remote_disks && global_ctx->data->getSettings()->vertical_merge_remote_filesystem_prefetch; -// if (ctx->use_prefetch && ctx->it_name_and_type != global_ctx->gathering_columns.end()) -// ctx->prepared_pipe = createPipeForReadingOneColumn(ctx->it_name_and_type->name); + if (ctx->use_prefetch && ctx->it_name_and_type != global_ctx->gathering_columns.end()) + ctx->prepared_pipeline = createPipelineForReadingOneColumn(ctx->it_name_and_type->name); return false; } -QueryPlan MergeTask::VerticalMergeStage::createPlanForReadingOneColumn(const String & column_name) const -{ - /// Read from all parts - std::vector plans; - for (const auto & part : global_ctx->future_part->parts) - { - auto plan_for_part = std::make_unique(); - createReadFromPartStep( - MergeTreeSequentialSourceType::Merge, - *plan_for_part, - *global_ctx->data, - global_ctx->storage_snapshot, - part, - Names{column_name}, - global_ctx->input_rows_filtered, - /*apply_deleted_mask=*/ true, - std::nullopt, - ctx->read_with_direct_io, - ctx->use_prefetch, - global_ctx->context, - getLogger("VerticalMergeStage")); - - plans.emplace_back(std::move(plan_for_part)); - } - - QueryPlan merge_parts_query_plan; - - /// Union of all parts streams - { - DataStreams input_streams; - input_streams.reserve(plans.size()); - for (auto & plan : plans) - input_streams.emplace_back(plan->getCurrentDataStream()); - - auto union_step = std::make_unique(std::move(input_streams)); - merge_parts_query_plan.unitePlans(std::move(union_step), std::move(plans)); - } - - return merge_parts_query_plan; -} - /// Gathers values from all parts for one column using rows sources temporary file class ColumnGathererStep : public ITransformingStep { @@ -916,32 +875,46 @@ private: const bool is_result_sparse; }; -void MergeTask::VerticalMergeStage::prepareVerticalMergeForOneColumn() const +MergeTask::VerticalMergeRuntimeContext::PreparedColumnPipeline MergeTask::VerticalMergeStage::createPipelineForReadingOneColumn(const String & column_name) const { - const auto & column_name = ctx->it_name_and_type->name; + /// Read from all parts + std::vector plans; + for (const auto & part : global_ctx->future_part->parts) + { + auto plan_for_part = std::make_unique(); + createReadFromPartStep( + MergeTreeSequentialSourceType::Merge, + *plan_for_part, + *global_ctx->data, + global_ctx->storage_snapshot, + part, + Names{column_name}, + global_ctx->input_rows_filtered, + /*apply_deleted_mask=*/ true, + std::nullopt, + ctx->read_with_direct_io, + ctx->use_prefetch, + global_ctx->context, + getLogger("VerticalMergeStage")); - ctx->progress_before = global_ctx->merge_list_element_ptr->progress.load(std::memory_order_relaxed); - global_ctx->column_progress = std::make_unique(ctx->progress_before, ctx->column_sizes->columnWeight(column_name)); + plans.emplace_back(std::move(plan_for_part)); + } -// Pipe pipe; -//// if (ctx->prepared_pipe) -//// { -//// pipe = std::move(*ctx->prepared_pipe); -//// -//// auto next_column_it = std::next(ctx->it_name_and_type); -//// if (next_column_it != global_ctx->gathering_columns.end()) -//// ctx->prepared_pipe = createPipeForReadingOneColumn(next_column_it->name); -//// } -//// else -// { -// pipe = createPipeForReadingOneColumn(column_name); -// } + QueryPlan merge_column_query_plan; - auto merge_column_query_plan = createPlanForReadingOneColumn(column_name); + /// Union of all parts streams + { + DataStreams input_streams; + input_streams.reserve(plans.size()); + for (auto & plan : plans) + input_streams.emplace_back(plan->getCurrentDataStream()); + + auto union_step = std::make_unique(std::move(input_streams)); + merge_column_query_plan.unitePlans(std::move(union_step), std::move(plans)); + } /// Add column gatherer step { -// ctx->rows_sources_read_buf->seek(0, 0); bool is_result_sparse = global_ctx->new_data_part->getSerialization(column_name)->getKind() == ISerialization::Kind::SPARSE; const auto data_settings = global_ctx->data->getSettings(); auto merge_step = std::make_unique( @@ -973,13 +946,36 @@ void MergeTask::VerticalMergeStage::prepareVerticalMergeForOneColumn() const } } - { - auto pipeline_settings = BuildQueryPipelineSettings::fromContext(global_ctx->context); - auto optimization_settings = QueryPlanOptimizationSettings::fromContext(global_ctx->context); - auto builder = merge_column_query_plan.buildQueryPipeline(optimization_settings, pipeline_settings); + auto pipeline_settings = BuildQueryPipelineSettings::fromContext(global_ctx->context); + auto optimization_settings = QueryPlanOptimizationSettings::fromContext(global_ctx->context); + auto builder = merge_column_query_plan.buildQueryPipeline(optimization_settings, pipeline_settings); - ctx->column_parts_pipeline = QueryPipelineBuilder::getPipeline(std::move(*builder)); + return {QueryPipelineBuilder::getPipeline(std::move(*builder)), std::move(indexes_to_recalc)}; +} + +void MergeTask::VerticalMergeStage::prepareVerticalMergeForOneColumn() const +{ + const auto & column_name = ctx->it_name_and_type->name; + + ctx->progress_before = global_ctx->merge_list_element_ptr->progress.load(std::memory_order_relaxed); + global_ctx->column_progress = std::make_unique(ctx->progress_before, ctx->column_sizes->columnWeight(column_name)); + + VerticalMergeRuntimeContext::PreparedColumnPipeline column_pipepline; + if (ctx->prepared_pipeline) + { + column_pipepline = std::move(*ctx->prepared_pipeline); + + /// Prepare next column pipeline to initiate prefetching + auto next_column_it = std::next(ctx->it_name_and_type); + if (next_column_it != global_ctx->gathering_columns.end()) + ctx->prepared_pipeline = createPipelineForReadingOneColumn(next_column_it->name); } + else + { + column_pipepline = createPipelineForReadingOneColumn(column_name); + } + + ctx->column_parts_pipeline = std::move(column_pipepline.pipeline); /// Dereference unique_ptr ctx->column_parts_pipeline.setProgressCallback(MergeProgressCallback( @@ -997,12 +993,16 @@ void MergeTask::VerticalMergeStage::prepareVerticalMergeForOneColumn() const global_ctx->metadata_snapshot, columns_list, ctx->compression_codec, - indexes_to_recalc, + column_pipepline.indexes_to_recalc, getStatisticsForColumns(columns_list, global_ctx->metadata_snapshot), &global_ctx->written_offset_columns, global_ctx->to->getIndexGranularity()); ctx->column_elems_written = 0; + + /// rows_sources_read_buf is reused for each column so we need to rewind it explicitly each time + /// This sharing also prevents from from running multiple merge of individual columns in parallel. + ctx->rows_sources_read_buf->seek(0, 0); } @@ -1673,8 +1673,8 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const merge_parts_query_plan.getCurrentDataStream(), SizeLimits(), 0 /*limit_hint*/, global_ctx->deduplicate_by_columns, - false, - true /*TODO: ??*/); + false /*pre_distinct*/, + true /*optimize_distinct_in_order TODO: looks like it shoud be enabled*/); deduplication_step->setStepDescription("Deduplication step"); merge_parts_query_plan.addStep(std::move(deduplication_step)); } diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index a30ab4712d5..bbe53c34c7e 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -333,8 +333,16 @@ private: Float64 progress_before = 0; std::unique_ptr column_to{nullptr}; -// TODO: is this really needed for prefetch? -// std::optional prepared_pipe; + + /// Used for prefetching. Right before starting merge of a column we create a pipeline for the next column + /// and it initiates prefetching of the first range of that column. + struct PreparedColumnPipeline + { + QueryPipeline pipeline; + MergeTreeIndices indexes_to_recalc; + }; + + std::optional prepared_pipeline; size_t max_delayed_streams = 0; bool use_prefetch = false; std::list> delayed_streams; @@ -379,7 +387,7 @@ private: bool executeVerticalMergeForOneColumn() const; void finalizeVerticalMergeForOneColumn() const; - QueryPlan createPlanForReadingOneColumn(const String & column_name) const; + VerticalMergeRuntimeContext::PreparedColumnPipeline createPipelineForReadingOneColumn(const String & column_name) const; VerticalMergeRuntimeContextPtr ctx; GlobalRuntimeContextPtr global_ctx; From 472e6eb856e338332fbebb2519066f093c18a15f Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Tue, 3 Sep 2024 17:16:43 +0200 Subject: [PATCH 137/215] typo --- src/Storages/MergeTree/MergeTask.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index a4104672de7..576ea341877 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -1674,7 +1674,7 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const SizeLimits(), 0 /*limit_hint*/, global_ctx->deduplicate_by_columns, false /*pre_distinct*/, - true /*optimize_distinct_in_order TODO: looks like it shoud be enabled*/); + true /*optimize_distinct_in_order TODO: looks like it should be enabled*/); deduplication_step->setStepDescription("Deduplication step"); merge_parts_query_plan.addStep(std::move(deduplication_step)); } From fc1228e3a40509c0566c89c5348c94664e3ce219 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 3 Sep 2024 20:45:43 +0000 Subject: [PATCH 138/215] Update settings history --- src/Core/SettingsChangesHistory.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 8c26fe8da44..3d0ecc32f79 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -76,7 +76,8 @@ static std::initializer_list Date: Wed, 4 Sep 2024 16:05:13 +0800 Subject: [PATCH 139/215] review fix --- src/Core/Settings.h | 5 ++-- src/Core/SettingsChangesHistory.cpp | 5 ++-- src/Interpreters/HashJoin/AddedColumns.h | 8 +++--- src/Interpreters/HashJoin/HashJoin.cpp | 26 +++++++++++++------ src/Interpreters/TableJoin.cpp | 1 + src/Interpreters/TableJoin.h | 2 ++ tests/performance/all_join_opt.xml | 4 +-- .../experimental_settings_ignore.txt | 1 + 8 files changed, 34 insertions(+), 18 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index c1433ca7250..386fd4e74ee 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -922,8 +922,9 @@ class IColumn; M(Bool, implicit_transaction, false, "If enabled and not already inside a transaction, wraps the query inside a full transaction (begin + commit or rollback)", 0) \ M(UInt64, grace_hash_join_initial_buckets, 1, "Initial number of grace hash join buckets", 0) \ M(UInt64, grace_hash_join_max_buckets, 1024, "Limit on the number of grace hash join buckets", 0) \ - M(Int32, join_to_sort_perkey_rows_threshold, 40, "The lower limit of per-key average rows in the right table to determine whether to sort it in hash join.", 0) \ - M(Int32, join_to_sort_table_rows_threshold, 10000, "The upper limit of rows in the right table to determine whether to sort it in hash join.", 0) \ + M(Int32, join_to_sort_perkey_rows_threshold, 40, "Rerange the right table by key in hash join when the per-key average rows of it exceed this value (means the table keys is dense) and its number of rows is not too many(controlled by `join_to_sort_table_rows_threshold`), to make the join output by the key's batch, which would improve performance.", 0) \ + M(Int32, join_to_sort_table_rows_threshold, 10000, "Rerange the right table by key in hash join when its number of rows not exceed this value and the table keys is dense (controlled by `join_to_sort_perkey_rows_threshold`), to make the join performance improve as output by the key's batch, but not cost too much on the table reranging.", 0) \ + M(Bool, allow_experimental_inner_join_right_table_sorting, false, "If it is set to true, and the conditions of `join_to_sort_perkey_rows_threshold` and `join_to_sort_perkey_rows_threshold` are met, then we will try to rerange the right table by key to improve the performance in hash join.", 0) \ M(Timezone, session_timezone, "", "This setting can be removed in the future due to potential caveats. It is experimental and is not suitable for production usage. The default timezone for current session or query. The server default timezone if empty.", 0) \ M(Bool, use_hive_partitioning, false, "Allows to use hive partitioning for File, URL, S3, AzureBlobStorage and HDFS engines.", 0)\ \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 392f0dbc2ee..2a38af85b08 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -95,8 +95,9 @@ static std::initializer_listavgPerKeyRows()) + , output_by_row_list_threshold(join.getTableJoin().outputByRowListPerkeyRowsThreshold()) + , join_data_sorted(join.getJoinedData()->sorted) , is_join_get(is_join_get_) { size_t num_columns_to_add = block_with_columns_to_add.columns(); @@ -113,9 +116,6 @@ public: if (columns[j]->isNullable() && !saved_column->isNullable()) nullable_column_ptrs[j] = typeid_cast(columns[j].get()); } - join_data_avg_perkey_rows = join.getJoinedData()->avgPerKeyRows(); - output_by_row_list_threshold = join.getTableJoin().outputByRowListPerkeyRowsThreshold(); - join_data_sorted = join.getJoinedData()->sorted; } size_t size() const { return columns.size(); } @@ -148,9 +148,9 @@ public: std::unique_ptr offsets_to_replicate; bool need_filter = false; bool output_by_row_list = false; - bool join_data_sorted = false; size_t join_data_avg_perkey_rows = 0; size_t output_by_row_list_threshold = 0; + bool join_data_sorted = false; IColumn::Filter filter; void reserve(bool need_replicate) diff --git a/src/Interpreters/HashJoin/HashJoin.cpp b/src/Interpreters/HashJoin/HashJoin.cpp index 59888d7a71d..5b095368131 100644 --- a/src/Interpreters/HashJoin/HashJoin.cpp +++ b/src/Interpreters/HashJoin/HashJoin.cpp @@ -1364,14 +1364,16 @@ template void HashJoin::tryRerangeRightTableDataImpl(Map & map [[maybe_unused]]) { constexpr JoinFeatures join_features; - if constexpr (join_features.is_all_join && (join_features.left || join_features.inner)) + if constexpr (!join_features.is_all_join || (!join_features.left && !join_features.inner)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Only left or inner join table can be reranged."); + else { auto merge_rows_into_one_block = [&](BlocksList & blocks, RowRefList & rows_ref) { auto it = rows_ref.begin(); if (it.ok()) { - if (blocks.empty() || blocks.back().rows() > DEFAULT_BLOCK_SIZE) + if (blocks.empty() || blocks.back().rows() >= DEFAULT_BLOCK_SIZE) blocks.emplace_back(it->block->cloneEmpty()); } else @@ -1384,7 +1386,7 @@ void HashJoin::tryRerangeRightTableDataImpl(Map & map [[maybe_unused]]) { for (size_t i = 0; i < block.columns(); ++i) { - auto & col = *(block.getByPosition(i).column->assumeMutable()); + auto & col = block.getByPosition(i).column->assumeMutableRef(); col.insertFrom(*it->block->getByPosition(i).column, it->row_num); } } @@ -1419,26 +1421,34 @@ void HashJoin::tryRerangeRightTableDataImpl(Map & map [[maybe_unused]]) void HashJoin::tryRerangeRightTableData() { - if ((kind != JoinKind::Inner && kind != JoinKind::Left) || strictness != JoinStrictness::All || table_join->getMixedJoinExpression()) + if (!table_join->allowJoinSorting() || table_join->getMixedJoinExpression() || !isInnerOrLeft(kind) || strictness != JoinStrictness::All) return; + /// We should not rerange the right table on such conditions: + /// 1. the right table is already reranged by key or it is empty. + /// 2. the join clauses size is greater than 1, like `...join on a.key1=b.key1 or a.key2=b.key2`, we can not rerange the right table on different set of keys. + /// 3. the number of right table rows exceed the threshold, which may result in a significant cost for reranging and lead to performance degradation. + /// 4. the keys of right table is very sparse, which may result in insignificant performance improvement after reranging by key. if (!data || data->sorted || data->blocks.empty() || data->maps.size() > 1 || data->rows_to_join > table_join->sortRightTableRowsThreshold() || data->avgPerKeyRows() < table_join->sortRightPerkeyRowsThreshold()) return; if (data->keys_to_join == 0) data->keys_to_join = getTotalRowCount(); + + /// If the there is no columns to add, means no columns to output, then the rerange would not improve performance by using column's `insertRangeFrom` + /// to replace column's `insertFrom` to make the output. if (sample_block_with_columns_to_add.columns() == 0) { - LOG_DEBUG(log, "The joined right table total rows :{}, total keys :{}, columns added:{}", - data->rows_to_join, data->keys_to_join, sample_block_with_columns_to_add.columns()); + LOG_DEBUG(log, "The joined right table total rows :{}, total keys :{}", data->rows_to_join, data->keys_to_join); return; } - joinDispatch( + [[maybe_unused]] bool result = joinDispatch( kind, strictness, data->maps.front(), - false, + /*prefer_use_maps_all*/ false, [&](auto kind_, auto strictness_, auto & map_) { tryRerangeRightTableDataImpl(map_); }); + chassert(result); data->sorted = true; } diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp index 8bcaef77939..8d79b88190b 100644 --- a/src/Interpreters/TableJoin.cpp +++ b/src/Interpreters/TableJoin.cpp @@ -118,6 +118,7 @@ TableJoin::TableJoin(const Settings & settings, VolumePtr tmp_volume_, Temporary , output_by_rowlist_perkey_rows_threshold(settings.join_output_by_rowlist_perkey_rows_threshold) , sort_right_perkey_rows_threshold(settings.join_to_sort_perkey_rows_threshold) , sort_right_table_rows_threshold(settings.join_to_sort_table_rows_threshold) + , allow_join_sorting(settings.allow_experimental_inner_join_right_table_sorting) , max_memory_usage(settings.max_memory_usage) , tmp_volume(tmp_volume_) , tmp_data(tmp_data_) diff --git a/src/Interpreters/TableJoin.h b/src/Interpreters/TableJoin.h index 09d7f0f2b2a..c7926271a67 100644 --- a/src/Interpreters/TableJoin.h +++ b/src/Interpreters/TableJoin.h @@ -151,6 +151,7 @@ private: const size_t output_by_rowlist_perkey_rows_threshold = 0; const size_t sort_right_perkey_rows_threshold = 0; const size_t sort_right_table_rows_threshold = 0; + const bool allow_join_sorting = false; /// Value if setting max_memory_usage for query, can be used when max_bytes_in_join is not specified. size_t max_memory_usage = 0; @@ -301,6 +302,7 @@ public: size_t outputByRowListPerkeyRowsThreshold() const { return output_by_rowlist_perkey_rows_threshold; } size_t sortRightPerkeyRowsThreshold() const { return sort_right_perkey_rows_threshold; } size_t sortRightTableRowsThreshold() const { return sort_right_table_rows_threshold; } + bool allowJoinSorting() const { return allow_join_sorting; } size_t defaultMaxBytes() const { return default_max_bytes; } size_t maxJoinedBlockRows() const { return max_joined_block_rows; } size_t maxRowsInRightBlock() const { return partial_merge_join_rows_in_right_blocks; } diff --git a/tests/performance/all_join_opt.xml b/tests/performance/all_join_opt.xml index 0ab9c39f67c..ed8805a2e5f 100644 --- a/tests/performance/all_join_opt.xml +++ b/tests/performance/all_join_opt.xml @@ -5,8 +5,8 @@ INSERT INTO test SELECT number % 10000, number % 10000, number % 10000 FROM numbers(10000000) INSERT INTO test1 SELECT number % 1000 , number % 1000, number % 1000 FROM numbers(100000) - SELECT MAX(test1.a) FROM test INNER JOIN test1 on test.b = test1.b - SELECT MAX(test1.a) FROM test LEFT JOIN test1 on test.b = test1.b + SELECT MAX(test1.a) FROM test INNER JOIN test1 on test.b = test1.b SETTINGS allow_experimental_inner_join_right_table_sorting=true, join_to_sort_table_rows_threshold=100000 + SELECT MAX(test1.a) FROM test LEFT JOIN test1 on test.b = test1.b SETTINGS allow_experimental_inner_join_right_table_sorting=true, join_to_sort_table_rows_threshold=100000 SELECT MAX(test1.a) FROM test RIGHT JOIN test1 on test.b = test1.b SELECT MAX(test1.a) FROM test FULL JOIN test1 on test.b = test1.b diff --git a/utils/check-style/experimental_settings_ignore.txt b/utils/check-style/experimental_settings_ignore.txt index 94c46cf562e..358374494a3 100644 --- a/utils/check-style/experimental_settings_ignore.txt +++ b/utils/check-style/experimental_settings_ignore.txt @@ -31,6 +31,7 @@ allow_experimental_statistics allow_experimental_time_series_table allow_experimental_undrop_table_query allow_experimental_usearch_index +allow_experimental_inner_join_right_table_sorting allow_get_client_http_header allow_introspection_functions allow_materialized_view_with_bad_select From 49548ed4d5b4fb30aeff0b020c5791537fbd21c1 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Wed, 4 Sep 2024 16:40:46 +0800 Subject: [PATCH 140/215] update the description --- src/Core/Settings.h | 4 ++-- src/Core/SettingsChangesHistory.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 386fd4e74ee..9b83507ce73 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -922,8 +922,8 @@ class IColumn; M(Bool, implicit_transaction, false, "If enabled and not already inside a transaction, wraps the query inside a full transaction (begin + commit or rollback)", 0) \ M(UInt64, grace_hash_join_initial_buckets, 1, "Initial number of grace hash join buckets", 0) \ M(UInt64, grace_hash_join_max_buckets, 1024, "Limit on the number of grace hash join buckets", 0) \ - M(Int32, join_to_sort_perkey_rows_threshold, 40, "Rerange the right table by key in hash join when the per-key average rows of it exceed this value (means the table keys is dense) and its number of rows is not too many(controlled by `join_to_sort_table_rows_threshold`), to make the join output by the key's batch, which would improve performance.", 0) \ - M(Int32, join_to_sort_table_rows_threshold, 10000, "Rerange the right table by key in hash join when its number of rows not exceed this value and the table keys is dense (controlled by `join_to_sort_perkey_rows_threshold`), to make the join performance improve as output by the key's batch, but not cost too much on the table reranging.", 0) \ + M(Int32, join_to_sort_perkey_rows_threshold, 40, "Rerange the right table by key in hash join when the per-key average rows of it exceed this value (means the table keys is dense) and its number of rows is not too many(controlled by `join_to_sort_table_rows_threshold`), to make the join output by the data batch of key, which would improve performance.", 0) \ + M(Int32, join_to_sort_table_rows_threshold, 10000, "Rerange the right table by key in hash join when its number of rows not exceed this value and the table keys is dense (controlled by `join_to_sort_perkey_rows_threshold`), to make the join performance improve as output by the data batch of key, but not cost too much on the table reranging.", 0) \ M(Bool, allow_experimental_inner_join_right_table_sorting, false, "If it is set to true, and the conditions of `join_to_sort_perkey_rows_threshold` and `join_to_sort_perkey_rows_threshold` are met, then we will try to rerange the right table by key to improve the performance in hash join.", 0) \ M(Timezone, session_timezone, "", "This setting can be removed in the future due to potential caveats. It is experimental and is not suitable for production usage. The default timezone for current session or query. The server default timezone if empty.", 0) \ M(Bool, use_hive_partitioning, false, "Allows to use hive partitioning for File, URL, S3, AzureBlobStorage and HDFS engines.", 0)\ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 2a38af85b08..7bac9c314e7 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -95,8 +95,8 @@ static std::initializer_list Date: Wed, 4 Sep 2024 10:33:21 +0000 Subject: [PATCH 141/215] add remove recursive entry point --- src/Common/ZooKeeper/IKeeper.h | 6 +++- src/Common/ZooKeeper/TestKeeper.cpp | 1 + src/Common/ZooKeeper/TestKeeper.h | 1 + src/Common/ZooKeeper/ZooKeeper.cpp | 34 ++++++++++++++----- src/Common/ZooKeeper/ZooKeeper.h | 2 +- src/Common/ZooKeeper/ZooKeeperCommon.cpp | 32 +++++++++++++++-- src/Common/ZooKeeper/ZooKeeperCommon.h | 19 +++++++++-- src/Common/ZooKeeper/ZooKeeperConstants.cpp | 1 + src/Common/ZooKeeper/ZooKeeperConstants.h | 1 + src/Common/ZooKeeper/ZooKeeperImpl.cpp | 21 +++++++++--- src/Common/ZooKeeper/ZooKeeperImpl.h | 1 + .../ZooKeeper/ZooKeeperWithFaultInjection.cpp | 4 +-- src/Coordination/KeeperConstants.h | 1 + src/Coordination/KeeperDispatcher.cpp | 1 + src/Coordination/KeeperFeatureFlags.h | 1 + src/Coordination/KeeperStorage.cpp | 2 ++ src/Interpreters/ZooKeeperLog.cpp | 1 + 17 files changed, 109 insertions(+), 20 deletions(-) diff --git a/src/Common/ZooKeeper/IKeeper.h b/src/Common/ZooKeeper/IKeeper.h index ce7489a33e5..f887e67bd92 100644 --- a/src/Common/ZooKeeper/IKeeper.h +++ b/src/Common/ZooKeeper/IKeeper.h @@ -238,10 +238,13 @@ struct RemoveRequest : virtual Request String path; int32_t version = -1; + /// strict limit for number of deleted nodes + uint32_t remove_nodes_limit = 1; + void addRootPath(const String & root_path) override; String getPath() const override { return path; } - size_t bytesSize() const override { return path.size() + sizeof(version); } + size_t bytesSize() const override { return path.size() + sizeof(version) + sizeof(remove_nodes_limit); } }; struct RemoveResponse : virtual Response @@ -585,6 +588,7 @@ public: virtual void remove( const String & path, int32_t version, + uint32_t remove_nodes_limit, RemoveCallback callback) = 0; virtual void exists( diff --git a/src/Common/ZooKeeper/TestKeeper.cpp b/src/Common/ZooKeeper/TestKeeper.cpp index 16ea412eb77..65a575549c4 100644 --- a/src/Common/ZooKeeper/TestKeeper.cpp +++ b/src/Common/ZooKeeper/TestKeeper.cpp @@ -759,6 +759,7 @@ void TestKeeper::create( void TestKeeper::remove( const String & path, int32_t version, + [[maybe_unused]] uint32_t remove_nodes_limit, // TODO(michicosun): enable RemoveCallback callback) { TestKeeperRemoveRequest request; diff --git a/src/Common/ZooKeeper/TestKeeper.h b/src/Common/ZooKeeper/TestKeeper.h index 562c313ac0e..14b37448a20 100644 --- a/src/Common/ZooKeeper/TestKeeper.h +++ b/src/Common/ZooKeeper/TestKeeper.h @@ -56,6 +56,7 @@ public: void remove( const String & path, int32_t version, + uint32_t remove_nodes_limit, RemoveCallback callback) override; void exists( diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 1a9ed4f1ee7..eb4f5e0725a 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -979,16 +979,34 @@ bool ZooKeeper::tryRemoveChildrenRecursive(const std::string & path, bool probab return removed_as_expected; } -void ZooKeeper::removeRecursive(const std::string & path) +void ZooKeeper::removeRecursive(const std::string & path) // TODO(michicosun) rewrite { - removeChildrenRecursive(path); - remove(path); + auto promise = std::make_shared>(); + auto future = promise->get_future(); + + auto callback = [promise](const Coordination::RemoveResponse & response) mutable + { + promise->set_value(response); + }; + + impl->remove(path, -1, /*remove_nodes_limit=*/ 100, std::move(callback)); + + if (future.wait_for(std::chrono::milliseconds(args.operation_timeout_ms)) != std::future_status::ready) + { + impl->finalize(fmt::format("Operation timeout on {} {}", Coordination::OpNum::RemoveRecursive, path)); + check(Coordination::Error::ZOPERATIONTIMEOUT, path); + } + else + { + auto response = future.get(); + check(response.error, path); + } } -void ZooKeeper::tryRemoveRecursive(const std::string & path) +Coordination::Error ZooKeeper::tryRemoveRecursive(const std::string & path) { tryRemoveChildrenRecursive(path); - tryRemove(path); + return tryRemove(path); } @@ -1367,7 +1385,7 @@ std::future ZooKeeper::asyncRemove(const std::stri promise->set_value(response); }; - impl->remove(path, version, std::move(callback)); + impl->remove(path, version, /*remove_nodes_limit=*/ 1, std::move(callback)); return future; } @@ -1390,7 +1408,7 @@ std::future ZooKeeper::asyncTryRemove(const std::s promise->set_value(response); }; - impl->remove(path, version, std::move(callback)); + impl->remove(path, version, /*remove_nodes_limit=*/ 1, std::move(callback)); return future; } @@ -1404,7 +1422,7 @@ std::future ZooKeeper::asyncTryRemoveNoThrow(const promise->set_value(response); }; - impl->remove(path, version, std::move(callback)); + impl->remove(path, version, /*remove_nodes_limit=*/ 1, std::move(callback)); return future; } diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 7ccdc9d1b7f..3b94e6004cb 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -487,7 +487,7 @@ public: /// this will not cause errors. /// For instance, you can call this method twice concurrently for the same node and the end /// result would be the same as for the single call. - void tryRemoveRecursive(const std::string & path); + Coordination::Error tryRemoveRecursive(const std::string & path); /// Similar to removeRecursive(...) and tryRemoveRecursive(...), but does not remove path itself. /// Node defined as RemoveException will not be deleted. diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.cpp b/src/Common/ZooKeeper/ZooKeeperCommon.cpp index dff14f74681..fb37aa6ac22 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.cpp +++ b/src/Common/ZooKeeper/ZooKeeperCommon.cpp @@ -1,5 +1,5 @@ -#include "Common/ZooKeeper/IKeeper.h" -#include "Common/ZooKeeper/ZooKeeperConstants.h" +#include +#include #include #include #include @@ -232,6 +232,27 @@ void ZooKeeperRemoveRequest::readImpl(ReadBuffer & in) Coordination::read(version, in); } +void ZooKeeperRemoveRecursiveRequest::writeImpl(WriteBuffer & out) const +{ + ZooKeeperRemoveRequest::writeImpl(out); + Coordination::write(remove_nodes_limit, out); +} + +void ZooKeeperRemoveRecursiveRequest::readImpl(ReadBuffer & in) +{ + ZooKeeperRemoveRequest::readImpl(in); + Coordination::read(remove_nodes_limit, in); +} + +std::string ZooKeeperRemoveRecursiveRequest::toStringImpl(bool short_format) const +{ + return fmt::format( + "{}\n" + "remove_nodes_limit = {}", + ZooKeeperRemoveRequest::toStringImpl(short_format), + remove_nodes_limit); +} + void ZooKeeperExistsRequest::writeImpl(WriteBuffer & out) const { Coordination::write(path, out); @@ -510,6 +531,11 @@ ZooKeeperMultiRequest::ZooKeeperMultiRequest(std::span(*concrete_request_remove)); } + else if (const auto * concrete_request_remove_recursive = dynamic_cast(generic_request.get())) + { + checkOperationType(Write); + requests.push_back(std::make_shared(*concrete_request_remove_recursive)); + } else if (const auto * concrete_request_set = dynamic_cast(generic_request.get())) { checkOperationType(Write); @@ -707,6 +733,7 @@ ZooKeeperResponsePtr ZooKeeperHeartbeatRequest::makeResponse() const { return se ZooKeeperResponsePtr ZooKeeperSyncRequest::makeResponse() const { return setTime(std::make_shared()); } ZooKeeperResponsePtr ZooKeeperAuthRequest::makeResponse() const { return setTime(std::make_shared()); } ZooKeeperResponsePtr ZooKeeperRemoveRequest::makeResponse() const { return setTime(std::make_shared()); } +ZooKeeperResponsePtr ZooKeeperRemoveRecursiveRequest::makeResponse() const { return setTime(std::make_shared()); } ZooKeeperResponsePtr ZooKeeperExistsRequest::makeResponse() const { return setTime(std::make_shared()); } ZooKeeperResponsePtr ZooKeeperGetRequest::makeResponse() const { return setTime(std::make_shared()); } ZooKeeperResponsePtr ZooKeeperSetRequest::makeResponse() const { return setTime(std::make_shared()); } @@ -1024,6 +1051,7 @@ ZooKeeperRequestFactory::ZooKeeperRequestFactory() registerZooKeeperRequest(*this); registerZooKeeperRequest(*this); registerZooKeeperRequest(*this); + registerZooKeeperRequest(*this); } PathMatchResult matchPath(std::string_view path, std::string_view match_to) diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.h b/src/Common/ZooKeeper/ZooKeeperCommon.h index fd6ec3cd375..ef17d069b4c 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.h +++ b/src/Common/ZooKeeper/ZooKeeperCommon.h @@ -258,7 +258,7 @@ struct ZooKeeperCreateIfNotExistsResponse : ZooKeeperCreateResponse using ZooKeeperCreateResponse::ZooKeeperCreateResponse; }; -struct ZooKeeperRemoveRequest final : RemoveRequest, ZooKeeperRequest +struct ZooKeeperRemoveRequest : RemoveRequest, ZooKeeperRequest { ZooKeeperRemoveRequest() = default; explicit ZooKeeperRemoveRequest(const RemoveRequest & base) : RemoveRequest(base) {} @@ -276,7 +276,17 @@ struct ZooKeeperRemoveRequest final : RemoveRequest, ZooKeeperRequest void createLogElements(LogElements & elems) const override; }; -struct ZooKeeperRemoveResponse final : RemoveResponse, ZooKeeperResponse +struct ZooKeeperRemoveRecursiveRequest : ZooKeeperRemoveRequest +{ + OpNum getOpNum() const override { return OpNum::RemoveRecursive; } + void writeImpl(WriteBuffer & out) const override; + void readImpl(ReadBuffer & in) override; + std::string toStringImpl(bool short_format) const override; + + ZooKeeperResponsePtr makeResponse() const override; +}; + +struct ZooKeeperRemoveResponse : RemoveResponse, ZooKeeperResponse { void readImpl(ReadBuffer &) override {} void writeImpl(WriteBuffer &) const override {} @@ -285,6 +295,11 @@ struct ZooKeeperRemoveResponse final : RemoveResponse, ZooKeeperResponse size_t bytesSize() const override { return RemoveResponse::bytesSize() + sizeof(xid) + sizeof(zxid); } }; +struct ZooKeeperRemoveRecursiveResponse : ZooKeeperRemoveResponse +{ + OpNum getOpNum() const override { return OpNum::RemoveRecursive; } +}; + struct ZooKeeperExistsRequest final : ExistsRequest, ZooKeeperRequest { ZooKeeperExistsRequest() = default; diff --git a/src/Common/ZooKeeper/ZooKeeperConstants.cpp b/src/Common/ZooKeeper/ZooKeeperConstants.cpp index cf8ba35e992..a2780dfd5e2 100644 --- a/src/Common/ZooKeeper/ZooKeeperConstants.cpp +++ b/src/Common/ZooKeeper/ZooKeeperConstants.cpp @@ -29,6 +29,7 @@ static const std::unordered_set VALID_OPERATIONS = static_cast(OpNum::GetACL), static_cast(OpNum::FilteredList), static_cast(OpNum::CheckNotExists), + static_cast(OpNum::RemoveRecursive), }; OpNum getOpNum(int32_t raw_op_num) diff --git a/src/Common/ZooKeeper/ZooKeeperConstants.h b/src/Common/ZooKeeper/ZooKeeperConstants.h index 1d9830505f8..9d8e2d4f857 100644 --- a/src/Common/ZooKeeper/ZooKeeperConstants.h +++ b/src/Common/ZooKeeper/ZooKeeperConstants.h @@ -40,6 +40,7 @@ enum class OpNum : int32_t FilteredList = 500, CheckNotExists = 501, CreateIfNotExists = 502, + RemoveRecursive = 503, SessionID = 997, /// Special internal request }; diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index ba622f30c91..5c4de98d5fc 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1333,14 +1333,27 @@ void ZooKeeper::create( void ZooKeeper::remove( const String & path, int32_t version, + uint32_t remove_nodes_limit, RemoveCallback callback) { - ZooKeeperRemoveRequest request; - request.path = path; - request.version = version; + std::shared_ptr request = nullptr; + + if (!isFeatureEnabled(KeeperFeatureFlag::REMOVE_RECURSIVE)) + { + if (remove_nodes_limit > 1) + throw Exception::fromMessage(Error::ZBADARGUMENTS, "RemoveRecursive request type cannot be used because it's not supported by the server"); + + request = std::make_shared(); + } + else + request = std::make_shared(); + + request->path = path; + request->version = version; + request->remove_nodes_limit = remove_nodes_limit; RequestInfo request_info; - request_info.request = std::make_shared(std::move(request)); + request_info.request = std::move(request); request_info.callback = [callback](const Response & response) { callback(dynamic_cast(response)); }; pushRequest(std::move(request_info)); diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.h b/src/Common/ZooKeeper/ZooKeeperImpl.h index 39082cd14c1..6e98df546d9 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.h +++ b/src/Common/ZooKeeper/ZooKeeperImpl.h @@ -144,6 +144,7 @@ public: void remove( const String & path, int32_t version, + uint32_t remove_nodes_limit, RemoveCallback callback) override; void exists( diff --git a/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.cpp b/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.cpp index fc246e263d9..dfd3faf8d01 100644 --- a/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.cpp +++ b/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.cpp @@ -587,7 +587,7 @@ zkutil::ZooKeeper::FutureRemove ZooKeeperWithFaultInjection::asyncTryRemove(std: promise->set_value(response); }; - keeper->impl->remove(path, version, std::move(callback)); + keeper->impl->remove(path, version, /*remove_nodes_limit=*/ 1, std::move(callback)); return future; } @@ -630,7 +630,7 @@ zkutil::ZooKeeper::FutureRemove ZooKeeperWithFaultInjection::asyncTryRemoveNoThr } }; - keeper->impl->remove(path, version, std::move(callback)); + keeper->impl->remove(path, version, /*remove_nodes_limit=*/ 1, std::move(callback)); return future; } diff --git a/src/Coordination/KeeperConstants.h b/src/Coordination/KeeperConstants.h index 08a7c85585a..d984d077872 100644 --- a/src/Coordination/KeeperConstants.h +++ b/src/Coordination/KeeperConstants.h @@ -11,6 +11,7 @@ enum class KeeperApiVersion : uint8_t WITH_FILTERED_LIST, WITH_MULTI_READ, WITH_CHECK_NOT_EXISTS, + WITH_REMOVE_RECURSIVE, }; const String keeper_system_path = "/keeper"; diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index 4a350077596..fd80ee0a7e2 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -86,6 +86,7 @@ bool checkIfRequestIncreaseMem(const Coordination::ZooKeeperRequestPtr & request break; } case Coordination::OpNum::Remove: + case Coordination::OpNum::RemoveRecursive: { Coordination::ZooKeeperRemoveRequest & remove_req = dynamic_cast(*sub_zk_request); memory_delta -= remove_req.bytesSize(); diff --git a/src/Coordination/KeeperFeatureFlags.h b/src/Coordination/KeeperFeatureFlags.h index 4e26ca60736..e70bd50cc88 100644 --- a/src/Coordination/KeeperFeatureFlags.h +++ b/src/Coordination/KeeperFeatureFlags.h @@ -12,6 +12,7 @@ enum class KeeperFeatureFlag : size_t MULTI_READ, CHECK_NOT_EXISTS, CREATE_IF_NOT_EXISTS, + REMOVE_RECURSIVE, }; class KeeperFeatureFlags diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index acdf209baae..54cdc8c11db 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -2128,6 +2128,7 @@ struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestPro concrete_requests.push_back(std::make_shared>(sub_zk_request)); break; case Coordination::OpNum::Remove: + case Coordination::OpNum::RemoveRecursive: check_operation_type(OperationType::Write); concrete_requests.push_back(std::make_shared>(sub_zk_request)); break; @@ -2400,6 +2401,7 @@ KeeperStorageRequestProcessorsFactory::KeeperStorageRequestProcessorsFa registerKeeperRequestProcessor>(*this); registerKeeperRequestProcessor>(*this); registerKeeperRequestProcessor>(*this); + registerKeeperRequestProcessor>(*this); } diff --git a/src/Interpreters/ZooKeeperLog.cpp b/src/Interpreters/ZooKeeperLog.cpp index 0d3063a569e..769757a5fba 100644 --- a/src/Interpreters/ZooKeeperLog.cpp +++ b/src/Interpreters/ZooKeeperLog.cpp @@ -93,6 +93,7 @@ ColumnsDescription ZooKeeperLogElement::getColumnsDescription() {"FilteredList", static_cast(Coordination::OpNum::FilteredList)}, {"CheckNotExists", static_cast(Coordination::OpNum::CheckNotExists)}, {"CreateIfNotExists", static_cast(Coordination::OpNum::CreateIfNotExists)}, + {"RemoveRecursive", static_cast(Coordination::OpNum::RemoveRecursive)}, }); auto error_enum = getCoordinationErrorCodesEnumType(); From f926a0fff7c870e91f19e5147d89304cbff3ca29 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 4 Sep 2024 11:25:22 +0000 Subject: [PATCH 142/215] Fix tidy build --- src/Storages/ObjectStorage/Azure/Configuration.h | 4 ++-- src/Storages/ObjectStorage/HDFS/Configuration.h | 4 ++-- src/Storages/ObjectStorage/Local/Configuration.h | 4 ++-- src/Storages/ObjectStorage/S3/Configuration.h | 4 ++-- src/Storages/ObjectStorage/StorageObjectStorage.h | 4 ---- 5 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/Storages/ObjectStorage/Azure/Configuration.h b/src/Storages/ObjectStorage/Azure/Configuration.h index 311ad76814d..c3adc86b124 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.h +++ b/src/Storages/ObjectStorage/Azure/Configuration.h @@ -52,8 +52,8 @@ public: std::string getTypeName() const override { return type_name; } std::string getEngineName() const override { return engine_name; } - std::string getSignatures(bool with_structure = true) const override { return with_structure ? signatures_with_structure : signatures_without_structure; } - size_t getMaxNumberOfArguments(bool with_structure = true) const override { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; } + std::string getSignatures(bool with_structure = true) const { return with_structure ? signatures_with_structure : signatures_without_structure; } + size_t getMaxNumberOfArguments(bool with_structure = true) const { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; } Path getPath() const override { return blob_path; } void setPath(const Path & path) override { blob_path = path; } diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.h b/src/Storages/ObjectStorage/HDFS/Configuration.h index 86373512fa6..206147d7e5e 100644 --- a/src/Storages/ObjectStorage/HDFS/Configuration.h +++ b/src/Storages/ObjectStorage/HDFS/Configuration.h @@ -37,8 +37,8 @@ public: std::string getTypeName() const override { return type_name; } std::string getEngineName() const override { return engine_name; } - std::string getSignatures(bool with_structure = true) const override { return with_structure ? signatures_with_structure : signatures_without_structure; } - size_t getMaxNumberOfArguments(bool with_structure = true) const override { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; } + std::string getSignatures(bool with_structure = true) const { return with_structure ? signatures_with_structure : signatures_without_structure; } + size_t getMaxNumberOfArguments(bool with_structure = true) const { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; } Path getPath() const override { return path; } void setPath(const Path & path_) override { path = path_; } diff --git a/src/Storages/ObjectStorage/Local/Configuration.h b/src/Storages/ObjectStorage/Local/Configuration.h index 71eec13ca31..84dc3855df3 100644 --- a/src/Storages/ObjectStorage/Local/Configuration.h +++ b/src/Storages/ObjectStorage/Local/Configuration.h @@ -40,8 +40,8 @@ public: std::string getTypeName() const override { return type_name; } std::string getEngineName() const override { return "Local"; } - std::string getSignatures(bool with_structure = true) const override { return with_structure ? signatures_with_structure : signatures_without_structure; } - size_t getMaxNumberOfArguments(bool with_structure = true) const override { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; } + std::string getSignatures(bool with_structure = true) const { return with_structure ? signatures_with_structure : signatures_without_structure; } + size_t getMaxNumberOfArguments(bool with_structure = true) const { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; } Path getPath() const override { return path; } void setPath(const Path & path_) override { path = path_; } diff --git a/src/Storages/ObjectStorage/S3/Configuration.h b/src/Storages/ObjectStorage/S3/Configuration.h index 63c1d24de62..b36df67fb0f 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.h +++ b/src/Storages/ObjectStorage/S3/Configuration.h @@ -61,8 +61,8 @@ public: std::string getEngineName() const override { return url.storage_name; } std::string getNamespaceType() const override { return namespace_name; } - std::string getSignatures(bool with_structure = true) const override { return with_structure ? signatures_with_structure : signatures_without_structure; } - size_t getMaxNumberOfArguments(bool with_structure = true) const override { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; } + std::string getSignatures(bool with_structure = true) const { return with_structure ? signatures_with_structure : signatures_without_structure; } + size_t getMaxNumberOfArguments(bool with_structure = true) const { return with_structure ? max_number_of_arguments_with_structure : max_number_of_arguments_without_structure; } Path getPath() const override { return url.key; } void setPath(const Path & path) override { url.key = path; } diff --git a/src/Storages/ObjectStorage/StorageObjectStorage.h b/src/Storages/ObjectStorage/StorageObjectStorage.h index ba7fd350bae..f39586c23b4 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorage.h +++ b/src/Storages/ObjectStorage/StorageObjectStorage.h @@ -170,10 +170,6 @@ public: /// buckets in S3. If object storage doesn't have any namepaces return empty string. virtual std::string getNamespaceType() const { return "namespace"; } - /// Return the string containing all supported signatures for this storage arguments. - virtual std::string getSignatures(bool with_structure = true) const = 0; - virtual size_t getMaxNumberOfArguments(bool with_structure = true) const = 0; - virtual Path getPath() const = 0; virtual void setPath(const Path & path) = 0; From dff153b59e72b103b6ba1e3c8dbe0310512a3071 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 4 Sep 2024 11:48:22 +0000 Subject: [PATCH 143/215] Null supports no settings --- src/Interpreters/InterpreterCreateQuery.cpp | 3 +++ .../test_restore_external_engines/test.py | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index e9f40bdbaf5..3014deab286 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1074,6 +1074,7 @@ namespace engine_ast->name = "Null"; engine_ast->no_empty_args = true; storage.set(storage.engine, engine_ast); + storage.settings = nullptr; } } @@ -1156,7 +1157,9 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const else if (getContext()->getSettingsRef().restore_replace_external_engines_to_null) { if (StorageFactory::instance().getStorageFeatures(create.storage->engine->name).source_access_type != AccessType::NONE) + { setNullTableEngine(*create.storage); + } } return; } diff --git a/tests/integration/test_restore_external_engines/test.py b/tests/integration/test_restore_external_engines/test.py index cf189f2a6ed..a975db05020 100644 --- a/tests/integration/test_restore_external_engines/test.py +++ b/tests/integration/test_restore_external_engines/test.py @@ -70,6 +70,12 @@ def get_mysql_conn(cluster): def fill_tables(cluster, dbname): fill_nodes(nodes, dbname) + node1.query( + f"""CREATE TABLE {dbname}.example_s3_engine_table (name String, value UInt32) +ENGINE = S3('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/test-data.csv.gz', 'CSV', 'gzip') +SETTINGS input_format_with_names_use_header = 0""" + ) + conn = get_mysql_conn(cluster) with conn.cursor() as cursor: @@ -136,6 +142,7 @@ def test_restore_table(start_cluster): node2.query(f"BACKUP DATABASE replicated TO {backup_name}") + node2.query("DROP TABLE replicated.example_s3_engine_table") node2.query("DROP TABLE replicated.mysql_schema_inference_engine") node2.query("DROP TABLE replicated.mysql_schema_inference_function") @@ -149,6 +156,13 @@ def test_restore_table(start_cluster): ) node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated") + assert ( + node1.query( + "SELECT engine FROM system.tables where database = 'replicated' and name = 'example_s3_engine_table'" + ) + == "S3\n" + ) + assert ( node1.query( "SELECT count(), sum(id) FROM replicated.mysql_schema_inference_engine" @@ -175,6 +189,7 @@ def test_restore_table_null(start_cluster): node2.query(f"BACKUP DATABASE replicated2 TO {backup_name}") + node2.query("DROP TABLE replicated2.example_s3_engine_table") node2.query("DROP TABLE replicated2.mysql_schema_inference_engine") node2.query("DROP TABLE replicated2.mysql_schema_inference_function") @@ -188,6 +203,13 @@ def test_restore_table_null(start_cluster): ) node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated2") + assert ( + node1.query( + "SELECT engine FROM system.tables where database = 'replicated2' and name = 'example_s3_engine_table'" + ) + == "Null\n" + ) + assert ( node1.query( "SELECT count(), sum(id) FROM replicated2.mysql_schema_inference_engine" From ef2dd93418bfb44f2fb687dc0ca38763e3cf1bbc Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 4 Sep 2024 19:43:39 +0000 Subject: [PATCH 144/215] crazy stuff --- src/Interpreters/InterpreterCreateQuery.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 3014deab286..3db3a75aaf7 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1070,11 +1070,15 @@ namespace void setNullTableEngine(ASTStorage & storage) { + storage.forEachPointerToChild([](void ** ptr) mutable + { + *ptr = nullptr; + }); + auto engine_ast = std::make_shared(); engine_ast->name = "Null"; engine_ast->no_empty_args = true; storage.set(storage.engine, engine_ast); - storage.settings = nullptr; } } From c6777af485fcd8aef5b759de9c1c531c74df6b8a Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Wed, 4 Sep 2024 22:59:18 +0000 Subject: [PATCH 145/215] add remove recursive support --- src/Coordination/KeeperStorage.cpp | 162 ++++++++++++++++++++++++++--- src/Coordination/KeeperStorage.h | 13 ++- 2 files changed, 161 insertions(+), 14 deletions(-) diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 54cdc8c11db..9d63a9e8691 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -1513,29 +1513,37 @@ struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestPr { if (request.restored_from_zookeeper_log) update_parent_pzxid(); + return {typename Storage::Delta{zxid, Coordination::Error::ZNONODE}}; } - else if (request.version != -1 && request.version != node->version) + + if (request.version != -1 && request.version != node->version) return {typename Storage::Delta{zxid, Coordination::Error::ZBADVERSION}}; - else if (node->numChildren() != 0) + + ToDeleteTreeCollector collector(storage, zxid, request.remove_nodes_limit); + bool limit_exceeded = collector.collect(request.path, *node); + + if (limit_exceeded) return {typename Storage::Delta{zxid, Coordination::Error::ZNOTEMPTY}}; if (request.restored_from_zookeeper_log) update_parent_pzxid(); - new_deltas.emplace_back( - std::string{parentNodePath(request.path)}, - zxid, - typename Storage::UpdateNodeDelta{[](typename Storage::Node & parent) - { - ++parent.cversion; - parent.decreaseNumChildren(); - }}); + auto delete_deltas = collector.extractDeltas(); - new_deltas.emplace_back(request.path, zxid, typename Storage::RemoveNodeDelta{request.version, node->ephemeralOwner()}); + for (const auto & delta : delete_deltas) + std::visit( + Overloaded{ + [&](const typename Storage::RemoveNodeDelta & remove_delta) + { + if (remove_delta.ephemeral_owner) + storage.unregisterEphemeralPath(remove_delta.ephemeral_owner, delta.path); + }, + [](auto && /* delta */) {}, + }, + delta.operation); - if (node->isEphemeral()) - storage.unregisterEphemeralPath(node->ephemeralOwner(), request.path); + new_deltas.insert(new_deltas.end(), std::make_move_iterator(delete_deltas.begin()), std::make_move_iterator(delete_deltas.end())); digest = storage.calculateNodesDigest(digest, new_deltas); @@ -1556,6 +1564,134 @@ struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestPr { return processWatchesImpl(this->zk_request->getPath(), watches, list_watches, Coordination::Event::DELETED); } + +private: + class ToDeleteTreeCollector + { + Storage & storage; + int64_t zxid; + uint32_t limit; + + uint32_t nodes_observed = 0; + std::vector deltas; + + public: + ToDeleteTreeCollector(Storage & storage_, int64_t zxid_, uint32_t limit_) + : storage(storage_) + , zxid(zxid_) + , limit(limit_) + { + } + + bool collect(StringRef root_path, const typename Storage::Node & root_node) + { + if (updateStats(root_node)) + return true; + + if (visitRocksDBNode(root_path) || visitMemNode(root_path)) + return true; + + /// After checking committed nodes there might be some in uncommitted state, + /// for example made on the previouse step of multi transaction. + return visitRootAndUncommitted(root_path, root_node); + } + + uint32_t getNumObserved() const + { + return nodes_observed; + } + + std::vector extractDeltas() + { + return std::move(deltas); + } + + private: + bool visitRocksDBNode(StringRef root_path) + { + if constexpr (Storage::use_rocksdb) + { + auto children = storage.container.getChildren(root_path.toString()); + + for (auto && [child_path, node] : children) + if (collect(child_path, node)) + return true; + } + + return false; + } + + bool visitMemNode(StringRef root_path) + { + if constexpr (!Storage::use_rocksdb) + { + std::filesystem::path root_fs_path(root_path.toString()); + + auto node_it = storage.container.find(root_path); + if (node_it != storage.container.end()) + { + auto children = node_it->value.getChildren(); + for (auto && child_name : children) + { + auto child_path = (root_fs_path / child_name.toView()).generic_string(); + + auto child_it = storage.container.find(child_path); + chassert(child_it != storage.container.end()); + + if (collect(child_path, child_it->value)) + return true; + } + } + } + + return false; + } + + bool visitRootAndUncommitted(StringRef root_path, const typename Storage::Node & root_node) + { + const auto & nodes = storage.uncommitted_state.nodes; + + /// nodes are sorted by paths with level locality + auto it = nodes.upper_bound(root_path.toView()); + + for (; it != nodes.end() && parentNodePath(it->first) == root_path; ++it) + { + chassert(it->second.node); + const typename Storage::Node & node = *it->second.node; + + if (updateStats(node)) + return true; + + if (visitRootAndUncommitted(it->first, node)) /// if child is uncommitted then all subtree is also uncommitted + return true; + } + + deltas.emplace_back( + parentNodePath(root_path).toString(), + zxid, + typename Storage::UpdateNodeDelta{ + [](typename Storage::Node & parent) + { + ++parent.cversion; + parent.decreaseNumChildren(); + } + }); + + deltas.emplace_back(root_path.toString(), zxid, typename Storage::RemoveNodeDelta{root_node.version, root_node.ephemeralOwner()}); + + return false; + } + + bool updateStats(const typename Storage::Node & root_node) + { + nodes_observed += 1 + root_node.numChildren(); /// root + all known children + + if (nodes_observed > limit) + return true; + + return false; + } + }; }; template diff --git a/src/Coordination/KeeperStorage.h b/src/Coordination/KeeperStorage.h index 4a9286d4835..c2f6e4c5a74 100644 --- a/src/Coordination/KeeperStorage.h +++ b/src/Coordination/KeeperStorage.h @@ -609,7 +609,18 @@ public: using is_transparent = void; // required to make find() work with different type than key_type }; - mutable std::unordered_map nodes; + struct PathCmp + { + using is_transparent = std::true_type; + + auto operator()(const std::string_view a, + const std::string_view b) const + { + return a.size() < b.size() || (a.size() == b.size() && a < b); + } + }; + + mutable std::map nodes; std::unordered_map, Hash, Equal> deltas_for_path; std::list deltas; From d14e03abade4a8a9552546c7969f2d3c2221354c Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Thu, 5 Sep 2024 09:54:27 +0800 Subject: [PATCH 146/215] fix tests incompatible and add new test example --- tests/performance/all_join_opt.xml | 4 ++-- .../03228_join_to_rerange_right_table.reference | 2 ++ .../03228_join_to_rerange_right_table.sql | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/03228_join_to_rerange_right_table.reference create mode 100644 tests/queries/0_stateless/03228_join_to_rerange_right_table.sql diff --git a/tests/performance/all_join_opt.xml b/tests/performance/all_join_opt.xml index ed8805a2e5f..0ab9c39f67c 100644 --- a/tests/performance/all_join_opt.xml +++ b/tests/performance/all_join_opt.xml @@ -5,8 +5,8 @@ INSERT INTO test SELECT number % 10000, number % 10000, number % 10000 FROM numbers(10000000) INSERT INTO test1 SELECT number % 1000 , number % 1000, number % 1000 FROM numbers(100000) - SELECT MAX(test1.a) FROM test INNER JOIN test1 on test.b = test1.b SETTINGS allow_experimental_inner_join_right_table_sorting=true, join_to_sort_table_rows_threshold=100000 - SELECT MAX(test1.a) FROM test LEFT JOIN test1 on test.b = test1.b SETTINGS allow_experimental_inner_join_right_table_sorting=true, join_to_sort_table_rows_threshold=100000 + SELECT MAX(test1.a) FROM test INNER JOIN test1 on test.b = test1.b + SELECT MAX(test1.a) FROM test LEFT JOIN test1 on test.b = test1.b SELECT MAX(test1.a) FROM test RIGHT JOIN test1 on test.b = test1.b SELECT MAX(test1.a) FROM test FULL JOIN test1 on test.b = test1.b diff --git a/tests/queries/0_stateless/03228_join_to_rerange_right_table.reference b/tests/queries/0_stateless/03228_join_to_rerange_right_table.reference new file mode 100644 index 00000000000..b62923296e5 --- /dev/null +++ b/tests/queries/0_stateless/03228_join_to_rerange_right_table.reference @@ -0,0 +1,2 @@ +9 +9 diff --git a/tests/queries/0_stateless/03228_join_to_rerange_right_table.sql b/tests/queries/0_stateless/03228_join_to_rerange_right_table.sql new file mode 100644 index 00000000000..e9001f92405 --- /dev/null +++ b/tests/queries/0_stateless/03228_join_to_rerange_right_table.sql @@ -0,0 +1,14 @@ +drop table if exists test_left; +drop table if exists test_right; + +CREATE TABLE test_left (a Int64, b String, c LowCardinality(String)) ENGINE = MergeTree() ORDER BY a; +CREATE TABLE test_right (a Int64, b String, c LowCardinality(String)) ENGINE = MergeTree() ORDER BY a; + +INSERT INTO test_left SELECT number % 10000, number % 10000, number % 10000 FROM numbers(100000); +INSERT INTO test_right SELECT number % 10 , number % 10, number % 10 FROM numbers(10000); + +SELECT MAX(test_right.a) FROM test_left INNER JOIN test_right on test_left.b = test_right.b SETTINGS allow_experimental_inner_join_right_table_sorting=true; +SELECT MAX(test_right.a) FROM test_left LEFT JOIN test_right on test_left.b = test_right.b SETTINGS allow_experimental_inner_join_right_table_sorting=true; + +drop table test_left; +drop table test_right; From 5180e58dca42959f3c9985459f6c847165e9bd06 Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Thu, 5 Sep 2024 11:01:37 +0000 Subject: [PATCH 147/215] fix collector --- src/Coordination/KeeperStorage.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 9d63a9e8691..aa5b9d539b2 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -1611,11 +1611,16 @@ private: { if constexpr (Storage::use_rocksdb) { + std::filesystem::path root_fs_path(root_path.toString()); auto children = storage.container.getChildren(root_path.toString()); - for (auto && [child_path, node] : children) + for (auto && [child_name, node] : children) + { + auto child_path = (root_fs_path / child_name).generic_string(); + if (collect(child_path, node)) return true; + } } return false; @@ -1657,12 +1662,13 @@ private: for (; it != nodes.end() && parentNodePath(it->first) == root_path; ++it) { chassert(it->second.node); + const String & path = it->first; const typename Storage::Node & node = *it->second.node; if (updateStats(node)) return true; - if (visitRootAndUncommitted(it->first, node)) /// if child is uncommitted then all subtree is also uncommitted + if (visitRootAndUncommitted(path, node)) /// if child is uncommitted then all subtree is also uncommitted return true; } @@ -1684,9 +1690,9 @@ private: bool updateStats(const typename Storage::Node & root_node) { - nodes_observed += 1 + root_node.numChildren(); /// root + all known children + nodes_observed += 1; - if (nodes_observed > limit) + if (nodes_observed + root_node.numChildren() > limit) return true; return false; From ae512fe5337bb24b0ad7cc2bf841d4392cdd9cf2 Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Thu, 5 Sep 2024 11:05:23 +0000 Subject: [PATCH 148/215] add test for single delete --- src/Coordination/tests/gtest_coordination.cpp | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index d39031773cd..413b5f18e33 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -3113,6 +3113,8 @@ TYPED_TEST(CoordinationTest, TestFeatureFlags) ASSERT_TRUE(feature_flags.isEnabled(KeeperFeatureFlag::FILTERED_LIST)); ASSERT_TRUE(feature_flags.isEnabled(KeeperFeatureFlag::MULTI_READ)); ASSERT_FALSE(feature_flags.isEnabled(KeeperFeatureFlag::CHECK_NOT_EXISTS)); + ASSERT_FALSE(feature_flags.isEnabled(KeeperFeatureFlag::CREATE_IF_NOT_EXISTS)); + ASSERT_FALSE(feature_flags.isEnabled(KeeperFeatureFlag::REMOVE_RECURSIVE)); } TYPED_TEST(CoordinationTest, TestSystemNodeModify) @@ -3374,6 +3376,171 @@ TYPED_TEST(CoordinationTest, TestReapplyingDeltas) ASSERT_TRUE(children1_set == children2_set); } +TYPED_TEST(CoordinationTest, TestRemoveRecursiveRequest) +{ + using namespace DB; + using namespace Coordination; + + using Storage = typename TestFixture::Storage; + + ChangelogDirTest rocks("./rocksdb"); + this->setRocksDBDirectory("./rocksdb"); + + Storage storage{500, "", this->keeper_context}; + + int32_t zxid = 0; + + const auto create = [&](const String & path, int create_mode) + { + int new_zxid = ++zxid; + + const auto create_request = std::make_shared(); + create_request->path = path; + create_request->is_ephemeral = create_mode == zkutil::CreateMode::Ephemeral || create_mode == zkutil::CreateMode::EphemeralSequential; + create_request->is_sequential = create_mode == zkutil::CreateMode::PersistentSequential || create_mode == zkutil::CreateMode::EphemeralSequential; + + storage.preprocessRequest(create_request, 1, 0, new_zxid); + auto responses = storage.processRequest(create_request, 1, new_zxid); + + EXPECT_EQ(responses.size(), 1); + EXPECT_EQ(responses[0].response->error, Coordination::Error::ZOK) << "Failed to create " << path; + }; + + const auto remove = [&](const String & path, int32_t version = -1, std::optional remove_nodes_limit = std::nullopt) + { + int new_zxid = ++zxid; + + std::shared_ptr remove_request; + + if (remove_nodes_limit.has_value()) + remove_request = std::make_shared(); + else + remove_request = std::make_shared(); + + remove_request->path = path; + remove_request->version = version; + remove_request->remove_nodes_limit = remove_nodes_limit.value_or(1); + + storage.preprocessRequest(remove_request, 1, 0, new_zxid); + return storage.processRequest(remove_request, 1, new_zxid); + }; + + const auto exists = [&](const String & path) + { + int new_zxid = ++zxid; + + const auto exists_request = std::make_shared(); + exists_request->path = path; + + storage.preprocessRequest(exists_request, 1, 0, new_zxid); + auto responses = storage.processRequest(exists_request, 1, new_zxid); + + EXPECT_EQ(responses.size(), 1); + return responses[0].response->error == Coordination::Error::ZOK; + }; + + { + SCOPED_TRACE("Single Remove Single Node"); + create("/T1", zkutil::CreateMode::Persistent); + + auto responses = remove("/T1"); + ASSERT_EQ(responses.size(), 1); + ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); + ASSERT_FALSE(exists("/T1")); + } + + { + SCOPED_TRACE("Single Remove Tree"); + create("/T2", zkutil::CreateMode::Persistent); + create("/T2/A", zkutil::CreateMode::Persistent); + + auto responses = remove("/T2"); + ASSERT_EQ(responses.size(), 1); + ASSERT_EQ(responses[0].response->error, Coordination::Error::ZNOTEMPTY); + ASSERT_TRUE(exists("/T2")); + } + + { + SCOPED_TRACE("Recursive Remove Single Node"); + create("/T3", zkutil::CreateMode::Persistent); + + auto responses = remove("/T3", 0, 100); + ASSERT_EQ(responses.size(), 1); + ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); + ASSERT_FALSE(exists("/T3")); + } + + { + SCOPED_TRACE("Recursive Remove Tree"); + create("/T4", zkutil::CreateMode::Persistent); + create("/T4/A", zkutil::CreateMode::Persistent); + + auto responses = remove("/T4", 0, 100); + ASSERT_EQ(responses.size(), 1); + ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); + ASSERT_FALSE(exists("/T4")); + ASSERT_FALSE(exists("/T4/A")); + } + + { + SCOPED_TRACE("Recursive Remove Tree Small Limit"); + create("/T5", zkutil::CreateMode::Persistent); + create("/T5/A", zkutil::CreateMode::Persistent); + create("/T5/B", zkutil::CreateMode::Persistent); + create("/T5/A/C", zkutil::CreateMode::Persistent); + + auto responses = remove("/T5", 0, 2); + ASSERT_EQ(responses.size(), 1); + ASSERT_EQ(responses[0].response->error, Coordination::Error::ZNOTEMPTY); + ASSERT_TRUE(exists("/T5")); + ASSERT_TRUE(exists("/T5/A")); + ASSERT_TRUE(exists("/T5/B")); + ASSERT_TRUE(exists("/T5/A/C")); + } + + { + SCOPED_TRACE("Recursive Remove Tree Small Limit"); + create("/T6", zkutil::CreateMode::Persistent); + create("/T6/A", zkutil::CreateMode::Persistent); + create("/T6/B", zkutil::CreateMode::Persistent); + create("/T6/A/C", zkutil::CreateMode::Persistent); + + auto responses = remove("/T6", 0, 2); + ASSERT_EQ(responses.size(), 1); + ASSERT_EQ(responses[0].response->error, Coordination::Error::ZNOTEMPTY); + ASSERT_TRUE(exists("/T6")); + ASSERT_TRUE(exists("/T6/A")); + ASSERT_TRUE(exists("/T6/B")); + ASSERT_TRUE(exists("/T6/A/C")); + } + + { + SCOPED_TRACE("Recursive Remove Ephemeral"); + create("/T7", zkutil::CreateMode::Ephemeral); + + auto responses = remove("/T7", 0, 100); + ASSERT_EQ(responses.size(), 1); + ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); + ASSERT_TRUE(!exists("/T7")); + } + + { + SCOPED_TRACE("Recursive Remove Tree With Ephemeral"); + create("/T8", zkutil::CreateMode::Persistent); + create("/T8/A", zkutil::CreateMode::Persistent); + create("/T8/B", zkutil::CreateMode::Ephemeral); + create("/T8/A/C", zkutil::CreateMode::Ephemeral); + + auto responses = remove("/T8", 0, 4); + ASSERT_EQ(responses.size(), 1); + ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); + ASSERT_TRUE(!exists("/T8")); + ASSERT_TRUE(!exists("/T8/A")); + ASSERT_TRUE(!exists("/T8/B")); + ASSERT_TRUE(!exists("/T8/A/C")); + } +} + /// INSTANTIATE_TEST_SUITE_P(CoordinationTestSuite, /// CoordinationTest, /// ::testing::ValuesIn(std::initializer_list{CompressionParam{true, ".zstd"}, CompressionParam{false, ""}})); From 6455e1dfa15572ab928ccb9907334d417cf69919 Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Thu, 5 Sep 2024 11:14:43 +0000 Subject: [PATCH 149/215] add ephemerals check --- src/Coordination/tests/gtest_coordination.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index 413b5f18e33..f902a806b37 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -3517,10 +3517,12 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveRequest) { SCOPED_TRACE("Recursive Remove Ephemeral"); create("/T7", zkutil::CreateMode::Ephemeral); + ASSERT_EQ(storage.ephemerals.size(), 1); auto responses = remove("/T7", 0, 100); ASSERT_EQ(responses.size(), 1); ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); + ASSERT_EQ(storage.ephemerals.size(), 0); ASSERT_TRUE(!exists("/T7")); } @@ -3530,10 +3532,12 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveRequest) create("/T8/A", zkutil::CreateMode::Persistent); create("/T8/B", zkutil::CreateMode::Ephemeral); create("/T8/A/C", zkutil::CreateMode::Ephemeral); + ASSERT_EQ(storage.ephemerals.size(), 1); auto responses = remove("/T8", 0, 4); ASSERT_EQ(responses.size(), 1); ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); + ASSERT_EQ(storage.ephemerals.size(), 0); ASSERT_TRUE(!exists("/T8")); ASSERT_TRUE(!exists("/T8/A")); ASSERT_TRUE(!exists("/T8/B")); From 20eaecc4f39adf73ac402c88d4a54d70f859453c Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Thu, 5 Sep 2024 13:50:26 +0200 Subject: [PATCH 150/215] Fix build --- src/Storages/MergeTree/MergeTask.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index e6d7b4656c9..398a9472456 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -1585,8 +1585,8 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const std::vector plans; for (size_t i = 0; i < global_ctx->future_part->parts.size(); ++i) { - if (part->getMarksCount() == 0) - LOG_TRACE(ctx->log, "Part {} is empty", part->name); + if (global_ctx->future_part->parts[i]->getMarksCount() == 0) + LOG_TRACE(ctx->log, "Part {} is empty", global_ctx->future_part->parts[i]->name); auto plan_for_part = std::make_unique(); createReadFromPartStep( From c3cc2a3fb19438043c076300cf59d5c1c2fd3957 Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Thu, 5 Sep 2024 16:14:50 +0000 Subject: [PATCH 151/215] intro new request type --- src/Common/ZooKeeper/IKeeper.cpp | 1 + src/Common/ZooKeeper/IKeeper.h | 25 ++- src/Common/ZooKeeper/TestKeeper.cpp | 9 +- src/Common/ZooKeeper/TestKeeper.h | 6 +- src/Common/ZooKeeper/ZooKeeper.cpp | 14 +- src/Common/ZooKeeper/ZooKeeperCommon.cpp | 16 +- src/Common/ZooKeeper/ZooKeeperCommon.h | 36 ++-- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 40 ++-- src/Common/ZooKeeper/ZooKeeperImpl.h | 6 +- .../ZooKeeper/ZooKeeperWithFaultInjection.cpp | 4 +- src/Coordination/KeeperDispatcher.cpp | 7 +- src/Coordination/KeeperStorage.cpp | 171 +++++++++++++----- 12 files changed, 235 insertions(+), 100 deletions(-) diff --git a/src/Common/ZooKeeper/IKeeper.cpp b/src/Common/ZooKeeper/IKeeper.cpp index 7cca262baca..34c9d94fca5 100644 --- a/src/Common/ZooKeeper/IKeeper.cpp +++ b/src/Common/ZooKeeper/IKeeper.cpp @@ -171,6 +171,7 @@ bool isUserError(Error zk_return_code) void CreateRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); } void RemoveRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); } +void RemoveRecursiveRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); } void ExistsRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); } void GetRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); } void SetRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); } diff --git a/src/Common/ZooKeeper/IKeeper.h b/src/Common/ZooKeeper/IKeeper.h index f887e67bd92..a0d6ae54f56 100644 --- a/src/Common/ZooKeeper/IKeeper.h +++ b/src/Common/ZooKeeper/IKeeper.h @@ -238,16 +238,30 @@ struct RemoveRequest : virtual Request String path; int32_t version = -1; + void addRootPath(const String & root_path) override; + String getPath() const override { return path; } + + size_t bytesSize() const override { return path.size() + sizeof(version); } +}; + +struct RemoveResponse : virtual Response +{ +}; + +struct RemoveRecursiveRequest : virtual Request +{ + String path; + /// strict limit for number of deleted nodes uint32_t remove_nodes_limit = 1; void addRootPath(const String & root_path) override; String getPath() const override { return path; } - size_t bytesSize() const override { return path.size() + sizeof(version) + sizeof(remove_nodes_limit); } + size_t bytesSize() const override { return path.size() + sizeof(remove_nodes_limit); } }; -struct RemoveResponse : virtual Response +struct RemoveRecursiveResponse : virtual Response { }; @@ -433,6 +447,7 @@ struct ErrorResponse : virtual Response using CreateCallback = std::function; using RemoveCallback = std::function; +using RemoveRecursiveCallback = std::function; using ExistsCallback = std::function; using GetCallback = std::function; using SetCallback = std::function; @@ -588,9 +603,13 @@ public: virtual void remove( const String & path, int32_t version, - uint32_t remove_nodes_limit, RemoveCallback callback) = 0; + virtual void removeRecursive( + const String & path, + uint32_t remove_nodes_limit, + RemoveRecursiveCallback callback) = 0; + virtual void exists( const String & path, ExistsCallback callback, diff --git a/src/Common/ZooKeeper/TestKeeper.cpp b/src/Common/ZooKeeper/TestKeeper.cpp index 65a575549c4..1ac085d6050 100644 --- a/src/Common/ZooKeeper/TestKeeper.cpp +++ b/src/Common/ZooKeeper/TestKeeper.cpp @@ -759,7 +759,6 @@ void TestKeeper::create( void TestKeeper::remove( const String & path, int32_t version, - [[maybe_unused]] uint32_t remove_nodes_limit, // TODO(michicosun): enable RemoveCallback callback) { TestKeeperRemoveRequest request; @@ -772,6 +771,14 @@ void TestKeeper::remove( pushRequest(std::move(request_info)); } +void TestKeeper::removeRecursive( + [[maybe_unused]] const String & path, + [[maybe_unused]] uint32_t remove_nodes_limit, + [[maybe_unused]] RemoveRecursiveCallback callback) +{ + /// TODO(michicosun) implement +} + void TestKeeper::exists( const String & path, ExistsCallback callback, diff --git a/src/Common/ZooKeeper/TestKeeper.h b/src/Common/ZooKeeper/TestKeeper.h index 14b37448a20..c32f0064dec 100644 --- a/src/Common/ZooKeeper/TestKeeper.h +++ b/src/Common/ZooKeeper/TestKeeper.h @@ -56,9 +56,13 @@ public: void remove( const String & path, int32_t version, - uint32_t remove_nodes_limit, RemoveCallback callback) override; + void removeRecursive( + const String & path, + uint32_t remove_nodes_limit, + RemoveRecursiveCallback callback) override; + void exists( const String & path, ExistsCallback callback, diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index eb4f5e0725a..93e833e87c8 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -981,16 +981,16 @@ bool ZooKeeper::tryRemoveChildrenRecursive(const std::string & path, bool probab void ZooKeeper::removeRecursive(const std::string & path) // TODO(michicosun) rewrite { - auto promise = std::make_shared>(); + auto promise = std::make_shared>(); auto future = promise->get_future(); - auto callback = [promise](const Coordination::RemoveResponse & response) mutable + auto callback = [promise](const Coordination::RemoveRecursiveResponse & response) mutable { promise->set_value(response); }; - impl->remove(path, -1, /*remove_nodes_limit=*/ 100, std::move(callback)); - + impl->removeRecursive(path, 100, std::move(callback)); + if (future.wait_for(std::chrono::milliseconds(args.operation_timeout_ms)) != std::future_status::ready) { impl->finalize(fmt::format("Operation timeout on {} {}", Coordination::OpNum::RemoveRecursive, path)); @@ -1385,7 +1385,7 @@ std::future ZooKeeper::asyncRemove(const std::stri promise->set_value(response); }; - impl->remove(path, version, /*remove_nodes_limit=*/ 1, std::move(callback)); + impl->remove(path, version, std::move(callback)); return future; } @@ -1408,7 +1408,7 @@ std::future ZooKeeper::asyncTryRemove(const std::s promise->set_value(response); }; - impl->remove(path, version, /*remove_nodes_limit=*/ 1, std::move(callback)); + impl->remove(path, version, std::move(callback)); return future; } @@ -1422,7 +1422,7 @@ std::future ZooKeeper::asyncTryRemoveNoThrow(const promise->set_value(response); }; - impl->remove(path, version, /*remove_nodes_limit=*/ 1, std::move(callback)); + impl->remove(path, version, std::move(callback)); return future; } diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.cpp b/src/Common/ZooKeeper/ZooKeeperCommon.cpp index fb37aa6ac22..3f9225e84dd 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.cpp +++ b/src/Common/ZooKeeper/ZooKeeperCommon.cpp @@ -232,24 +232,24 @@ void ZooKeeperRemoveRequest::readImpl(ReadBuffer & in) Coordination::read(version, in); } -void ZooKeeperRemoveRecursiveRequest::writeImpl(WriteBuffer & out) const +void ZooKeeperRemoveRecursiveRequest::writeImpl(WriteBuffer & out) const { - ZooKeeperRemoveRequest::writeImpl(out); + Coordination::write(path, out); Coordination::write(remove_nodes_limit, out); } -void ZooKeeperRemoveRecursiveRequest::readImpl(ReadBuffer & in) +void ZooKeeperRemoveRecursiveRequest::readImpl(ReadBuffer & in) { - ZooKeeperRemoveRequest::readImpl(in); + Coordination::read(path, in); Coordination::read(remove_nodes_limit, in); } -std::string ZooKeeperRemoveRecursiveRequest::toStringImpl(bool short_format) const +std::string ZooKeeperRemoveRecursiveRequest::toStringImpl(bool /*short_format*/) const { return fmt::format( - "{}\n" + "path = {}\n" "remove_nodes_limit = {}", - ZooKeeperRemoveRequest::toStringImpl(short_format), + path, remove_nodes_limit); } @@ -531,7 +531,7 @@ ZooKeeperMultiRequest::ZooKeeperMultiRequest(std::span(*concrete_request_remove)); } - else if (const auto * concrete_request_remove_recursive = dynamic_cast(generic_request.get())) + else if (const auto * concrete_request_remove_recursive = dynamic_cast(generic_request.get())) { checkOperationType(Write); requests.push_back(std::make_shared(*concrete_request_remove_recursive)); diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.h b/src/Common/ZooKeeper/ZooKeeperCommon.h index ef17d069b4c..66c075b277b 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.h +++ b/src/Common/ZooKeeper/ZooKeeperCommon.h @@ -258,7 +258,7 @@ struct ZooKeeperCreateIfNotExistsResponse : ZooKeeperCreateResponse using ZooKeeperCreateResponse::ZooKeeperCreateResponse; }; -struct ZooKeeperRemoveRequest : RemoveRequest, ZooKeeperRequest +struct ZooKeeperRemoveRequest final : RemoveRequest, ZooKeeperRequest { ZooKeeperRemoveRequest() = default; explicit ZooKeeperRemoveRequest(const RemoveRequest & base) : RemoveRequest(base) {} @@ -276,17 +276,7 @@ struct ZooKeeperRemoveRequest : RemoveRequest, ZooKeeperRequest void createLogElements(LogElements & elems) const override; }; -struct ZooKeeperRemoveRecursiveRequest : ZooKeeperRemoveRequest -{ - OpNum getOpNum() const override { return OpNum::RemoveRecursive; } - void writeImpl(WriteBuffer & out) const override; - void readImpl(ReadBuffer & in) override; - std::string toStringImpl(bool short_format) const override; - - ZooKeeperResponsePtr makeResponse() const override; -}; - -struct ZooKeeperRemoveResponse : RemoveResponse, ZooKeeperResponse +struct ZooKeeperRemoveResponse final : RemoveResponse, ZooKeeperResponse { void readImpl(ReadBuffer &) override {} void writeImpl(WriteBuffer &) const override {} @@ -295,9 +285,29 @@ struct ZooKeeperRemoveResponse : RemoveResponse, ZooKeeperResponse size_t bytesSize() const override { return RemoveResponse::bytesSize() + sizeof(xid) + sizeof(zxid); } }; -struct ZooKeeperRemoveRecursiveResponse : ZooKeeperRemoveResponse +struct ZooKeeperRemoveRecursiveRequest final : RemoveRecursiveRequest, ZooKeeperRequest { + ZooKeeperRemoveRecursiveRequest() = default; + explicit ZooKeeperRemoveRecursiveRequest(const RemoveRecursiveRequest & base) : RemoveRecursiveRequest(base) {} + OpNum getOpNum() const override { return OpNum::RemoveRecursive; } + void writeImpl(WriteBuffer & out) const override; + void readImpl(ReadBuffer & in) override; + std::string toStringImpl(bool short_format) const override; + + ZooKeeperResponsePtr makeResponse() const override; + bool isReadRequest() const override { return false; } + + size_t bytesSize() const override { return RemoveRecursiveRequest::bytesSize() + sizeof(xid); } +}; + +struct ZooKeeperRemoveRecursiveResponse : RemoveRecursiveResponse, ZooKeeperResponse +{ + void readImpl(ReadBuffer &) override {} + void writeImpl(WriteBuffer &) const override {} + OpNum getOpNum() const override { return OpNum::RemoveRecursive; } + + size_t bytesSize() const override { return RemoveRecursiveResponse::bytesSize() + sizeof(xid) + sizeof(zxid); } }; struct ZooKeeperExistsRequest final : ExistsRequest, ZooKeeperRequest diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 5c4de98d5fc..a6dd9738e17 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1333,33 +1333,39 @@ void ZooKeeper::create( void ZooKeeper::remove( const String & path, int32_t version, - uint32_t remove_nodes_limit, RemoveCallback callback) { - std::shared_ptr request = nullptr; - - if (!isFeatureEnabled(KeeperFeatureFlag::REMOVE_RECURSIVE)) - { - if (remove_nodes_limit > 1) - throw Exception::fromMessage(Error::ZBADARGUMENTS, "RemoveRecursive request type cannot be used because it's not supported by the server"); - - request = std::make_shared(); - } - else - request = std::make_shared(); - - request->path = path; - request->version = version; - request->remove_nodes_limit = remove_nodes_limit; + ZooKeeperRemoveRequest request; + request.path = path; + request.version = version; RequestInfo request_info; - request_info.request = std::move(request); + request_info.request = std::make_shared(std::move(request)); request_info.callback = [callback](const Response & response) { callback(dynamic_cast(response)); }; pushRequest(std::move(request_info)); ProfileEvents::increment(ProfileEvents::ZooKeeperRemove); } +void ZooKeeper::removeRecursive( + const String &path, + uint32_t remove_nodes_limit, + RemoveRecursiveCallback callback) +{ + if (!isFeatureEnabled(KeeperFeatureFlag::REMOVE_RECURSIVE)) + throw Exception::fromMessage(Error::ZBADARGUMENTS, "RemoveRecursive request type cannot be used because it's not supported by the server"); + + ZooKeeperRemoveRecursiveRequest request; + request.path = path; + request.remove_nodes_limit = remove_nodes_limit; + + RequestInfo request_info; + request_info.request = std::make_shared(std::move(request)); + request_info.callback = [callback](const Response & response) { callback(dynamic_cast(response)); }; + + pushRequest(std::move(request_info)); + ProfileEvents::increment(ProfileEvents::ZooKeeperRemove); +} void ZooKeeper::exists( const String & path, diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.h b/src/Common/ZooKeeper/ZooKeeperImpl.h index 6e98df546d9..47d2ab8f401 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.h +++ b/src/Common/ZooKeeper/ZooKeeperImpl.h @@ -144,9 +144,13 @@ public: void remove( const String & path, int32_t version, - uint32_t remove_nodes_limit, RemoveCallback callback) override; + void removeRecursive( + const String &path, + uint32_t remove_nodes_limit, + RemoveRecursiveCallback callback) override; + void exists( const String & path, ExistsCallback callback, diff --git a/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.cpp b/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.cpp index dfd3faf8d01..fc246e263d9 100644 --- a/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.cpp +++ b/src/Common/ZooKeeper/ZooKeeperWithFaultInjection.cpp @@ -587,7 +587,7 @@ zkutil::ZooKeeper::FutureRemove ZooKeeperWithFaultInjection::asyncTryRemove(std: promise->set_value(response); }; - keeper->impl->remove(path, version, /*remove_nodes_limit=*/ 1, std::move(callback)); + keeper->impl->remove(path, version, std::move(callback)); return future; } @@ -630,7 +630,7 @@ zkutil::ZooKeeper::FutureRemove ZooKeeperWithFaultInjection::asyncTryRemoveNoThr } }; - keeper->impl->remove(path, version, /*remove_nodes_limit=*/ 1, std::move(callback)); + keeper->impl->remove(path, version, std::move(callback)); return future; } diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp index fd80ee0a7e2..c91cdb53222 100644 --- a/src/Coordination/KeeperDispatcher.cpp +++ b/src/Coordination/KeeperDispatcher.cpp @@ -86,12 +86,17 @@ bool checkIfRequestIncreaseMem(const Coordination::ZooKeeperRequestPtr & request break; } case Coordination::OpNum::Remove: - case Coordination::OpNum::RemoveRecursive: { Coordination::ZooKeeperRemoveRequest & remove_req = dynamic_cast(*sub_zk_request); memory_delta -= remove_req.bytesSize(); break; } + case Coordination::OpNum::RemoveRecursive: + { + Coordination::ZooKeeperRemoveRecursiveRequest & remove_req = dynamic_cast(*sub_zk_request); + memory_delta -= remove_req.bytesSize(); + break; + } default: break; } diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index aa5b9d539b2..613714263ef 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -1462,16 +1462,41 @@ struct KeeperStorageGetRequestProcessor final : public KeeperStorageRequestProce } }; +namespace +{ + +template +void addUpdateParentPzxidDelta(Storage & storage, std::vector & deltas, int64_t zxid, StringRef path) +{ + auto parent_path = parentNodePath(path); + if (!storage.uncommitted_state.getNode(parent_path)) + return; + + deltas.emplace_back( + std::string{parent_path}, + zxid, + typename Storage::UpdateNodeDelta + { + [zxid](Storage::Node & parent) + { + parent.pzxid = std::max(parent.pzxid, zxid); + } + } + ); +} + +} + template struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestProcessor { + using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor; + bool checkAuth(Storage & storage, int64_t session_id, bool is_local) const override { return storage.checkACL(parentNodePath(this->zk_request->getPath()), Coordination::ACL::Delete, session_id, is_local); } - using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor; - std::vector preprocess(Storage & storage, int64_t zxid, int64_t /*session_id*/, int64_t /*time*/, uint64_t & digest, const KeeperContext & keeper_context) const override { @@ -1488,62 +1513,35 @@ struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestPr return {typename Storage::Delta{zxid, Coordination::Error::ZBADARGUMENTS}}; } - const auto update_parent_pzxid = [&]() - { - auto parent_path = parentNodePath(request.path); - if (!storage.uncommitted_state.getNode(parent_path)) - return; - - new_deltas.emplace_back( - std::string{parent_path}, - zxid, - typename Storage::UpdateNodeDelta - { - [zxid](Storage::Node & parent) - { - parent.pzxid = std::max(parent.pzxid, zxid); - } - } - ); - }; - auto node = storage.uncommitted_state.getNode(request.path); if (!node) { if (request.restored_from_zookeeper_log) - update_parent_pzxid(); - + addUpdateParentPzxidDelta(storage, new_deltas, zxid, request.path); return {typename Storage::Delta{zxid, Coordination::Error::ZNONODE}}; } - - if (request.version != -1 && request.version != node->version) + else if (request.version != -1 && request.version != node->version) return {typename Storage::Delta{zxid, Coordination::Error::ZBADVERSION}}; - - ToDeleteTreeCollector collector(storage, zxid, request.remove_nodes_limit); - bool limit_exceeded = collector.collect(request.path, *node); - - if (limit_exceeded) + else if (node->numChildren() != 0) return {typename Storage::Delta{zxid, Coordination::Error::ZNOTEMPTY}}; if (request.restored_from_zookeeper_log) - update_parent_pzxid(); + addUpdateParentPzxidDelta(storage, new_deltas, zxid, request.path); - auto delete_deltas = collector.extractDeltas(); + new_deltas.emplace_back( + std::string{parentNodePath(request.path)}, + zxid, + typename Storage::UpdateNodeDelta{[](typename Storage::Node & parent) + { + ++parent.cversion; + parent.decreaseNumChildren(); + }}); - for (const auto & delta : delete_deltas) - std::visit( - Overloaded{ - [&](const typename Storage::RemoveNodeDelta & remove_delta) - { - if (remove_delta.ephemeral_owner) - storage.unregisterEphemeralPath(remove_delta.ephemeral_owner, delta.path); - }, - [](auto && /* delta */) {}, - }, - delta.operation); + new_deltas.emplace_back(request.path, zxid, typename Storage::RemoveNodeDelta{request.version, node->ephemeralOwner()}); - new_deltas.insert(new_deltas.end(), std::make_move_iterator(delete_deltas.begin()), std::make_move_iterator(delete_deltas.end())); + if (node->isEphemeral()) + storage.unregisterEphemeralPath(node->ephemeralOwner(), request.path); digest = storage.calculateNodesDigest(digest, new_deltas); @@ -1564,6 +1562,84 @@ struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestPr { return processWatchesImpl(this->zk_request->getPath(), watches, list_watches, Coordination::Event::DELETED); } +}; + +template +struct KeeperStorageRemoveRecursiveRequestProcessor final : public KeeperStorageRequestProcessor +{ + using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor; + + bool checkAuth(Storage & storage, int64_t session_id, bool is_local) const override + { + return storage.checkACL(parentNodePath(this->zk_request->getPath()), Coordination::ACL::Delete, session_id, is_local); + } + + std::vector + preprocess(Storage & storage, int64_t zxid, int64_t /*session_id*/, int64_t /*time*/, uint64_t & digest, const KeeperContext & keeper_context) const override + { + ProfileEvents::increment(ProfileEvents::KeeperRemoveRequest); + Coordination::ZooKeeperRemoveRecursiveRequest & request = dynamic_cast(*this->zk_request); + + std::vector new_deltas; + + if (Coordination::matchPath(request.path, keeper_system_path) != Coordination::PathMatchResult::NOT_MATCH) + { + auto error_msg = fmt::format("Trying to delete an internal Keeper path ({}) which is not allowed", request.path); + + handleSystemNodeModification(keeper_context, error_msg); + return {typename Storage::Delta{zxid, Coordination::Error::ZBADARGUMENTS}}; + } + + auto node = storage.uncommitted_state.getNode(request.path); + + if (!node) + { + if (request.restored_from_zookeeper_log) + addUpdateParentPzxidDelta(storage, new_deltas, zxid, request.path); + + return {typename Storage::Delta{zxid, Coordination::Error::ZNONODE}}; + } + + ToDeleteTreeCollector collector(storage, zxid, request.remove_nodes_limit); + bool limit_exceeded = collector.collect(request.path, *node); + + if (limit_exceeded) + return {typename Storage::Delta{zxid, Coordination::Error::ZNOTEMPTY}}; + + if (request.restored_from_zookeeper_log) + addUpdateParentPzxidDelta(storage, new_deltas, zxid, request.path); + + auto delete_deltas = collector.extractDeltas(); + + for (const auto & delta : delete_deltas) + { + const auto * remove_delta = std::get_if(&delta.operation); + if (remove_delta && remove_delta->ephemeral_owner) + storage.unregisterEphemeralPath(remove_delta->ephemeral_owner, delta.path); + } + + new_deltas.insert(new_deltas.end(), std::make_move_iterator(delete_deltas.begin()), std::make_move_iterator(delete_deltas.end())); + + digest = storage.calculateNodesDigest(digest, new_deltas); + + return new_deltas; + } + + Coordination::ZooKeeperResponsePtr process(Storage & storage, int64_t zxid) const override + { + Coordination::ZooKeeperResponsePtr response_ptr = this->zk_request->makeResponse(); + Coordination::ZooKeeperRemoveRecursiveResponse & response = dynamic_cast(*response_ptr); + + response.error = storage.commit(zxid); + return response_ptr; + } + + KeeperStorageBase::ResponsesForSessions + processWatches(KeeperStorageBase::Watches & watches, KeeperStorageBase::Watches & list_watches) const override + { + /// TODO(michicosun) rewrite + return processWatchesImpl(this->zk_request->getPath(), watches, list_watches, Coordination::Event::DELETED); + } private: class ToDeleteTreeCollector @@ -2270,10 +2346,13 @@ struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestPro concrete_requests.push_back(std::make_shared>(sub_zk_request)); break; case Coordination::OpNum::Remove: - case Coordination::OpNum::RemoveRecursive: check_operation_type(OperationType::Write); concrete_requests.push_back(std::make_shared>(sub_zk_request)); break; + case Coordination::OpNum::RemoveRecursive: + check_operation_type(OperationType::Write); + concrete_requests.push_back(std::make_shared>(sub_zk_request)); + break; case Coordination::OpNum::Set: check_operation_type(OperationType::Write); concrete_requests.push_back(std::make_shared>(sub_zk_request)); @@ -2543,7 +2622,7 @@ KeeperStorageRequestProcessorsFactory::KeeperStorageRequestProcessorsFa registerKeeperRequestProcessor>(*this); registerKeeperRequestProcessor>(*this); registerKeeperRequestProcessor>(*this); - registerKeeperRequestProcessor>(*this); + registerKeeperRequestProcessor>(*this); } From cb1c11c74ae33d9de40ffc86dd7f2dc976be309b Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Fri, 6 Sep 2024 00:08:08 +0000 Subject: [PATCH 152/215] change traverse --- src/Coordination/KeeperStorage.cpp | 130 +++++++++++++++++++---------- 1 file changed, 84 insertions(+), 46 deletions(-) diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 613714263ef..9378b30ea52 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -1642,14 +1642,17 @@ struct KeeperStorageRemoveRecursiveRequestProcessor final : public KeeperStorage } private: + using SNode = typename Storage::Node; + class ToDeleteTreeCollector { Storage & storage; int64_t zxid; uint32_t limit; - uint32_t nodes_observed = 0; - std::vector deltas; + uint32_t max_level = 0; + uint32_t nodes_observed = 1; /// root node + std::unordered_map> by_level_deltas; public: ToDeleteTreeCollector(Storage & storage_, int64_t zxid_, uint32_t limit_) @@ -1659,31 +1662,60 @@ private: { } - bool collect(StringRef root_path, const typename Storage::Node & root_node) + bool collect(StringRef root_path, const SNode & root_node) { - if (updateStats(root_node)) + std::deque steps; + steps.push_back(Step{root_path.toString(), &root_node, 0}); + + if (observeNode(root_node)) return true; - if (visitRocksDBNode(root_path) || visitMemNode(root_path)) - return true; + while (!steps.empty()) + { + Step step = std::move(steps.front()); + steps.pop_front(); - /// After checking committed nodes there might be some in uncommitted state, - /// for example made on the previouse step of multi transaction. - return visitRootAndUncommitted(root_path, root_node); - } + StringRef path = step.path; + uint32_t level = step.level; + const SNode * node = nullptr; - uint32_t getNumObserved() const - { - return nodes_observed; + if (auto * rdb = std::get_if(&step.node)) + node = rdb; + else + node = std::get(step.node); + + chassert(!path.empty()); + chassert(node != nullptr); + + if (visitRocksDBNode(steps, path, level) || visitMemNode(steps, path, level) || visitRootAndUncommitted(steps, path, *node, level)) + return true; + } + + return false; } std::vector extractDeltas() { + std::vector deltas; + + for (ssize_t level = max_level; level >= 0; --level) + { + auto & level_deltas = by_level_deltas[static_cast(level)]; + deltas.insert(deltas.end(), std::make_move_iterator(level_deltas.begin()), std::make_move_iterator(level_deltas.end())); + } + return std::move(deltas); } private: - bool visitRocksDBNode(StringRef root_path) + struct Step + { + String path; + std::variant node; + uint32_t level; + }; + + bool visitRocksDBNode(std::deque & steps, StringRef root_path, uint32_t level) { if constexpr (Storage::use_rocksdb) { @@ -1692,86 +1724,92 @@ private: for (auto && [child_name, node] : children) { - auto child_path = (root_fs_path / child_name).generic_string(); - - if (collect(child_path, node)) + if (observeNode(node)) return true; + + auto child_path = (root_fs_path / child_name).generic_string(); + steps.push_back(Step{std::move(child_path), std::move(node), level + 1}); } } return false; } - bool visitMemNode(StringRef root_path) + bool visitMemNode(std::deque & steps, StringRef root_path, uint32_t level) { if constexpr (!Storage::use_rocksdb) { std::filesystem::path root_fs_path(root_path.toString()); auto node_it = storage.container.find(root_path); - if (node_it != storage.container.end()) + if (node_it == storage.container.end()) + return false; + + + auto children = node_it->value.getChildren(); + for (auto && child_name : children) { - auto children = node_it->value.getChildren(); - for (auto && child_name : children) - { - auto child_path = (root_fs_path / child_name.toView()).generic_string(); + auto child_path = (root_fs_path / child_name.toView()).generic_string(); - auto child_it = storage.container.find(child_path); - chassert(child_it != storage.container.end()); + auto child_it = storage.container.find(child_path); + chassert(child_it != storage.container.end()); - if (collect(child_path, child_it->value)) - return true; - } + if (observeNode(child_it->value)) + return true; + + steps.push_back(Step{std::move(child_path), child_it->value, level + 1}); } } return false; } - bool visitRootAndUncommitted(StringRef root_path, const typename Storage::Node & root_node) + bool visitRootAndUncommitted(std::deque & steps, StringRef root_path, const SNode & root_node, uint32_t level) { const auto & nodes = storage.uncommitted_state.nodes; /// nodes are sorted by paths with level locality - auto it = nodes.upper_bound(root_path.toView()); + auto it = nodes.upper_bound(root_path.toString() + "/"); for (; it != nodes.end() && parentNodePath(it->first) == root_path; ++it) { chassert(it->second.node); const String & path = it->first; - const typename Storage::Node & node = *it->second.node; + const SNode & node = *it->second.node; - if (updateStats(node)) + if (observeNode(node)) return true; - if (visitRootAndUncommitted(path, node)) /// if child is uncommitted then all subtree is also uncommitted - return true; + steps.push_back(Step{path, &node, level + 1}); } - deltas.emplace_back( + addDelta(root_path, root_node, level); + + return false; + } + + void addDelta(StringRef root_path, const SNode & root_node, uint32_t level) + { + max_level = std::max(max_level, level); + + by_level_deltas[level].emplace_back( parentNodePath(root_path).toString(), zxid, typename Storage::UpdateNodeDelta{ - [](typename Storage::Node & parent) + [](SNode & parent) { ++parent.cversion; parent.decreaseNumChildren(); } }); - deltas.emplace_back(root_path.toString(), zxid, typename Storage::RemoveNodeDelta{root_node.version, root_node.ephemeralOwner()}); - - return false; + by_level_deltas[level].emplace_back(root_path.toString(), zxid, typename Storage::RemoveNodeDelta{root_node.version, root_node.ephemeralOwner()}); } - bool updateStats(const typename Storage::Node & root_node) + bool observeNode(const SNode & node) { - nodes_observed += 1; - - if (nodes_observed + root_node.numChildren() > limit) - return true; - - return false; + nodes_observed += node.numChildren(); + return nodes_observed > limit; } }; }; From aa2721f7e49d54e97e177ee75ca9e5b39e1a2beb Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Fri, 6 Sep 2024 00:09:46 +0000 Subject: [PATCH 153/215] update tests --- src/Common/ZooKeeper/Types.h | 1 + src/Common/ZooKeeper/ZooKeeper.cpp | 8 ++ src/Coordination/tests/gtest_coordination.cpp | 126 +++++++++++++++--- 3 files changed, 114 insertions(+), 21 deletions(-) diff --git a/src/Common/ZooKeeper/Types.h b/src/Common/ZooKeeper/Types.h index d2876adaabc..4a163c15838 100644 --- a/src/Common/ZooKeeper/Types.h +++ b/src/Common/ZooKeeper/Types.h @@ -31,6 +31,7 @@ using AsyncResponses = std::vector>>; Coordination::RequestPtr makeCreateRequest(const std::string & path, const std::string & data, int create_mode, bool ignore_if_exists = false); Coordination::RequestPtr makeRemoveRequest(const std::string & path, int version); +Coordination::RequestPtr makeRemoveRecursiveRequest(const std::string & path, uint32_t remove_nodes_limit); Coordination::RequestPtr makeSetRequest(const std::string & path, const std::string & data, int version); Coordination::RequestPtr makeCheckRequest(const std::string & path, int version); diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 93e833e87c8..65c3fdba8d2 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -1637,6 +1637,14 @@ Coordination::RequestPtr makeRemoveRequest(const std::string & path, int version return request; } +Coordination::RequestPtr makeRemoveRecursiveRequest(const std::string & path, uint32_t remove_nodes_limit) +{ + auto request = std::make_shared(); + request->path = path; + request->remove_nodes_limit = remove_nodes_limit; + return request; +} + Coordination::RequestPtr makeSetRequest(const std::string & path, const std::string & data, int version) { auto request = std::make_shared(); diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index f902a806b37..27061b15b90 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -3406,20 +3406,25 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveRequest) EXPECT_EQ(responses[0].response->error, Coordination::Error::ZOK) << "Failed to create " << path; }; - const auto remove = [&](const String & path, int32_t version = -1, std::optional remove_nodes_limit = std::nullopt) + const auto remove = [&](const String & path, int32_t version = -1) { int new_zxid = ++zxid; - std::shared_ptr remove_request; - - if (remove_nodes_limit.has_value()) - remove_request = std::make_shared(); - else - remove_request = std::make_shared(); - + auto remove_request = std::make_shared(); remove_request->path = path; remove_request->version = version; - remove_request->remove_nodes_limit = remove_nodes_limit.value_or(1); + + storage.preprocessRequest(remove_request, 1, 0, new_zxid); + return storage.processRequest(remove_request, 1, new_zxid); + }; + + const auto remove_recursive = [&](const String & path, uint32_t remove_nodes_limit = 1) + { + int new_zxid = ++zxid; + + auto remove_request = std::make_shared(); + remove_request->path = path; + remove_request->remove_nodes_limit = remove_nodes_limit; storage.preprocessRequest(remove_request, 1, 0, new_zxid); return storage.processRequest(remove_request, 1, new_zxid); @@ -3464,7 +3469,7 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveRequest) SCOPED_TRACE("Recursive Remove Single Node"); create("/T3", zkutil::CreateMode::Persistent); - auto responses = remove("/T3", 0, 100); + auto responses = remove_recursive("/T3", 100); ASSERT_EQ(responses.size(), 1); ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); ASSERT_FALSE(exists("/T3")); @@ -3475,7 +3480,7 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveRequest) create("/T4", zkutil::CreateMode::Persistent); create("/T4/A", zkutil::CreateMode::Persistent); - auto responses = remove("/T4", 0, 100); + auto responses = remove_recursive("/T4", 100); ASSERT_EQ(responses.size(), 1); ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); ASSERT_FALSE(exists("/T4")); @@ -3489,7 +3494,7 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveRequest) create("/T5/B", zkutil::CreateMode::Persistent); create("/T5/A/C", zkutil::CreateMode::Persistent); - auto responses = remove("/T5", 0, 2); + auto responses = remove_recursive("/T5", 2); ASSERT_EQ(responses.size(), 1); ASSERT_EQ(responses[0].response->error, Coordination::Error::ZNOTEMPTY); ASSERT_TRUE(exists("/T5")); @@ -3499,19 +3504,19 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveRequest) } { - SCOPED_TRACE("Recursive Remove Tree Small Limit"); + SCOPED_TRACE("Recursive Remove Tree Big Limit"); create("/T6", zkutil::CreateMode::Persistent); create("/T6/A", zkutil::CreateMode::Persistent); create("/T6/B", zkutil::CreateMode::Persistent); create("/T6/A/C", zkutil::CreateMode::Persistent); - auto responses = remove("/T6", 0, 2); + auto responses = remove_recursive("/T6", 4); ASSERT_EQ(responses.size(), 1); - ASSERT_EQ(responses[0].response->error, Coordination::Error::ZNOTEMPTY); - ASSERT_TRUE(exists("/T6")); - ASSERT_TRUE(exists("/T6/A")); - ASSERT_TRUE(exists("/T6/B")); - ASSERT_TRUE(exists("/T6/A/C")); + ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); + ASSERT_FALSE(exists("/T6")); + ASSERT_FALSE(exists("/T6/A")); + ASSERT_FALSE(exists("/T6/B")); + ASSERT_FALSE(exists("/T6/A/C")); } { @@ -3519,7 +3524,7 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveRequest) create("/T7", zkutil::CreateMode::Ephemeral); ASSERT_EQ(storage.ephemerals.size(), 1); - auto responses = remove("/T7", 0, 100); + auto responses = remove_recursive("/T7", 100); ASSERT_EQ(responses.size(), 1); ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); ASSERT_EQ(storage.ephemerals.size(), 0); @@ -3534,7 +3539,7 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveRequest) create("/T8/A/C", zkutil::CreateMode::Ephemeral); ASSERT_EQ(storage.ephemerals.size(), 1); - auto responses = remove("/T8", 0, 4); + auto responses = remove_recursive("/T8", 4); ASSERT_EQ(responses.size(), 1); ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); ASSERT_EQ(storage.ephemerals.size(), 0); @@ -3545,6 +3550,85 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveRequest) } } +TYPED_TEST(CoordinationTest, TestRemoveRecursiveInMultiRequest) +{ + using namespace DB; + using namespace Coordination; + + using Storage = typename TestFixture::Storage; + + ChangelogDirTest rocks("./rocksdb"); + this->setRocksDBDirectory("./rocksdb"); + + Storage storage{500, "", this->keeper_context}; + int zxid = 0; + + Coordination::Requests ops; + ops.push_back(zkutil::makeCreateRequest("/A", "A", zkutil::CreateMode::Persistent)); + ops.push_back(zkutil::makeCreateRequest("/A/B", "B", zkutil::CreateMode::Persistent)); + ops.push_back(zkutil::makeCreateRequest("/A/C", "C", zkutil::CreateMode::Ephemeral)); + ops.push_back(zkutil::makeCreateRequest("/A/B/D", "D", zkutil::CreateMode::Ephemeral)); + + const auto exists = [&](const String & path) + { + int new_zxid = ++zxid; + + const auto exists_request = std::make_shared(); + exists_request->path = path; + + storage.preprocessRequest(exists_request, 1, 0, new_zxid); + auto responses = storage.processRequest(exists_request, 1, new_zxid); + + EXPECT_EQ(responses.size(), 1); + return responses[0].response->error == Coordination::Error::ZOK; + }; + + const auto is_multi_ok = [&](Coordination::ZooKeeperResponsePtr response) + { + const auto & multi_response = dynamic_cast(*response); + + for (const auto & op_response : multi_response.responses) + if (op_response->error != Coordination::Error::ZOK) + return false; + + return true; + }; + + { + SCOPED_TRACE("Remove In Multi Tx"); + int new_zxid = ++zxid; + + ops.push_back(zkutil::makeRemoveRequest("/A", -1)); + const auto request = std::make_shared(ops, ACLs{}); + + storage.preprocessRequest(request, 1, 0, new_zxid); + auto responses = storage.processRequest(request, 1, new_zxid); + ops.pop_back(); + + ASSERT_EQ(responses.size(), 1); + ASSERT_FALSE(is_multi_ok(responses[0].response)); + } + + { + SCOPED_TRACE("Recursive Remove In Multi Tx"); + int new_zxid = ++zxid; + + ops.push_back(zkutil::makeRemoveRecursiveRequest("/A", 4)); + const auto request = std::make_shared(ops, ACLs{}); + + storage.preprocessRequest(request, 1, 0, new_zxid); + auto responses = storage.processRequest(request, 1, new_zxid); + ops.pop_back(); + + ASSERT_EQ(responses.size(), 1); + ASSERT_TRUE(is_multi_ok(responses[0].response)); + ASSERT_FALSE(exists("/A")); + ASSERT_FALSE(exists("/A/C")); + ASSERT_FALSE(exists("/A/B")); + ASSERT_FALSE(exists("/A/B/D")); + } +} + /// INSTANTIATE_TEST_SUITE_P(CoordinationTestSuite, /// CoordinationTest, /// ::testing::ValuesIn(std::initializer_list{CompressionParam{true, ".zstd"}, CompressionParam{false, ""}})); From f8b6025e078a1cf7327154f389b0c32ffbf77a94 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Fri, 6 Sep 2024 09:43:32 +0800 Subject: [PATCH 154/215] rename to allow_experimental_join_right_table_sorting and modify comments --- src/Core/Settings.h | 6 +++--- src/Core/SettingsChangesHistory.cpp | 6 +++--- utils/check-style/experimental_settings_ignore.txt | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 9b83507ce73..88555c67e24 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -922,9 +922,9 @@ class IColumn; M(Bool, implicit_transaction, false, "If enabled and not already inside a transaction, wraps the query inside a full transaction (begin + commit or rollback)", 0) \ M(UInt64, grace_hash_join_initial_buckets, 1, "Initial number of grace hash join buckets", 0) \ M(UInt64, grace_hash_join_max_buckets, 1024, "Limit on the number of grace hash join buckets", 0) \ - M(Int32, join_to_sort_perkey_rows_threshold, 40, "Rerange the right table by key in hash join when the per-key average rows of it exceed this value (means the table keys is dense) and its number of rows is not too many(controlled by `join_to_sort_table_rows_threshold`), to make the join output by the data batch of key, which would improve performance.", 0) \ - M(Int32, join_to_sort_table_rows_threshold, 10000, "Rerange the right table by key in hash join when its number of rows not exceed this value and the table keys is dense (controlled by `join_to_sort_perkey_rows_threshold`), to make the join performance improve as output by the data batch of key, but not cost too much on the table reranging.", 0) \ - M(Bool, allow_experimental_inner_join_right_table_sorting, false, "If it is set to true, and the conditions of `join_to_sort_perkey_rows_threshold` and `join_to_sort_perkey_rows_threshold` are met, then we will try to rerange the right table by key to improve the performance in hash join.", 0) \ + M(Int32, join_to_sort_perkey_rows_threshold, 40, "Rerange the right table by key in left or inner hash join when the per-key average rows of it exceed this value (means the table keys is dense) and its number of rows is not too many(controlled by `join_to_sort_table_rows_threshold`), to make the join output by the data batch of key, which would improve performance.", 0) \ + M(Int32, join_to_sort_table_rows_threshold, 10000, "Rerange the right table by key in left or inner hash join when its number of rows not exceed this value and the table keys is dense (controlled by `join_to_sort_perkey_rows_threshold`), to make the join performance improve as output by the data batch of key, but not cost too much on the table reranging.", 0) \ + M(Bool, allow_experimental_join_right_table_sorting, false, "If it is set to true, and the conditions of `join_to_sort_perkey_rows_threshold` and `join_to_sort_perkey_rows_threshold` are met, then we will try to rerange the right table by key to improve the performance in left or inner hash join.", 0) \ M(Timezone, session_timezone, "", "This setting can be removed in the future due to potential caveats. It is experimental and is not suitable for production usage. The default timezone for current session or query. The server default timezone if empty.", 0) \ M(Bool, use_hive_partitioning, false, "Allows to use hive partitioning for File, URL, S3, AzureBlobStorage and HDFS engines.", 0)\ \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 7bac9c314e7..4ac7a0f2d8d 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -95,9 +95,9 @@ static std::initializer_list Date: Fri, 6 Sep 2024 10:15:43 +0800 Subject: [PATCH 155/215] modify test --- src/Interpreters/TableJoin.cpp | 2 +- .../queries/0_stateless/03228_join_to_rerange_right_table.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp index 8d79b88190b..519264dd0e4 100644 --- a/src/Interpreters/TableJoin.cpp +++ b/src/Interpreters/TableJoin.cpp @@ -118,7 +118,7 @@ TableJoin::TableJoin(const Settings & settings, VolumePtr tmp_volume_, Temporary , output_by_rowlist_perkey_rows_threshold(settings.join_output_by_rowlist_perkey_rows_threshold) , sort_right_perkey_rows_threshold(settings.join_to_sort_perkey_rows_threshold) , sort_right_table_rows_threshold(settings.join_to_sort_table_rows_threshold) - , allow_join_sorting(settings.allow_experimental_inner_join_right_table_sorting) + , allow_join_sorting(settings.allow_experimental_join_right_table_sorting) , max_memory_usage(settings.max_memory_usage) , tmp_volume(tmp_volume_) , tmp_data(tmp_data_) diff --git a/tests/queries/0_stateless/03228_join_to_rerange_right_table.sql b/tests/queries/0_stateless/03228_join_to_rerange_right_table.sql index e9001f92405..f3ee0f0b933 100644 --- a/tests/queries/0_stateless/03228_join_to_rerange_right_table.sql +++ b/tests/queries/0_stateless/03228_join_to_rerange_right_table.sql @@ -7,8 +7,8 @@ CREATE TABLE test_right (a Int64, b String, c LowCardinality(String)) ENGINE = M INSERT INTO test_left SELECT number % 10000, number % 10000, number % 10000 FROM numbers(100000); INSERT INTO test_right SELECT number % 10 , number % 10, number % 10 FROM numbers(10000); -SELECT MAX(test_right.a) FROM test_left INNER JOIN test_right on test_left.b = test_right.b SETTINGS allow_experimental_inner_join_right_table_sorting=true; -SELECT MAX(test_right.a) FROM test_left LEFT JOIN test_right on test_left.b = test_right.b SETTINGS allow_experimental_inner_join_right_table_sorting=true; +SELECT MAX(test_right.a) FROM test_left INNER JOIN test_right on test_left.b = test_right.b SETTINGS allow_experimental_join_right_table_sorting=true; +SELECT MAX(test_right.a) FROM test_left LEFT JOIN test_right on test_left.b = test_right.b SETTINGS allow_experimental_join_right_table_sorting=true; drop table test_left; drop table test_right; From 231a7c97ccf61b4a3511d0def3abd49469d31f3f Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Fri, 6 Sep 2024 08:34:59 +0000 Subject: [PATCH 156/215] move code --- src/Coordination/KeeperStorage.cpp | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 9378b30ea52..bb6dd04eeec 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -1654,6 +1654,13 @@ private: uint32_t nodes_observed = 1; /// root node std::unordered_map> by_level_deltas; + struct Step + { + String path; + std::variant node; + uint32_t level; + }; + public: ToDeleteTreeCollector(Storage & storage_, int64_t zxid_, uint32_t limit_) : storage(storage_) @@ -1665,11 +1672,12 @@ private: bool collect(StringRef root_path, const SNode & root_node) { std::deque steps; - steps.push_back(Step{root_path.toString(), &root_node, 0}); - if (observeNode(root_node)) + if (checkLimits(root_node)) return true; + steps.push_back(Step{root_path.toString(), &root_node, 0}); + while (!steps.empty()) { Step step = std::move(steps.front()); @@ -1708,13 +1716,6 @@ private: } private: - struct Step - { - String path; - std::variant node; - uint32_t level; - }; - bool visitRocksDBNode(std::deque & steps, StringRef root_path, uint32_t level) { if constexpr (Storage::use_rocksdb) @@ -1724,7 +1725,7 @@ private: for (auto && [child_name, node] : children) { - if (observeNode(node)) + if (checkLimits(node)) return true; auto child_path = (root_fs_path / child_name).generic_string(); @@ -1754,7 +1755,7 @@ private: auto child_it = storage.container.find(child_path); chassert(child_it != storage.container.end()); - if (observeNode(child_it->value)) + if (checkLimits(child_it->value)) return true; steps.push_back(Step{std::move(child_path), child_it->value, level + 1}); @@ -1777,7 +1778,7 @@ private: const String & path = it->first; const SNode & node = *it->second.node; - if (observeNode(node)) + if (checkLimits(node)) return true; steps.push_back(Step{path, &node, level + 1}); @@ -1806,7 +1807,7 @@ private: by_level_deltas[level].emplace_back(root_path.toString(), zxid, typename Storage::RemoveNodeDelta{root_node.version, root_node.ephemeralOwner()}); } - bool observeNode(const SNode & node) + bool checkLimits(const SNode & node) { nodes_observed += node.numChildren(); return nodes_observed > limit; From 2bbe93353187cd7aba95e6cb6d487fa553309b28 Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Fri, 6 Sep 2024 09:33:47 +0000 Subject: [PATCH 157/215] fix watches logic --- src/Coordination/KeeperStorage.cpp | 31 ++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index bb6dd04eeec..24d95a89a4d 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -1129,6 +1129,12 @@ struct KeeperStorageRequestProcessor return {}; } + virtual KeeperStorageBase::ResponsesForSessions + processWatches(const Storage & /*storage*/, int64_t /*zxid*/, KeeperStorageBase::Watches & watches, KeeperStorageBase::Watches & list_watches) const + { + return processWatches(watches, list_watches); + } + virtual bool checkAuth(Storage & /*storage*/, int64_t /*session_id*/, bool /*is_local*/) const { return true; } virtual ~KeeperStorageRequestProcessor() = default; @@ -1635,10 +1641,23 @@ struct KeeperStorageRemoveRecursiveRequestProcessor final : public KeeperStorage } KeeperStorageBase::ResponsesForSessions - processWatches(KeeperStorageBase::Watches & watches, KeeperStorageBase::Watches & list_watches) const override + processWatches(const Storage & storage, int64_t zxid, KeeperStorageBase::Watches & watches, KeeperStorageBase::Watches & list_watches) const override { - /// TODO(michicosun) rewrite - return processWatchesImpl(this->zk_request->getPath(), watches, list_watches, Coordination::Event::DELETED); + /// need to iterate over zxid deltas and update watches for deleted tree. + const auto & deltas = storage.uncommitted_state.deltas; + + KeeperStorageBase::ResponsesForSessions responses; + for (auto it = deltas.rbegin(); it != deltas.rend() && it->zxid == zxid; ++it) + { + const auto * remove_delta = std::get_if(&it->operation); + if (remove_delta) + { + auto new_responses = processWatchesImpl(it->path, watches, list_watches, Coordination::Event::DELETED); + responses.insert(responses.end(), std::make_move_iterator(new_responses.begin()), std::make_move_iterator(new_responses.end())); + } + } + + return responses; } private: @@ -2511,12 +2530,12 @@ struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestPro } KeeperStorageBase::ResponsesForSessions - processWatches(typename Storage::Watches & watches, typename Storage::Watches & list_watches) const override + processWatches(const Storage & storage, int64_t zxid, typename Storage::Watches & watches, typename Storage::Watches & list_watches) const override { typename Storage::ResponsesForSessions result; for (const auto & generic_request : concrete_requests) { - auto responses = generic_request->processWatches(watches, list_watches); + auto responses = generic_request->processWatches(storage, zxid, watches, list_watches); result.insert(result.end(), responses.begin(), responses.end()); } return result; @@ -2980,7 +2999,7 @@ KeeperStorage::ResponsesForSessions KeeperStorage::process /// If this requests processed successfully we need to check watches if (response->error == Coordination::Error::ZOK) { - auto watch_responses = request_processor->processWatches(watches, list_watches); + auto watch_responses = request_processor->processWatches(*this, zxid, watches, list_watches); results.insert(results.end(), watch_responses.begin(), watch_responses.end()); } From cf0e0b766da015d7b13f2c43b669c135daf43c14 Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Fri, 6 Sep 2024 09:34:26 +0000 Subject: [PATCH 158/215] add test for watches --- src/Coordination/tests/gtest_coordination.cpp | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index 27061b15b90..037a102082c 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -3629,6 +3629,102 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveInMultiRequest) } } +TYPED_TEST(CoordinationTest, TestRemoveRecursiveWatches) +{ + using namespace DB; + using namespace Coordination; + + using Storage = typename TestFixture::Storage; + + ChangelogDirTest rocks("./rocksdb"); + this->setRocksDBDirectory("./rocksdb"); + + Storage storage{500, "", this->keeper_context}; + int zxid = 0; + + const auto create = [&](const String & path, int create_mode) + { + int new_zxid = ++zxid; + + const auto create_request = std::make_shared(); + create_request->path = path; + create_request->is_ephemeral = create_mode == zkutil::CreateMode::Ephemeral || create_mode == zkutil::CreateMode::EphemeralSequential; + create_request->is_sequential = create_mode == zkutil::CreateMode::PersistentSequential || create_mode == zkutil::CreateMode::EphemeralSequential; + + storage.preprocessRequest(create_request, 1, 0, new_zxid); + auto responses = storage.processRequest(create_request, 1, new_zxid); + + EXPECT_EQ(responses.size(), 1); + EXPECT_EQ(responses[0].response->error, Coordination::Error::ZOK) << "Failed to create " << path; + }; + + const auto add_watch = [&](const String & path) + { + int new_zxid = ++zxid; + + const auto exists_request = std::make_shared(); + exists_request->path = path; + exists_request->has_watch = true; + + storage.preprocessRequest(exists_request, 1, 0, new_zxid); + auto responses = storage.processRequest(exists_request, 1, new_zxid); + + EXPECT_EQ(responses.size(), 1); + EXPECT_EQ(responses[0].response->error, Coordination::Error::ZOK); + }; + + const auto add_list_watch = [&](const String & path) + { + int new_zxid = ++zxid; + + const auto list_request = std::make_shared(); + list_request->path = path; + list_request->has_watch = true; + + storage.preprocessRequest(list_request, 1, 0, new_zxid); + auto responses = storage.processRequest(list_request, 1, new_zxid); + + EXPECT_EQ(responses.size(), 1); + EXPECT_EQ(responses[0].response->error, Coordination::Error::ZOK); + }; + + create("/A", zkutil::CreateMode::Persistent); + create("/A/B", zkutil::CreateMode::Persistent); + create("/A/C", zkutil::CreateMode::Ephemeral); + create("/A/B/D", zkutil::CreateMode::Ephemeral); + + add_watch("/A"); + add_watch("/A/B"); + add_watch("/A/C"); + add_watch("/A/B/D"); + add_list_watch("/A"); + add_list_watch("/A/B"); + ASSERT_EQ(storage.watches.size(), 4); + ASSERT_EQ(storage.list_watches.size(), 2); + + int new_zxid = ++zxid; + + auto remove_request = std::make_shared(); + remove_request->path = "/A"; + remove_request->remove_nodes_limit = 4; + + storage.preprocessRequest(remove_request, 1, 0, new_zxid); + auto responses = storage.processRequest(remove_request, 1, new_zxid); + + ASSERT_EQ(responses.size(), 7); + + for (size_t i = 0; i < 7; ++i) + { + ASSERT_EQ(responses[i].response->error, Coordination::Error::ZOK); + + if (const auto * watch_response = dynamic_cast(responses[i].response.get())) + ASSERT_EQ(watch_response->type, Coordination::Event::DELETED); + } + + ASSERT_EQ(storage.watches.size(), 0); + ASSERT_EQ(storage.list_watches.size(), 0); +} + /// INSTANTIATE_TEST_SUITE_P(CoordinationTestSuite, /// CoordinationTest, /// ::testing::ValuesIn(std::initializer_list{CompressionParam{true, ".zstd"}, CompressionParam{false, ""}})); From 068ada57baace782d98923de2174d50aebc16bf0 Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Fri, 6 Sep 2024 12:23:57 +0000 Subject: [PATCH 159/215] update api --- src/Common/ZooKeeper/ZooKeeper.cpp | 33 ++++++++++++++++++++---------- src/Common/ZooKeeper/ZooKeeper.h | 13 ++++++------ 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index 65c3fdba8d2..ae60520affb 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -979,8 +979,26 @@ bool ZooKeeper::tryRemoveChildrenRecursive(const std::string & path, bool probab return removed_as_expected; } -void ZooKeeper::removeRecursive(const std::string & path) // TODO(michicosun) rewrite +void ZooKeeper::removeRecursive(const std::string & path, uint32_t remove_nodes_limit) { + if (!isFeatureEnabled(DB::KeeperFeatureFlag::REMOVE_RECURSIVE)) + { + removeChildrenRecursive(path); + remove(path); + return; + } + + check(tryRemoveRecursive(path, remove_nodes_limit), path); +} + +Coordination::Error ZooKeeper::tryRemoveRecursive(const std::string & path, uint32_t remove_nodes_limit) +{ + if (!isFeatureEnabled(DB::KeeperFeatureFlag::REMOVE_RECURSIVE)) + { + tryRemoveChildrenRecursive(path); + return tryRemove(path); + } + auto promise = std::make_shared>(); auto future = promise->get_future(); @@ -989,27 +1007,20 @@ void ZooKeeper::removeRecursive(const std::string & path) // TODO(michicosun) re promise->set_value(response); }; - impl->removeRecursive(path, 100, std::move(callback)); + impl->removeRecursive(path, remove_nodes_limit, std::move(callback)); if (future.wait_for(std::chrono::milliseconds(args.operation_timeout_ms)) != std::future_status::ready) { impl->finalize(fmt::format("Operation timeout on {} {}", Coordination::OpNum::RemoveRecursive, path)); - check(Coordination::Error::ZOPERATIONTIMEOUT, path); + return Coordination::Error::ZOPERATIONTIMEOUT; } else { auto response = future.get(); - check(response.error, path); + return response.error; } } -Coordination::Error ZooKeeper::tryRemoveRecursive(const std::string & path) -{ - tryRemoveChildrenRecursive(path); - return tryRemove(path); -} - - namespace { struct WaitForDisappearState diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h index 3b94e6004cb..29c4fbc9b74 100644 --- a/src/Common/ZooKeeper/ZooKeeper.h +++ b/src/Common/ZooKeeper/ZooKeeper.h @@ -479,15 +479,16 @@ public: Int64 getClientID(); - /// Remove the node with the subtree. If someone concurrently adds or removes a node - /// in the subtree, the result is undefined. - void removeRecursive(const std::string & path); + /// Remove the node with the subtree. + /// If Keeper supports RemoveRecursive operation then it will be performed atomically. + /// Otherwise if someone concurrently adds or removes a node in the subtree, the result is undefined. + void removeRecursive(const std::string & path, uint32_t remove_nodes_limit = 100); - /// Remove the node with the subtree. If someone concurrently removes a node in the subtree, - /// this will not cause errors. + /// Same as removeRecursive but in case if Keeper does not supports RemoveRecursive and + /// if someone concurrently removes a node in the subtree, this will not cause errors. /// For instance, you can call this method twice concurrently for the same node and the end /// result would be the same as for the single call. - Coordination::Error tryRemoveRecursive(const std::string & path); + Coordination::Error tryRemoveRecursive(const std::string & path, uint32_t remove_nodes_limit = 100); /// Similar to removeRecursive(...) and tryRemoveRecursive(...), but does not remove path itself. /// Node defined as RemoveException will not be deleted. From 81972b97e7bb555b4dc4672fdcb765dfb4b5bd31 Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Fri, 6 Sep 2024 13:23:57 +0000 Subject: [PATCH 160/215] support test keeper --- src/Common/ZooKeeper/TestKeeper.cpp | 100 ++++++++++++++++++++++++++-- 1 file changed, 96 insertions(+), 4 deletions(-) diff --git a/src/Common/ZooKeeper/TestKeeper.cpp b/src/Common/ZooKeeper/TestKeeper.cpp index 1ac085d6050..ae8d95b806a 100644 --- a/src/Common/ZooKeeper/TestKeeper.cpp +++ b/src/Common/ZooKeeper/TestKeeper.cpp @@ -90,6 +90,36 @@ struct TestKeeperRemoveRequest final : RemoveRequest, TestKeeperRequest } }; +struct TestKeeperRemoveRecursiveRequest final : RemoveRecursiveRequest, TestKeeperRequest +{ + TestKeeperRemoveRecursiveRequest() = default; + explicit TestKeeperRemoveRecursiveRequest(const RemoveRecursiveRequest & base) : RemoveRecursiveRequest(base) {} + ResponsePtr createResponse() const override; + std::pair process(TestKeeper::Container & container, int64_t zxid) const override; + + void processWatches(TestKeeper::Watches & node_watches, TestKeeper::Watches & list_watches) const override + { + std::vector> deleted; + + auto add_deleted_watches = [&](TestKeeper::Watches & w) + { + for (const auto & [watch_path, _] : w) + if (watch_path.starts_with(path)) + deleted.emplace_back(watch_path, std::count(watch_path.begin(), watch_path.end(), '/')); + }; + + add_deleted_watches(node_watches); + add_deleted_watches(list_watches); + std::sort(deleted.begin(), deleted.end(), [](const auto & lhs, const auto & rhs) + { + return lhs.second < rhs.second; + }); + + for (const auto & [watch_path, _] : deleted) + processWatchesImpl(watch_path, node_watches, list_watches); + } +}; + struct TestKeeperExistsRequest final : ExistsRequest, TestKeeperRequest { ResponsePtr createResponse() const override; @@ -175,6 +205,10 @@ struct TestKeeperMultiRequest final : MultiRequest, TestKeeperRequest { requests.push_back(std::make_shared(*concrete_request_remove)); } + else if (const auto * concrete_request_remove_recursive = dynamic_cast(generic_request.get())) + { + requests.push_back(std::make_shared(*concrete_request_remove_recursive)); + } else if (const auto * concrete_request_set = dynamic_cast(generic_request.get())) { requests.push_back(std::make_shared(*concrete_request_set)); @@ -313,6 +347,56 @@ std::pair TestKeeperRemoveRequest::process(TestKeeper::Contai return { std::make_shared(response), undo }; } +std::pair TestKeeperRemoveRecursiveRequest::process(TestKeeper::Container & container, int64_t zxid) const +{ + RemoveRecursiveResponse response; + response.zxid = zxid; + Undo undo; + + auto root_it = container.find(path); + if (root_it == container.end()) + { + response.error = Error::ZNONODE; + return { std::make_shared(response), undo }; + } + + std::vector> children; + + for (const auto & [child_path, child_node] : container) + if (child_path.starts_with(path)) + children.emplace_back(child_path, child_node); + + if (children.size() > remove_nodes_limit) + { + response.error = Error::ZNOTEMPTY; + return { std::make_shared(response), undo }; + } + + auto & parent = container.at(parentPath(path)); + --parent.stat.numChildren; + ++parent.stat.cversion; + + for (const auto & [child_path, child_node] : children) + { + auto child_it = container.find(child_path); + chassert(child_it != container.end()); + container.erase(child_it); + } + + response.error = Error::ZOK; + undo = [&container, dead = std::move(children), root_path = path]() + { + for (auto && [child_path, child_node] : dead) + container.emplace(child_path, child_node); + + auto & undo_parent = container.at(parentPath(root_path)); + ++undo_parent.stat.numChildren; + --undo_parent.stat.cversion; + }; + + return { std::make_shared(response), undo }; +} + std::pair TestKeeperExistsRequest::process(TestKeeper::Container & container, int64_t zxid) const { ExistsResponse response; @@ -530,6 +614,7 @@ std::pair TestKeeperMultiRequest::process(TestKeeper::Contain ResponsePtr TestKeeperCreateRequest::createResponse() const { return std::make_shared(); } ResponsePtr TestKeeperRemoveRequest::createResponse() const { return std::make_shared(); } +ResponsePtr TestKeeperRemoveRecursiveRequest::createResponse() const { return std::make_shared(); } ResponsePtr TestKeeperExistsRequest::createResponse() const { return std::make_shared(); } ResponsePtr TestKeeperGetRequest::createResponse() const { return std::make_shared(); } ResponsePtr TestKeeperSetRequest::createResponse() const { return std::make_shared(); } @@ -772,11 +857,18 @@ void TestKeeper::remove( } void TestKeeper::removeRecursive( - [[maybe_unused]] const String & path, - [[maybe_unused]] uint32_t remove_nodes_limit, - [[maybe_unused]] RemoveRecursiveCallback callback) + const String & path, + uint32_t remove_nodes_limit, + RemoveRecursiveCallback callback) { - /// TODO(michicosun) implement + TestKeeperRemoveRecursiveRequest request; + request.path = path; + request.remove_nodes_limit = remove_nodes_limit; + + RequestInfo request_info; + request_info.request = std::make_shared(std::move(request)); + request_info.callback = [callback](const Response & response) { callback(dynamic_cast(response)); }; + pushRequest(std::move(request_info)); } void TestKeeper::exists( From ebd9c294aa32db185e280cba7407f7e6ef2730d0 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 6 Sep 2024 20:09:48 +0000 Subject: [PATCH 161/215] Fix build --- src/Processors/QueryPlan/ReadFromMergeTree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index f9accaaec57..218f0a61a48 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -355,7 +355,7 @@ std::unique_ptr ReadFromMergeTree::createLocalParallelReplica const bool enable_parallel_reading = true; return std::make_unique( prepared_parts, - alter_conversions_for_parts, + mutations_snapshot, all_column_names, data, getQueryInfo(), From 6309f36232a28e3455ac8d057513983caa499a61 Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Mon, 9 Sep 2024 09:44:19 +0000 Subject: [PATCH 162/215] add limit option to rmr command --- programs/keeper-client/Commands.cpp | 11 ++++++++++- programs/keeper-client/Commands.h | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/programs/keeper-client/Commands.cpp b/programs/keeper-client/Commands.cpp index 4ad2eb31e6d..b4a5329b01f 100644 --- a/programs/keeper-client/Commands.cpp +++ b/programs/keeper-client/Commands.cpp @@ -506,14 +506,23 @@ bool RMRCommand::parse(IParser::Pos & pos, std::shared_ptr & nod return false; node->args.push_back(std::move(path)); + ASTPtr remove_nodes_limit; + if (ParserUnsignedInteger{}.parse(pos, remove_nodes_limit, expected)) + node->args.push_back(remove_nodes_limit->as().value); + else + node->args.push_back(UInt64(100)); + return true; } void RMRCommand::execute(const ASTKeeperQuery * query, KeeperClient * client) const { String path = client->getAbsolutePath(query->args[0].safeGet()); + UInt64 remove_nodes_limit = query->args[1].safeGet(); + client->askConfirmation( - "You are going to recursively delete path " + path, [client, path] { client->zookeeper->removeRecursive(path); }); + "You are going to recursively delete path " + path, + [client, path, remove_nodes_limit] { client->zookeeper->removeRecursive(path, static_cast(remove_nodes_limit)); }); } bool ReconfigCommand::parse(IParser::Pos & pos, std::shared_ptr & node, DB::Expected & expected) const diff --git a/programs/keeper-client/Commands.h b/programs/keeper-client/Commands.h index 686a752b6b6..da577ce1e65 100644 --- a/programs/keeper-client/Commands.h +++ b/programs/keeper-client/Commands.h @@ -184,7 +184,7 @@ class RMRCommand : public IKeeperClientCommand void execute(const ASTKeeperQuery * query, KeeperClient * client) const override; - String getHelpMessage() const override { return "{} -- Recursively deletes path. Confirmation required"; } + String getHelpMessage() const override { return "{} [limit] -- Recursively deletes path if the subtree size is smaller than the limit. Confirmation required (default limit = 100)"; } }; class ReconfigCommand : public IKeeperClientCommand From 08368d1d65a8fc5f6caff986d17c4de438d18114 Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Mon, 9 Sep 2024 09:46:17 +0000 Subject: [PATCH 163/215] update keeper client docs --- docs/en/operations/utilities/clickhouse-keeper-client.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/utilities/clickhouse-keeper-client.md b/docs/en/operations/utilities/clickhouse-keeper-client.md index a66ecbc1372..2cd7be99dfb 100644 --- a/docs/en/operations/utilities/clickhouse-keeper-client.md +++ b/docs/en/operations/utilities/clickhouse-keeper-client.md @@ -53,7 +53,7 @@ keeper foo bar - `touch ''` -- Creates new node with an empty string as value. Doesn't throw an exception if the node already exists - `get ''` -- Returns the node's value - `rm '' [version]` -- Removes the node only if version matches (default: -1) -- `rmr ''` -- Recursively deletes path. Confirmation required +- `rmr '' [limit]` -- Recursively deletes path if the subtree size is smaller than the limit. Confirmation required (default limit = 100) - `flwc ` -- Executes four-letter-word command - `help` -- Prints this message - `get_direct_children_number '[path]'` -- Get numbers of direct children nodes under a specific path From 7d042be8eb50e5492706773035e683d92454678d Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Mon, 9 Sep 2024 13:21:00 +0200 Subject: [PATCH 164/215] A simple interface to find temporary files by their logical names when building pipeline from query plan --- .../QueryPlan/BuildQueryPipelineSettings.h | 3 ++ src/Processors/QueryPlan/TemporaryFiles.h | 31 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 src/Processors/QueryPlan/TemporaryFiles.h diff --git a/src/Processors/QueryPlan/BuildQueryPipelineSettings.h b/src/Processors/QueryPlan/BuildQueryPipelineSettings.h index 3b5e4e06953..d99f9a7d1f1 100644 --- a/src/Processors/QueryPlan/BuildQueryPipelineSettings.h +++ b/src/Processors/QueryPlan/BuildQueryPipelineSettings.h @@ -12,12 +12,15 @@ namespace DB struct Settings; class QueryStatus; using QueryStatusPtr = std::shared_ptr; +struct ITemporaryFileLookup; +using TemporaryFileLookupPtr = std::shared_ptr; struct BuildQueryPipelineSettings { ExpressionActionsSettings actions_settings; QueryStatusPtr process_list_element; ProgressCallback progress_callback = nullptr; + TemporaryFileLookupPtr temporary_file_lookup; const ExpressionActionsSettings & getActionsSettings() const { return actions_settings; } static BuildQueryPipelineSettings fromContext(ContextPtr from); diff --git a/src/Processors/QueryPlan/TemporaryFiles.h b/src/Processors/QueryPlan/TemporaryFiles.h new file mode 100644 index 00000000000..943a2c2b0a4 --- /dev/null +++ b/src/Processors/QueryPlan/TemporaryFiles.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ + +class WriteBuffer; +class ReadBuffer; + +/// Interface for accessing temporary files by some logical name (or id). +/// While building query pipeline processors can lookup temporary files by some id and use them for writing and/or reading temporary data +/// without knowing what exactly is behind the name: local file, memory buffer, object in cloud storage, etc. +struct ITemporaryFileLookup : boost::noncopyable +{ + virtual ~ITemporaryFileLookup() = default; + + /// Give the caller a temporary write buffer, but don't give away the ownership. + virtual WriteBuffer & getTemporaryFileForWriting(const String & file_id) = 0; + + /// Give the caller a temporary read buffer, it exclusively belongs to the caller. + /// Other callers can get their own read buffer for the same temporary file. + virtual std::unique_ptr getTemporaryFileForReading(const String & file_id) = 0; +}; + +using TemporaryFileLookupPtr = std::shared_ptr; + +} From 5a8bd5b4f561cbfb2e40d483ee75f8a90225b85a Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Mon, 9 Sep 2024 13:22:38 +0200 Subject: [PATCH 165/215] Encapsulte and cleanup "rows_sources" temp file usage --- .../Transforms/ColumnGathererTransform.cpp | 5 +- .../Transforms/ColumnGathererTransform.h | 4 +- src/Storages/MergeTree/MergeTask.cpp | 143 +++++++++++------- src/Storages/MergeTree/MergeTask.h | 10 +- 4 files changed, 98 insertions(+), 64 deletions(-) diff --git a/src/Processors/Transforms/ColumnGathererTransform.cpp b/src/Processors/Transforms/ColumnGathererTransform.cpp index 52fa42fdb51..f266d5c2e2f 100644 --- a/src/Processors/Transforms/ColumnGathererTransform.cpp +++ b/src/Processors/Transforms/ColumnGathererTransform.cpp @@ -183,13 +183,14 @@ void ColumnGathererStream::consume(Input & input, size_t source_num) ColumnGathererTransform::ColumnGathererTransform( const Block & header, size_t num_inputs, - ReadBuffer & row_sources_buf_, + std::unique_ptr row_sources_buf_, size_t block_preferred_size_rows_, size_t block_preferred_size_bytes_, bool is_result_sparse_) : IMergingTransform( num_inputs, header, header, /*have_all_inputs_=*/ true, /*limit_hint_=*/ 0, /*always_read_till_end_=*/ false, - num_inputs, row_sources_buf_, block_preferred_size_rows_, block_preferred_size_bytes_, is_result_sparse_) + num_inputs, *row_sources_buf_, block_preferred_size_rows_, block_preferred_size_bytes_, is_result_sparse_) + , row_sources_buf_holder(std::move(row_sources_buf_)) , log(getLogger("ColumnGathererStream")) { if (header.columns() != 1) diff --git a/src/Processors/Transforms/ColumnGathererTransform.h b/src/Processors/Transforms/ColumnGathererTransform.h index fbc9a6bfcc6..ce2671ce0bf 100644 --- a/src/Processors/Transforms/ColumnGathererTransform.h +++ b/src/Processors/Transforms/ColumnGathererTransform.h @@ -115,7 +115,7 @@ public: ColumnGathererTransform( const Block & header, size_t num_inputs, - ReadBuffer & row_sources_buf_, + std::unique_ptr row_sources_buf_, size_t block_preferred_size_rows_, size_t block_preferred_size_bytes_, bool is_result_sparse_); @@ -124,6 +124,8 @@ public: protected: void onFinish() override; + + std::unique_ptr row_sources_buf_holder; /// Keep ownership of row_sources_buf while it's in use by ColumnGathererStream. LoggerPtr log; }; diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 398a9472456..8c1dca7ac10 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -90,6 +91,62 @@ static ColumnsStatistics getStatisticsForColumns( return all_statistics; } +/// Manages the "rows_sources" temporary file that is used during vertical merge. +class RowsSourcesTemporaryFile : public ITemporaryFileLookup +{ +public: + /// A logical name of the temporary file under which it will be known to the plan steps that use it. + static constexpr auto FILE_ID = "rows_sources"; + + explicit RowsSourcesTemporaryFile(TemporaryDataOnDiskScopePtr temporary_data_on_disk_) + : tmp_disk(std::make_unique(temporary_data_on_disk_)) + , uncompressed_write_buffer(tmp_disk->createRawStream()) + , tmp_file_name_on_disk(uncompressed_write_buffer->getFileName()) + { + } + + WriteBuffer & getTemporaryFileForWriting(const String & name) override + { + if (name != FILE_ID) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected temporary file name requested: {}", name); + + if (write_buffer) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Temporary file was already requested for writing, there musto be only one writer"); + + write_buffer = (std::make_unique(*uncompressed_write_buffer)); + return *write_buffer; + } + + std::unique_ptr getTemporaryFileForReading(const String & name) override + { + if (name != FILE_ID) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected temporary file name requested: {}", name); + + if (!finalized) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Temporary file is not finalized yet"); + + /// Reopen the file for each read so that multiple reads can be performed in parallel and there is no need to seek to the beginning. + auto raw_file_read_buffer = std::make_unique(tmp_file_name_on_disk); + return std::make_unique(std::move(raw_file_read_buffer)); + } + + /// Returns written data size in bytes + size_t finalizeWriting() + { + write_buffer->finalize(); + uncompressed_write_buffer->finalize(); + finalized = true; + return write_buffer->count(); + } + +private: + std::unique_ptr tmp_disk; + std::unique_ptr uncompressed_write_buffer; + std::unique_ptr write_buffer; + const String tmp_file_name_on_disk; + bool finalized = false; +}; + static void addMissedColumnsToSerializationInfos( size_t num_rows_in_parts, const Names & part_columns, @@ -364,8 +421,6 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const ctx->compression_codec = global_ctx->data->getCompressionCodecForPart( global_ctx->merge_list_element_ptr->total_size_bytes_compressed, global_ctx->new_data_part->ttl_infos, global_ctx->time_of_merge); - ctx->tmp_disk = std::make_unique(global_ctx->context->getTempDataOnDisk()); - switch (global_ctx->chosen_merge_algorithm) { case MergeAlgorithm::Horizontal: @@ -378,8 +433,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const } case MergeAlgorithm::Vertical: { - ctx->rows_sources_uncompressed_write_buf = ctx->tmp_disk->createRawStream(); - ctx->rows_sources_write_buf = std::make_unique(*ctx->rows_sources_uncompressed_write_buf); + ctx->rows_sources_temporary_file = std::make_shared(global_ctx->context->getTempDataOnDisk()); std::map local_merged_column_to_size; for (const auto & part : global_ctx->future_part->parts) @@ -499,11 +553,9 @@ MergeTask::StageRuntimeContextPtr MergeTask::ExecuteAndFinalizeHorizontalPart::g auto new_ctx = std::make_shared(); - new_ctx->rows_sources_write_buf = std::move(ctx->rows_sources_write_buf); - new_ctx->rows_sources_uncompressed_write_buf = std::move(ctx->rows_sources_uncompressed_write_buf); + new_ctx->rows_sources_temporary_file = std::move(ctx->rows_sources_temporary_file); new_ctx->column_sizes = std::move(ctx->column_sizes); new_ctx->compression_codec = std::move(ctx->compression_codec); - new_ctx->tmp_disk = std::move(ctx->tmp_disk); new_ctx->it_name_and_type = std::move(ctx->it_name_and_type); new_ctx->read_with_direct_io = std::move(ctx->read_with_direct_io); new_ctx->need_sync = std::move(ctx->need_sync); @@ -760,11 +812,7 @@ bool MergeTask::VerticalMergeStage::prepareVerticalMergeForAllColumns() const global_ctx->merge_list_element_ptr->progress.store(ctx->column_sizes->keyColumnsWeight(), std::memory_order_relaxed); /// Ensure data has written to disk. - ctx->rows_sources_write_buf->finalize(); - ctx->rows_sources_uncompressed_write_buf->finalize(); - ctx->rows_sources_uncompressed_write_buf->finalize(); - - size_t rows_sources_count = ctx->rows_sources_write_buf->count(); + size_t rows_sources_count = ctx->rows_sources_temporary_file->finalizeWriting(); /// In special case, when there is only one source part, and no rows were skipped, we may have /// skipped writing rows_sources file. Otherwise rows_sources_count must be equal to the total /// number of input rows. @@ -775,29 +823,6 @@ bool MergeTask::VerticalMergeStage::prepareVerticalMergeForAllColumns() const "of bytes written to rows_sources file ({}). It is a bug.", sum_input_rows_exact, input_rows_filtered, rows_sources_count); - /// TemporaryDataOnDisk::createRawStream returns WriteBufferFromFile implementing IReadableWriteBuffer - /// and we expect to get ReadBufferFromFile here. - /// So, it's relatively safe to use dynamic_cast here and downcast to ReadBufferFromFile. - auto * wbuf_readable = dynamic_cast(ctx->rows_sources_uncompressed_write_buf.get()); - std::unique_ptr reread_buf = wbuf_readable ? wbuf_readable->tryGetReadBuffer() : nullptr; - if (!reread_buf) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot read temporary file {}", ctx->rows_sources_uncompressed_write_buf->getFileName()); - - auto * reread_buffer_raw = dynamic_cast(reread_buf.get()); - if (!reread_buffer_raw) - { - const auto & reread_buf_ref = *reread_buf; - throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected ReadBufferFromFileBase, but got {}", demangle(typeid(reread_buf_ref).name())); - } - /// Move ownership from std::unique_ptr to std::unique_ptr for CompressedReadBufferFromFile. - /// First, release ownership from unique_ptr to base type. - reread_buf.release(); /// NOLINT(bugprone-unused-return-value,hicpp-ignored-remove-result): we already have the pointer value in `reread_buffer_raw` - - /// Then, move ownership to unique_ptr to concrete type. - std::unique_ptr reread_buffer_from_file(reread_buffer_raw); - - /// CompressedReadBufferFromFile expects std::unique_ptr as argument. - ctx->rows_sources_read_buf = std::make_unique(std::move(reread_buffer_from_file)); ctx->it_name_and_type = global_ctx->gathering_columns.cbegin(); const auto & settings = global_ctx->context->getSettingsRef(); @@ -828,12 +853,12 @@ class ColumnGathererStep : public ITransformingStep public: ColumnGathererStep( const DataStream & input_stream_, - CompressedReadBufferFromFile * rows_sources_read_buf_, + const String & rows_sources_temporary_file_name_, UInt64 merge_block_size_rows_, UInt64 merge_block_size_bytes_, bool is_result_sparse_) : ITransformingStep(input_stream_, input_stream_.header, getTraits()) - , rows_sources_read_buf(rows_sources_read_buf_) + , rows_sources_temporary_file_name(rows_sources_temporary_file_name_) , merge_block_size_rows(merge_block_size_rows_) , merge_block_size_bytes(merge_block_size_bytes_) , is_result_sparse(is_result_sparse_) @@ -841,17 +866,20 @@ public: String getName() const override { return "ColumnGatherer"; } - void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & /*pipelineSettings*/) override + void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & pipeline_settings) override { const auto &header = pipeline.getHeader(); const auto input_streams_count = pipeline.getNumStreams(); - rows_sources_read_buf->seek(0, 0); + if (!pipeline_settings.temporary_file_lookup) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Temporary file lookup is not set in pipeline settings for vertical merge"); + + auto rows_sources_read_buf = pipeline_settings.temporary_file_lookup->getTemporaryFileForReading(rows_sources_temporary_file_name); auto transform = std::make_unique( header, input_streams_count, - *rows_sources_read_buf, + std::move(rows_sources_read_buf), merge_block_size_rows, merge_block_size_bytes, is_result_sparse); @@ -881,7 +909,7 @@ private: } MergeTreeData::MergingParams merging_params{}; - CompressedReadBufferFromFile * rows_sources_read_buf; + const String rows_sources_temporary_file_name; const UInt64 merge_block_size_rows; const UInt64 merge_block_size_bytes; const bool is_result_sparse; @@ -932,7 +960,7 @@ MergeTask::VerticalMergeRuntimeContext::PreparedColumnPipeline MergeTask::Vertic const auto data_settings = global_ctx->data->getSettings(); auto merge_step = std::make_unique( merge_column_query_plan.getCurrentDataStream(), - ctx->rows_sources_read_buf.get(), //global_ctx->rows_sources_temporary_file_name, + RowsSourcesTemporaryFile::FILE_ID, data_settings->merge_max_block_size, data_settings->merge_max_block_size_bytes, is_result_sparse); @@ -960,7 +988,8 @@ MergeTask::VerticalMergeRuntimeContext::PreparedColumnPipeline MergeTask::Vertic } auto pipeline_settings = BuildQueryPipelineSettings::fromContext(global_ctx->context); - auto optimization_settings = QueryPlanOptimizationSettings::fromContext(global_ctx->context); + pipeline_settings.temporary_file_lookup = ctx->rows_sources_temporary_file; + auto optimization_settings = QueryPlanOptimizationSettings::fromContext(global_ctx->context); auto builder = merge_column_query_plan.buildQueryPipeline(optimization_settings, pipeline_settings); return {QueryPipelineBuilder::getPipeline(std::move(*builder)), std::move(indexes_to_recalc)}; @@ -1012,10 +1041,6 @@ void MergeTask::VerticalMergeStage::prepareVerticalMergeForOneColumn() const global_ctx->to->getIndexGranularity()); ctx->column_elems_written = 0; - - /// rows_sources_read_buf is reused for each column so we need to rewind it explicitly each time - /// This sharing also prevents from from running multiple merge of individual columns in parallel. - ctx->rows_sources_read_buf->seek(0, 0); } @@ -1329,7 +1354,7 @@ public: const SortDescription & sort_description_, const Names partition_key_columns_, const MergeTreeData::MergingParams & merging_params_, - WriteBuffer * rows_sources_write_buf_, + const String & rows_sources_temporary_file_name_, UInt64 merge_block_size_rows_, UInt64 merge_block_size_bytes_, bool blocks_are_granules_size_, @@ -1339,7 +1364,7 @@ public: , sort_description(sort_description_) , partition_key_columns(partition_key_columns_) , merging_params(merging_params_) - , rows_sources_write_buf(rows_sources_write_buf_) + , rows_sources_temporary_file_name(rows_sources_temporary_file_name_) , merge_block_size_rows(merge_block_size_rows_) , merge_block_size_bytes(merge_block_size_bytes_) , blocks_are_granules_size(blocks_are_granules_size_) @@ -1349,7 +1374,7 @@ public: String getName() const override { return "ApplyMergePolicy"; } - void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & /*pipelineSettings*/) override + void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & pipeline_settings) override { /// The order of the streams is important: when the key is matched, the elements go in the order of the source stream number. /// In the merged part, the lines with the same key must be in the ascending order of the identifier of original part, @@ -1359,6 +1384,14 @@ public: const auto &header = pipeline.getHeader(); const auto input_streams_count = pipeline.getNumStreams(); + WriteBuffer * rows_sources_write_buf = nullptr; + if (!rows_sources_temporary_file_name.empty()) + { + if (!pipeline_settings.temporary_file_lookup) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Temporary file lookup is not set in pipeline settings for vertical merge"); + rows_sources_write_buf = &pipeline_settings.temporary_file_lookup->getTemporaryFileForWriting(rows_sources_temporary_file_name); + } + switch (merging_params.mode) { case MergeTreeData::MergingParams::Ordinary: @@ -1448,7 +1481,7 @@ private: const SortDescription sort_description; const Names partition_key_columns; const MergeTreeData::MergingParams merging_params{}; - WriteBuffer * rows_sources_write_buf; + const String rows_sources_temporary_file_name; const UInt64 merge_block_size_rows; const UInt64 merge_block_size_bytes; const bool blocks_are_granules_size; @@ -1645,8 +1678,9 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const for (size_t i = 0; i < sort_columns_size; ++i) sort_description.emplace_back(sort_columns[i], 1, 1); + const bool is_vertical_merge = (global_ctx->chosen_merge_algorithm == MergeAlgorithm::Vertical); /// If merge is vertical we cannot calculate it - ctx->blocks_are_granules_size = (global_ctx->chosen_merge_algorithm == MergeAlgorithm::Vertical); + ctx->blocks_are_granules_size = is_vertical_merge; if (global_ctx->cleanup && !data_settings->allow_experimental_replacing_merge_with_cleanup) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental merges with CLEANUP are not allowed"); @@ -1656,7 +1690,7 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const sort_description, partition_key_columns, ctx->merging_params, - ctx->rows_sources_write_buf.get(), + (is_vertical_merge ? RowsSourcesTemporaryFile::FILE_ID : ""), /// rows_sources temporaty file is used only for vertical merge data_settings->merge_max_block_size, data_settings->merge_max_block_size_bytes, ctx->blocks_are_granules_size, @@ -1722,7 +1756,8 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const { auto pipeline_settings = BuildQueryPipelineSettings::fromContext(global_ctx->context); - auto optimization_settings = QueryPlanOptimizationSettings::fromContext(global_ctx->context); + pipeline_settings.temporary_file_lookup = ctx->rows_sources_temporary_file; + auto optimization_settings = QueryPlanOptimizationSettings::fromContext(global_ctx->context); auto builder = merge_parts_query_plan.buildQueryPipeline(optimization_settings, pipeline_settings); global_ctx->merged_pipeline = QueryPipelineBuilder::getPipeline(std::move(*builder)); diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index 84f00d521fd..ef187e1de2f 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -40,6 +40,7 @@ namespace DB class MergeTask; using MergeTaskPtr = std::shared_ptr; +class RowsSourcesTemporaryFile; /** * Overview of the merge algorithm @@ -227,14 +228,12 @@ private: bool need_prefix; MergeTreeData::MergingParams merging_params{}; - TemporaryDataOnDiskPtr tmp_disk{nullptr}; DiskPtr disk{nullptr}; bool need_remove_expired_values{false}; bool force_ttl{false}; CompressionCodecPtr compression_codec{nullptr}; size_t sum_input_rows_upper_bound{0}; - std::unique_ptr rows_sources_uncompressed_write_buf{nullptr}; - std::unique_ptr rows_sources_write_buf{nullptr}; + std::shared_ptr rows_sources_temporary_file; std::optional column_sizes{}; /// For projections to rebuild @@ -314,11 +313,9 @@ private: struct VerticalMergeRuntimeContext : public IStageRuntimeContext { /// Begin dependencies from previous stage - std::unique_ptr rows_sources_uncompressed_write_buf{nullptr}; - std::unique_ptr rows_sources_write_buf{nullptr}; + std::shared_ptr rows_sources_temporary_file; std::optional column_sizes; CompressionCodecPtr compression_codec; - TemporaryDataOnDiskPtr tmp_disk{nullptr}; std::list::const_iterator it_name_and_type; bool read_with_direct_io{false}; bool need_sync{false}; @@ -350,7 +347,6 @@ private: size_t column_elems_written{0}; QueryPipeline column_parts_pipeline; std::unique_ptr executor; - std::unique_ptr rows_sources_read_buf{nullptr}; UInt64 elapsed_execute_ns{0}; }; From 1bcc4ba823805bed282133fb7035b73598641fc6 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Mon, 9 Sep 2024 15:30:19 +0200 Subject: [PATCH 166/215] Renamed ApplyMergeStep into MergePartsStep --- src/Storages/MergeTree/MergeTask.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 398a9472456..3ca909a2d09 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -1321,10 +1321,10 @@ bool MergeTask::execute() /// Apply merge strategy (Ordinary, Colapsing, Aggregating, etc) to the stream -class ApplyMergeStep : public ITransformingStep +class MergePartsStep : public ITransformingStep { public: - ApplyMergeStep( + MergePartsStep( const DataStream & input_stream_, const SortDescription & sort_description_, const Names partition_key_columns_, @@ -1347,7 +1347,7 @@ public: , time_of_merge(time_of_merge_) {} - String getName() const override { return "ApplyMergePolicy"; } + String getName() const override { return "MergeParts"; } void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & /*pipelineSettings*/) override { @@ -1651,7 +1651,7 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const if (global_ctx->cleanup && !data_settings->allow_experimental_replacing_merge_with_cleanup) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental merges with CLEANUP are not allowed"); - auto merge_step = std::make_unique( + auto merge_step = std::make_unique( merge_parts_query_plan.getCurrentDataStream(), sort_description, partition_key_columns, From 8c1f434b1ac2c9fbb83561a43a6ee10f20d81974 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Mon, 9 Sep 2024 15:31:43 +0200 Subject: [PATCH 167/215] Do column materialization using ActionsDAG::addMaterializingOutputActions instead of a special step --- src/Storages/MergeTree/MergeTask.cpp | 42 +--------------------------- 1 file changed, 1 insertion(+), 41 deletions(-) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 3ca909a2d09..33cdff10b6a 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -1456,45 +1456,6 @@ private: const time_t time_of_merge{0}; }; - -class MaterializingStep : public ITransformingStep -{ -public: - explicit MaterializingStep( - const DataStream & input_stream_) - : ITransformingStep(input_stream_, input_stream_.header, getTraits()) - {} - - String getName() const override { return "Materializing"; } - - void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override - { - pipeline.addTransform(std::make_shared(input_streams.front().header)); - } - - void updateOutputStream() override - { - output_stream = createOutputStream(input_streams.front(), input_streams.front().header, getDataStreamTraits()); - } - -private: - static Traits getTraits() - { - return ITransformingStep::Traits - { - { - .returns_single_stream = true, - .preserves_number_of_streams = true, - .preserves_sorting = true, - }, - { - .preserves_number_of_rows = true, - } - }; - } -}; - - class TTLStep : public ITransformingStep { public: @@ -1709,12 +1670,11 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const if (!global_ctx->merging_skip_indexes.empty()) { auto indices_expression_dag = global_ctx->merging_skip_indexes.getSingleExpressionForIndices(global_ctx->metadata_snapshot->getColumns(), global_ctx->data->getContext())->getActionsDAG().clone(); + indices_expression_dag.addMaterializingOutputActions(); /// Const columns cannot be written without materialization. auto calculate_indices_expression_step = std::make_unique( merge_parts_query_plan.getCurrentDataStream(), std::move(indices_expression_dag)); merge_parts_query_plan.addStep(std::move(calculate_indices_expression_step)); - /// TODO: what is the purpose of MaterializingTransform in the original code? - merge_parts_query_plan.addStep(std::make_unique(merge_parts_query_plan.getCurrentDataStream())); } if (!subqueries.empty()) From 4da1e10ac66059b47a89e69327ea79d487e2847f Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Mon, 9 Sep 2024 16:01:00 +0200 Subject: [PATCH 168/215] Move sorting key calculation step outside the loop --- src/Storages/MergeTree/MergeTask.cpp | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 33cdff10b6a..9a1e749734c 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -1540,8 +1540,6 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const global_ctx->horizontal_stage_progress = std::make_unique( ctx->column_sizes ? ctx->column_sizes->keyColumnsWeight() : 1.0); - auto sorting_key_expression_dag = global_ctx->metadata_snapshot->getSortingKey().expression->getActionsDAG().clone(); - /// Read from all parts std::vector plans; for (size_t i = 0; i < global_ctx->future_part->parts.size(); ++i) @@ -1566,15 +1564,6 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const global_ctx->context, ctx->log); - if (global_ctx->metadata_snapshot->hasSortingKey()) - { - /// Calculate sorting key expressions so that they are available for merge sorting. - auto calculate_sorting_key_expression_step = std::make_unique( - plan_for_part->getCurrentDataStream(), - sorting_key_expression_dag.clone()); /// TODO: can we avoid cloning here? - plan_for_part->addStep(std::move(calculate_sorting_key_expression_step)); - } - plans.emplace_back(std::move(plan_for_part)); } @@ -1591,6 +1580,16 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const merge_parts_query_plan.unitePlans(std::move(union_step), std::move(plans)); } + if (global_ctx->metadata_snapshot->hasSortingKey()) + { + /// Calculate sorting key expressions so that they are available for merge sorting. + auto sorting_key_expression_dag = global_ctx->metadata_snapshot->getSortingKey().expression->getActionsDAG().clone(); + auto calculate_sorting_key_expression_step = std::make_unique( + merge_parts_query_plan.getCurrentDataStream(), + std::move(sorting_key_expression_dag)); + merge_parts_query_plan.addStep(std::move(calculate_sorting_key_expression_step)); + } + /// Merge { Names sort_columns = global_ctx->metadata_snapshot->getSortingKeyColumns(); From 38b8edae971b26fbb6c2201d727450cc1e1ed923 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Mon, 9 Sep 2024 19:34:59 +0200 Subject: [PATCH 169/215] Fix the case with empty temp file --- src/Storages/MergeTree/MergeTask.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 8c1dca7ac10..37eb94f269b 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include @@ -125,6 +125,10 @@ public: if (!finalized) throw Exception(ErrorCodes::LOGICAL_ERROR, "Temporary file is not finalized yet"); + /// tmp_disk might not create real file if no data was written to it. + if (final_size == 0) + return std::make_unique(); + /// Reopen the file for each read so that multiple reads can be performed in parallel and there is no need to seek to the beginning. auto raw_file_read_buffer = std::make_unique(tmp_file_name_on_disk); return std::make_unique(std::move(raw_file_read_buffer)); @@ -136,7 +140,8 @@ public: write_buffer->finalize(); uncompressed_write_buffer->finalize(); finalized = true; - return write_buffer->count(); + final_size = write_buffer->count(); + return final_size; } private: @@ -145,6 +150,7 @@ private: std::unique_ptr write_buffer; const String tmp_file_name_on_disk; bool finalized = false; + size_t final_size = 0; }; static void addMissedColumnsToSerializationInfos( From 597181c45e2395991cbb032c7eb2dc3542124e6c Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Tue, 10 Sep 2024 16:32:52 +0800 Subject: [PATCH 170/215] review --- src/Core/Settings.h | 6 +++--- src/Core/SettingsChangesHistory.cpp | 6 +++--- src/Interpreters/HashJoin/HashJoin.cpp | 2 +- src/Interpreters/TableJoin.cpp | 4 ++-- src/Interpreters/TableJoin.h | 8 ++++---- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 88555c67e24..28041089d9f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -922,9 +922,9 @@ class IColumn; M(Bool, implicit_transaction, false, "If enabled and not already inside a transaction, wraps the query inside a full transaction (begin + commit or rollback)", 0) \ M(UInt64, grace_hash_join_initial_buckets, 1, "Initial number of grace hash join buckets", 0) \ M(UInt64, grace_hash_join_max_buckets, 1024, "Limit on the number of grace hash join buckets", 0) \ - M(Int32, join_to_sort_perkey_rows_threshold, 40, "Rerange the right table by key in left or inner hash join when the per-key average rows of it exceed this value (means the table keys is dense) and its number of rows is not too many(controlled by `join_to_sort_table_rows_threshold`), to make the join output by the data batch of key, which would improve performance.", 0) \ - M(Int32, join_to_sort_table_rows_threshold, 10000, "Rerange the right table by key in left or inner hash join when its number of rows not exceed this value and the table keys is dense (controlled by `join_to_sort_perkey_rows_threshold`), to make the join performance improve as output by the data batch of key, but not cost too much on the table reranging.", 0) \ - M(Bool, allow_experimental_join_right_table_sorting, false, "If it is set to true, and the conditions of `join_to_sort_perkey_rows_threshold` and `join_to_sort_perkey_rows_threshold` are met, then we will try to rerange the right table by key to improve the performance in left or inner hash join.", 0) \ + M(Int32, join_to_sort_minimum_perkey_rows, 40, "The lower limit of per-key average rows in the right table to determine whether to rerange the right table by key in left or inner join. This setting ensures that the optimization is not applied for sparse table keys", 0) \ + M(Int32, join_to_sort_maximum_table_rows, 10000, "The maximum number of rows in the right table to determine whether to rerange the right table by key in left or inner join.", 0) \ + M(Bool, allow_experimental_join_right_table_sorting, false, "If it is set to true, and the conditions of `join_to_sort_minimum_perkey_rows` and `join_to_sort_maximum_table_rows` are met, rerange the right table by key to improve the performance in left or inner hash join.", 0) \ M(Timezone, session_timezone, "", "This setting can be removed in the future due to potential caveats. It is experimental and is not suitable for production usage. The default timezone for current session or query. The server default timezone if empty.", 0) \ M(Bool, use_hive_partitioning, false, "Allows to use hive partitioning for File, URL, S3, AzureBlobStorage and HDFS engines.", 0)\ \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 4ac7a0f2d8d..da29b6b11cd 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -95,9 +95,9 @@ static std::initializer_listsorted || data->blocks.empty() || data->maps.size() > 1 || data->rows_to_join > table_join->sortRightTableRowsThreshold() || data->avgPerKeyRows() < table_join->sortRightPerkeyRowsThreshold()) + if (!data || data->sorted || data->blocks.empty() || data->maps.size() > 1 || data->rows_to_join > table_join->sortRightMaximumTableRows() || data->avgPerKeyRows() < table_join->sortRightMinimumPerkeyRows()) return; if (data->keys_to_join == 0) diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp index 519264dd0e4..59a0374051f 100644 --- a/src/Interpreters/TableJoin.cpp +++ b/src/Interpreters/TableJoin.cpp @@ -116,8 +116,8 @@ TableJoin::TableJoin(const Settings & settings, VolumePtr tmp_volume_, Temporary , max_files_to_merge(settings.join_on_disk_max_files_to_merge) , temporary_files_codec(settings.temporary_files_codec) , output_by_rowlist_perkey_rows_threshold(settings.join_output_by_rowlist_perkey_rows_threshold) - , sort_right_perkey_rows_threshold(settings.join_to_sort_perkey_rows_threshold) - , sort_right_table_rows_threshold(settings.join_to_sort_table_rows_threshold) + , sort_right_minimum_perkey_rows(settings.join_to_sort_minimum_perkey_rows) + , sort_right_maximum_table_rows(settings.join_to_sort_maximum_table_rows) , allow_join_sorting(settings.allow_experimental_join_right_table_sorting) , max_memory_usage(settings.max_memory_usage) , tmp_volume(tmp_volume_) diff --git a/src/Interpreters/TableJoin.h b/src/Interpreters/TableJoin.h index c7926271a67..e1bae55a4ed 100644 --- a/src/Interpreters/TableJoin.h +++ b/src/Interpreters/TableJoin.h @@ -149,8 +149,8 @@ private: const size_t max_files_to_merge = 0; const String temporary_files_codec = "LZ4"; const size_t output_by_rowlist_perkey_rows_threshold = 0; - const size_t sort_right_perkey_rows_threshold = 0; - const size_t sort_right_table_rows_threshold = 0; + const size_t sort_right_minimum_perkey_rows = 0; + const size_t sort_right_maximum_table_rows = 0; const bool allow_join_sorting = false; /// Value if setting max_memory_usage for query, can be used when max_bytes_in_join is not specified. @@ -300,8 +300,8 @@ public: } size_t outputByRowListPerkeyRowsThreshold() const { return output_by_rowlist_perkey_rows_threshold; } - size_t sortRightPerkeyRowsThreshold() const { return sort_right_perkey_rows_threshold; } - size_t sortRightTableRowsThreshold() const { return sort_right_table_rows_threshold; } + size_t sortRightMinimumPerkeyRows() const { return sort_right_minimum_perkey_rows; } + size_t sortRightMaximumTableRows() const { return sort_right_maximum_table_rows; } bool allowJoinSorting() const { return allow_join_sorting; } size_t defaultMaxBytes() const { return default_max_bytes; } size_t maxJoinedBlockRows() const { return max_joined_block_rows; } From 89dd3188bb937db9dfc8b755e8a0a96f3c51adb0 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 10 Sep 2024 13:35:18 +0200 Subject: [PATCH 171/215] add a setting to disallow DETACH PERMANENTLY in Replicated --- src/Core/ServerSettings.h | 1 + src/Databases/DatabaseReplicated.cpp | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h index 79173503f28..5b2aaf5407a 100644 --- a/src/Core/ServerSettings.h +++ b/src/Core/ServerSettings.h @@ -148,6 +148,7 @@ namespace DB M(Bool, storage_metadata_write_full_object_key, false, "Write disk metadata files with VERSION_FULL_OBJECT_KEY format", 0) \ M(UInt64, max_materialized_views_count_for_table, 0, "A limit on the number of materialized views attached to a table.", 0) \ M(UInt32, max_database_replicated_create_table_thread_pool_size, 1, "The number of threads to create tables during replica recovery in DatabaseReplicated. Zero means number of threads equal number of cores.", 0) \ + M(Bool, database_replicated_allow_detach_permanently, true, "Allow detaching tables permanently in Replicated databases", 0) \ M(Bool, format_alter_operations_with_parentheses, false, "If enabled, each operation in alter queries will be surrounded with parentheses in formatted queries to make them less ambiguous.", 0) \ M(String, default_replica_path, "/clickhouse/tables/{uuid}/{shard}", "The path to the table in ZooKeeper", 0) \ M(String, default_replica_name, "{replica}", "The replica name in ZooKeeper", 0) \ diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 8e3378bcc12..a9009e57dde 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -63,6 +63,7 @@ namespace ErrorCodes extern const int NO_ACTIVE_REPLICAS; extern const int CANNOT_GET_REPLICATED_DATABASE_SNAPSHOT; extern const int CANNOT_RESTORE_TABLE; + extern const int SUPPORT_IS_DISABLED; } static constexpr const char * REPLICATED_DATABASE_MARK = "DatabaseReplicated"; @@ -1693,6 +1694,9 @@ void DatabaseReplicated::detachTablePermanently(ContextPtr local_context, const { waitDatabaseStarted(); + if (!local_context->getServerSettings().database_replicated_allow_detach_permanently) + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Support for DETACH TABLE PERMANENTLY is disabled"); + auto txn = local_context->getZooKeeperMetadataTransaction(); assert(!ddl_worker->isCurrentlyActive() || txn); if (txn && txn->isInitialQuery()) From 0453d5e91aadc73ad93b813677fe8e0d70c7fd13 Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Tue, 10 Sep 2024 12:26:03 +0000 Subject: [PATCH 172/215] small fixes --- src/Common/ZooKeeper/TestKeeper.cpp | 6 +++++- src/Coordination/KeeperStorage.cpp | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Common/ZooKeeper/TestKeeper.cpp b/src/Common/ZooKeeper/TestKeeper.cpp index ae8d95b806a..3982971163c 100644 --- a/src/Common/ZooKeeper/TestKeeper.cpp +++ b/src/Common/ZooKeeper/TestKeeper.cpp @@ -362,9 +362,13 @@ std::pair TestKeeperRemoveRecursiveRequest::process(TestKeepe std::vector> children; - for (const auto & [child_path, child_node] : container) + for (auto it = std::next(root_it); it != container.end(); ++it) + { + const auto & [child_path, child_node] = *it; + if (child_path.starts_with(path)) children.emplace_back(child_path, child_node); + } if (children.size() > remove_nodes_limit) { diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 24d95a89a4d..1bd6c27fe63 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -1766,7 +1766,7 @@ private: return false; - auto children = node_it->value.getChildren(); + const auto & children = node_it->value.getChildren(); for (auto && child_name : children) { auto child_path = (root_fs_path / child_name.toView()).generic_string(); From 0bd7ef39c41502a2d0cea7cb34fd12e9b9e8de30 Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Tue, 10 Sep 2024 12:27:34 +0000 Subject: [PATCH 173/215] unify processWatches --- src/Coordination/KeeperStorage.cpp | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 1bd6c27fe63..503eddac16e 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -1124,17 +1124,11 @@ struct KeeperStorageRequestProcessor } virtual KeeperStorageBase::ResponsesForSessions - processWatches(KeeperStorageBase::Watches & /*watches*/, KeeperStorageBase::Watches & /*list_watches*/) const + processWatches(const Storage & /*storage*/, int64_t /*zxid*/, KeeperStorageBase::Watches & /*watches*/, KeeperStorageBase::Watches & /*list_watches*/) const { return {}; } - virtual KeeperStorageBase::ResponsesForSessions - processWatches(const Storage & /*storage*/, int64_t /*zxid*/, KeeperStorageBase::Watches & watches, KeeperStorageBase::Watches & list_watches) const - { - return processWatches(watches, list_watches); - } - virtual bool checkAuth(Storage & /*storage*/, int64_t /*session_id*/, bool /*is_local*/) const { return true; } virtual ~KeeperStorageRequestProcessor() = default; @@ -1247,7 +1241,7 @@ struct KeeperStorageCreateRequestProcessor final : public KeeperStorageRequestPr using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor; KeeperStorageBase::ResponsesForSessions - processWatches(KeeperStorageBase::Watches & watches, KeeperStorageBase::Watches & list_watches) const override + processWatches(const Storage & /*storage*/, int64_t /*zxid*/, KeeperStorageBase::Watches & watches, KeeperStorageBase::Watches & list_watches) const override { return processWatchesImpl(this->zk_request->getPath(), watches, list_watches, Coordination::Event::CREATED); } @@ -1564,7 +1558,7 @@ struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestPr } KeeperStorageBase::ResponsesForSessions - processWatches(KeeperStorageBase::Watches & watches, KeeperStorageBase::Watches & list_watches) const override + processWatches(const Storage & /*storage*/, int64_t /*zxid*/, KeeperStorageBase::Watches & watches, KeeperStorageBase::Watches & list_watches) const override { return processWatchesImpl(this->zk_request->getPath(), watches, list_watches, Coordination::Event::DELETED); } @@ -1985,7 +1979,7 @@ struct KeeperStorageSetRequestProcessor final : public KeeperStorageRequestProce } KeeperStorageBase::ResponsesForSessions - processWatches(typename Storage::Watches & watches, typename Storage::Watches & list_watches) const override + processWatches(const Storage & /*storage*/, int64_t /*zxid*/, typename Storage::Watches & watches, typename Storage::Watches & list_watches) const override { return processWatchesImpl(this->zk_request->getPath(), watches, list_watches, Coordination::Event::CHANGED); } From 6719112b02e67ecc24346ac81fdb504f97e047a9 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 10 Sep 2024 15:08:58 +0200 Subject: [PATCH 174/215] Add azure-queue.md --- .../table-engines/integrations/azure-queue.md | 72 +++++++++++++++++++ .../engines/table-engines/integrations/s3.md | 2 +- .../table-engines/integrations/s3queue.md | 31 ++++---- 3 files changed, 88 insertions(+), 17 deletions(-) create mode 100644 docs/en/engines/table-engines/integrations/azure-queue.md diff --git a/docs/en/engines/table-engines/integrations/azure-queue.md b/docs/en/engines/table-engines/integrations/azure-queue.md new file mode 100644 index 00000000000..7c3d510ddcc --- /dev/null +++ b/docs/en/engines/table-engines/integrations/azure-queue.md @@ -0,0 +1,72 @@ +--- +slug: /en/engines/table-engines/integrations/azure-queue +sidebar_position: 181 +sidebar_label: AzureQueue +--- + +# AzureQueue Table Engine + +This engine provides an integration with [Azure Blob Storage](https://azure.microsoft.com/en-us/products/storage/blobs) ecosystem, allowing streaming data import. + +## Create Table {#creating-a-table} + +``` sql +CREATE TABLE test (name String, value UInt32) + ENGINE = AzureQueue(...) + [SETTINGS] + [mode = '',] + [after_processing = 'keep',] + [keeper_path = '',] + ... +``` + +**Engine parameters** + +`AzureQueue` parameters are the same as `AzureBlobStorage` table engine supports. See parameters section [here](../../../engines/table-engines/integrations/azureBlobStorage.md). + +**Example** + +```sql +CREATE TABLE s3queue_engine_table (name String, value UInt32) +ENGINE=AzureQueue('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite1:10000/devstoreaccount1/data/') +SETTINGS + mode = 'unordered' +``` + +## Settings {#settings} + +The set of supported settings is the same as for `S3Queue` table engine, but without `s3queue_` prefix. See [full list of settings settings](../../../engines/table-engines/integrations/s3queue.md#settings). + +## Description {#description} + +`SELECT` is not particularly useful for streaming import (except for debugging), because each file can be imported only once. It is more practical to create real-time threads using [materialized views](../../../sql-reference/statements/create/view.md). To do this: + +1. Use the engine to create a table for consuming from specified path in S3 and consider it a data stream. +2. Create a table with the desired structure. +3. Create a materialized view that converts data from the engine and puts it into a previously created table. + +When the `MATERIALIZED VIEW` joins the engine, it starts collecting data in the background. + +Example: + +``` sql + CREATE TABLE s3queue_engine_table (name String, value UInt32) + ENGINE=AzureQueue('', 'CSV', 'gzip') + SETTINGS + mode = 'unordered'; + + CREATE TABLE stats (name String, value UInt32) + ENGINE = MergeTree() ORDER BY name; + + CREATE MATERIALIZED VIEW consumer TO stats + AS SELECT name, value FROM s3queue_engine_table; + + SELECT * FROM stats ORDER BY name; +``` + +## Virtual columns {#virtual-columns} + +- `_path` — Path to the file. +- `_file` — Name of the file. + +For more information about virtual columns see [here](../../../engines/table-engines/index.md#table_engines-virtual_columns). diff --git a/docs/en/engines/table-engines/integrations/s3.md b/docs/en/engines/table-engines/integrations/s3.md index 48a08dfa499..f02d0563491 100644 --- a/docs/en/engines/table-engines/integrations/s3.md +++ b/docs/en/engines/table-engines/integrations/s3.md @@ -35,7 +35,7 @@ CREATE TABLE s3_engine_table (name String, value UInt32) [SETTINGS ...] ``` -### Engine parameters +### Engine parameters {#parameters} - `path` — Bucket url with path to file. Supports following wildcards in readonly mode: `*`, `**`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings. For more information see [below](#wildcards-in-path). - `NOSIGN` - If this keyword is provided in place of credentials, all the requests will not be signed. diff --git a/docs/en/engines/table-engines/integrations/s3queue.md b/docs/en/engines/table-engines/integrations/s3queue.md index 06325fa15fb..f1957cfe1ce 100644 --- a/docs/en/engines/table-engines/integrations/s3queue.md +++ b/docs/en/engines/table-engines/integrations/s3queue.md @@ -5,6 +5,7 @@ sidebar_label: S3Queue --- # S3Queue Table Engine + This engine provides integration with [Amazon S3](https://aws.amazon.com/s3/) ecosystem and allows streaming import. This engine is similar to the [Kafka](../../../engines/table-engines/integrations/kafka.md), [RabbitMQ](../../../engines/table-engines/integrations/rabbitmq.md) engines, but provides S3-specific features. ## Create Table {#creating-a-table} @@ -16,27 +17,25 @@ CREATE TABLE s3_queue_engine_table (name String, value UInt32) [mode = '',] [after_processing = 'keep',] [keeper_path = '',] - [s3queue_loading_retries = 0,] - [s3queue_processing_threads_num = 1,] - [s3queue_enable_logging_to_s3queue_log = 0,] - [s3queue_polling_min_timeout_ms = 1000,] - [s3queue_polling_max_timeout_ms = 10000,] - [s3queue_polling_backoff_ms = 0,] - [s3queue_tracked_file_ttl_sec = 0,] - [s3queue_tracked_files_limit = 1000,] - [s3queue_cleanup_interval_min_ms = 10000,] - [s3queue_cleanup_interval_max_ms = 30000,] + [loading_retries = 0,] + [processing_threads_num = 1,] + [enable_logging_to_s3queue_log = 0,] + [polling_min_timeout_ms = 1000,] + [polling_max_timeout_ms = 10000,] + [polling_backoff_ms = 0,] + [tracked_file_ttl_sec = 0,] + [tracked_files_limit = 1000,] + [cleanup_interval_min_ms = 10000,] + [cleanup_interval_max_ms = 30000,] ``` -Starting with `24.7` settings without `s3queue_` prefix are also supported. +:::warning +Before `24.7`, it is required to use `s3queue_` prefix for all settings apart from `mode`, `after_processing` and `keeper_path`. +::: **Engine parameters** -- `path` — Bucket url with path to file. Supports following wildcards in readonly mode: `*`, `**`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings. For more information see [below](#wildcards-in-path). -- `NOSIGN` - If this keyword is provided in place of credentials, all the requests will not be signed. -- `format` — The [format](../../../interfaces/formats.md#formats) of the file. -- `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. Parameter is optional. If credentials are not specified, they are used from the configuration file. For more information see [Using S3 for Data Storage](../mergetree-family/mergetree.md#table_engine-mergetree-s3). -- `compression` — Compression type. Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. Parameter is optional. By default, it will autodetect compression by file extension. +`S3Queue` parameters are the same as `S3` table engine supports. See parameters section [here](../../../engines/table-engines/integrations/s3.md#parameters). **Example** From 1aacb48bcf7abddaaac9bdb2c723189b2329167b Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Tue, 10 Sep 2024 14:34:27 +0000 Subject: [PATCH 175/215] fix collector --- src/Coordination/KeeperStorage.cpp | 73 ++++++++++++------ src/Coordination/KeeperStorage.h | 1 + src/Coordination/tests/gtest_coordination.cpp | 77 +++++++++++++++++-- 3 files changed, 123 insertions(+), 28 deletions(-) diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 503eddac16e..87d29e0ff65 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -832,6 +832,15 @@ std::shared_ptr KeeperStorage::UncommittedS return tryGetNodeFromStorage(path); } +template +const typename Container::Node * KeeperStorage::UncommittedState::getActualNodeView(StringRef path, const Node & storage_node) const +{ + if (auto node_it = nodes.find(path.toView()); node_it != nodes.end()) + return node_it->second.node.get(); + + return &storage_node; +} + template Coordination::ACLs KeeperStorage::UncommittedState::getACLs(StringRef path) const { @@ -1686,7 +1695,7 @@ private: { std::deque steps; - if (checkLimits(root_node)) + if (checkLimits(&root_node)) return true; steps.push_back(Step{root_path.toString(), &root_node, 0}); @@ -1698,17 +1707,20 @@ private: StringRef path = step.path; uint32_t level = step.level; - const SNode * node = nullptr; + const SNode * node_ptr = nullptr; if (auto * rdb = std::get_if(&step.node)) - node = rdb; + node_ptr = rdb; else - node = std::get(step.node); + node_ptr = std::get(step.node); chassert(!path.empty()); - chassert(node != nullptr); + chassert(node_ptr != nullptr); - if (visitRocksDBNode(steps, path, level) || visitMemNode(steps, path, level) || visitRootAndUncommitted(steps, path, *node, level)) + const auto & node = *node_ptr; + chassert(storage.uncommitted_state.getActualNodeView(path, node) != nullptr); /// explicitly check that node is not deleted + + if (visitRocksDBNode(steps, path, level) || visitMemNode(steps, path, level) || visitRootAndUncommitted(steps, path, node, level)) return true; } @@ -1736,13 +1748,18 @@ private: std::filesystem::path root_fs_path(root_path.toString()); auto children = storage.container.getChildren(root_path.toString()); - for (auto && [child_name, node] : children) + for (auto && [child_name, child_node] : children) { - if (checkLimits(node)) + auto child_path = (root_fs_path / child_name).generic_string(); + const auto actual_child_node_ptr = storage.uncommitted_state.getActualNodeView(child_path, child_node); + + if (actual_child_node_ptr == nullptr) /// node was deleted in previous step of multi transaction + continue; + + if (checkLimits(actual_child_node_ptr)) return true; - auto child_path = (root_fs_path / child_name).generic_string(); - steps.push_back(Step{std::move(child_path), std::move(node), level + 1}); + steps.push_back(Step{std::move(child_path), std::move(child_node), level + 1}); } } @@ -1753,25 +1770,30 @@ private: { if constexpr (!Storage::use_rocksdb) { - std::filesystem::path root_fs_path(root_path.toString()); - auto node_it = storage.container.find(root_path); if (node_it == storage.container.end()) return false; - + std::filesystem::path root_fs_path(root_path.toString()); const auto & children = node_it->value.getChildren(); - for (auto && child_name : children) + + for (const auto & child_name : children) { auto child_path = (root_fs_path / child_name.toView()).generic_string(); auto child_it = storage.container.find(child_path); chassert(child_it != storage.container.end()); + const auto & child_node = child_it->value; - if (checkLimits(child_it->value)) + const auto actual_child_node_ptr = storage.uncommitted_state.getActualNodeView(child_path, child_node); + + if (actual_child_node_ptr == nullptr) /// node was deleted in previous step of multi transaction + continue; + + if (checkLimits(actual_child_node_ptr)) return true; - steps.push_back(Step{std::move(child_path), child_it->value, level + 1}); + steps.push_back(Step{std::move(child_path), &child_node, level + 1}); } } @@ -1787,14 +1809,18 @@ private: for (; it != nodes.end() && parentNodePath(it->first) == root_path; ++it) { - chassert(it->second.node); - const String & path = it->first; - const SNode & node = *it->second.node; + const auto actual_child_node_ptr = it->second.node.get(); - if (checkLimits(node)) + if (actual_child_node_ptr == nullptr) /// node was deleted in previous step of multi transaction + continue; + + if (checkLimits(actual_child_node_ptr)) return true; - steps.push_back(Step{path, &node, level + 1}); + const String & child_path = it->first; + const SNode & child_node = *it->second.node; + + steps.push_back(Step{child_path, &child_node, level + 1}); } addDelta(root_path, root_node, level); @@ -1820,9 +1846,10 @@ private: by_level_deltas[level].emplace_back(root_path.toString(), zxid, typename Storage::RemoveNodeDelta{root_node.version, root_node.ephemeralOwner()}); } - bool checkLimits(const SNode & node) + bool checkLimits(const SNode * node) { - nodes_observed += node.numChildren(); + chassert(node != nullptr); + nodes_observed += node->numChildren(); return nodes_observed > limit; } }; diff --git a/src/Coordination/KeeperStorage.h b/src/Coordination/KeeperStorage.h index c2f6e4c5a74..904af76ef37 100644 --- a/src/Coordination/KeeperStorage.h +++ b/src/Coordination/KeeperStorage.h @@ -566,6 +566,7 @@ public: void rollback(int64_t rollback_zxid); std::shared_ptr getNode(StringRef path) const; + const Node * getActualNodeView(StringRef path, const Node & storage_node) const; Coordination::ACLs getACLs(StringRef path) const; void applyDelta(const Delta & delta); diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index 037a102082c..deb6aaed8c9 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -3563,11 +3563,15 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveInMultiRequest) Storage storage{500, "", this->keeper_context}; int zxid = 0; - Coordination::Requests ops; - ops.push_back(zkutil::makeCreateRequest("/A", "A", zkutil::CreateMode::Persistent)); - ops.push_back(zkutil::makeCreateRequest("/A/B", "B", zkutil::CreateMode::Persistent)); - ops.push_back(zkutil::makeCreateRequest("/A/C", "C", zkutil::CreateMode::Ephemeral)); - ops.push_back(zkutil::makeCreateRequest("/A/B/D", "D", zkutil::CreateMode::Ephemeral)); + auto prepare_create_tree = []() + { + return Coordination::Requests{ + zkutil::makeCreateRequest("/A", "A", zkutil::CreateMode::Persistent), + zkutil::makeCreateRequest("/A/B", "B", zkutil::CreateMode::Persistent), + zkutil::makeCreateRequest("/A/C", "C", zkutil::CreateMode::Ephemeral), + zkutil::makeCreateRequest("/A/B/D", "D", zkutil::CreateMode::Ephemeral), + }; + }; const auto exists = [&](const String & path) { @@ -3597,6 +3601,7 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveInMultiRequest) { SCOPED_TRACE("Remove In Multi Tx"); int new_zxid = ++zxid; + auto ops = prepare_create_tree(); ops.push_back(zkutil::makeRemoveRequest("/A", -1)); const auto request = std::make_shared(ops, ACLs{}); @@ -3612,6 +3617,7 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveInMultiRequest) { SCOPED_TRACE("Recursive Remove In Multi Tx"); int new_zxid = ++zxid; + auto ops = prepare_create_tree(); ops.push_back(zkutil::makeRemoveRecursiveRequest("/A", 4)); const auto request = std::make_shared(ops, ACLs{}); @@ -3627,6 +3633,67 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveInMultiRequest) ASSERT_FALSE(exists("/A/B")); ASSERT_FALSE(exists("/A/B/D")); } + + { + SCOPED_TRACE("Recursive Remove With Regular In Multi Tx"); + int new_zxid = ++zxid; + auto ops = prepare_create_tree(); + + ops.push_back(zkutil::makeRemoveRequest("/A/C", -1)); + ops.push_back(zkutil::makeRemoveRecursiveRequest("/A", 4)); + const auto request = std::make_shared(ops, ACLs{}); + + storage.preprocessRequest(request, 1, 0, new_zxid); + auto responses = storage.processRequest(request, 1, new_zxid); + ops.pop_back(); + ops.pop_back(); + + ASSERT_EQ(responses.size(), 1); + ASSERT_TRUE(is_multi_ok(responses[0].response)); + ASSERT_FALSE(exists("/A")); + ASSERT_FALSE(exists("/A/C")); + ASSERT_FALSE(exists("/A/B")); + ASSERT_FALSE(exists("/A/B/D")); + } + + { + SCOPED_TRACE("Recursive Remove From Committed and Uncommitted states"); + int create_zxid = ++zxid; + auto ops = prepare_create_tree(); + + /// First create nodes + const auto create_request = std::make_shared(ops, ACLs{}); + storage.preprocessRequest(create_request, 1, 0, create_zxid); + auto create_responses = storage.processRequest(create_request, 1, create_zxid); + ASSERT_EQ(create_responses.size(), 1); + ASSERT_TRUE(is_multi_ok(create_responses[0].response)); + ASSERT_TRUE(exists("/A")); + ASSERT_TRUE(exists("/A/C")); + ASSERT_TRUE(exists("/A/B")); + ASSERT_TRUE(exists("/A/B/D")); + + /// Remove node A/C as a single remove request. + /// Remove all other as remove recursive request. + /// In this case we should list storage to understand the tree topology + /// but ignore already deleted nodes in uncommitted state. + + int remove_zxid = ++zxid; + ops = { + zkutil::makeRemoveRequest("/A/C", -1), + zkutil::makeRemoveRecursiveRequest("/A", 3), + }; + const auto remove_request = std::make_shared(ops, ACLs{}); + + storage.preprocessRequest(remove_request, 1, 0, remove_zxid); + auto remove_responses = storage.processRequest(remove_request, 1, remove_zxid); + + ASSERT_EQ(remove_responses.size(), 1); + ASSERT_TRUE(is_multi_ok(remove_responses[0].response)); + ASSERT_FALSE(exists("/A")); + ASSERT_FALSE(exists("/A/C")); + ASSERT_FALSE(exists("/A/B")); + ASSERT_FALSE(exists("/A/B/D")); + } } TYPED_TEST(CoordinationTest, TestRemoveRecursiveWatches) From acbeaa6d3339b7206c6f9e822445952fe664a807 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Tue, 10 Sep 2024 16:14:53 +0100 Subject: [PATCH 176/215] fix --- tests/integration/test_disks_app_func/test.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_disks_app_func/test.py b/tests/integration/test_disks_app_func/test.py index 56ea5c8846a..a4b2399e117 100644 --- a/tests/integration/test_disks_app_func/test.py +++ b/tests/integration/test_disks_app_func/test.py @@ -13,8 +13,20 @@ def started_cluster(): main_configs=["config.xml"], with_minio=True, ) - cluster.start() + + # local disk requires its `path` directory to exist. + # the two paths below belong to `test1` and `test2` disks + node = cluster.instances["disks_app_test"] + for path in ["path1", "path2"]: + node.exec_in_container( + [ + "bash", + "-c", + f"mkdir -p /var/lib/clickhouse/{path}", + ] + ) + yield cluster finally: From 04dcf73e8fb98fbcdee2ead46e7177c961e1b5f6 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 10 Sep 2024 15:35:59 +0000 Subject: [PATCH 177/215] Fix crash in sqidDecode --- src/Functions/sqid.cpp | 2 +- tests/queries/0_stateless/02933_sqid.reference | 1 + tests/queries/0_stateless/02933_sqid.sql | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Functions/sqid.cpp b/src/Functions/sqid.cpp index 0e133590b84..32434eb5e73 100644 --- a/src/Functions/sqid.cpp +++ b/src/Functions/sqid.cpp @@ -124,7 +124,7 @@ public: std::string_view sqid = col_non_const->getDataAt(i).toView(); std::vector integers = sqids.decode(String(sqid)); res_nested_data.insert(integers.begin(), integers.end()); - res_offsets_data.push_back(integers.size()); + res_offsets_data.push_back(i == 0 ? integers.size() : res_offsets_data.back() + integers.size()); } } else diff --git a/tests/queries/0_stateless/02933_sqid.reference b/tests/queries/0_stateless/02933_sqid.reference index a559bacb0ac..4597e2347e3 100644 --- a/tests/queries/0_stateless/02933_sqid.reference +++ b/tests/queries/0_stateless/02933_sqid.reference @@ -13,5 +13,6 @@ Td1EnWQo [1,2,3,4] XMbT -- invalid sqid [] +-- bug 69450 -- alias XMbT diff --git a/tests/queries/0_stateless/02933_sqid.sql b/tests/queries/0_stateless/02933_sqid.sql index 81d4b2bc35c..822fe33df51 100644 --- a/tests/queries/0_stateless/02933_sqid.sql +++ b/tests/queries/0_stateless/02933_sqid.sql @@ -25,5 +25,12 @@ SELECT sqidEncode(toNullable(materialize(1)), toLowCardinality(materialize(2))); SELECT '-- invalid sqid'; SELECT sqidDecode('invalid sqid'); +SELECT '-- bug 69450'; +DROP TABLE IF EXISTS tab; +CREATE TABLE tab (id String) ENGINE = MergeTree ORDER BY id; +INSERT INTO tab SELECT * FROM generateRandom() LIMIT 1000000; +SELECT sqidDecode(id) FROM tab FORMAT Null; +DROP TABLE tab; + SELECT '-- alias'; SELECT sqid(1, 2); From d43264c44e80f1f51604d266e5b1085462552aff Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 10 Sep 2024 17:41:52 +0200 Subject: [PATCH 178/215] Quick fix for s3queue problem --- .../StorageObjectStorageQueue.cpp | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index 9452ce81e9e..55a1d43b26b 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -64,9 +64,7 @@ namespace void checkAndAdjustSettings( ObjectStorageQueueSettings & queue_settings, - ASTStorage * engine_args, - bool is_attach, - const LoggerPtr & log) + bool is_attach) { if (!is_attach && !queue_settings.mode.changed) { @@ -85,16 +83,6 @@ namespace "Setting `cleanup_interval_min_ms` ({}) must be less or equal to `cleanup_interval_max_ms` ({})", queue_settings.cleanup_interval_min_ms, queue_settings.cleanup_interval_max_ms); } - - if (!is_attach && !queue_settings.processing_threads_num.changed) - { - queue_settings.processing_threads_num = std::max(getNumberOfPhysicalCPUCores(), 16); - engine_args->settings->as()->changes.insertSetting( - "processing_threads_num", - queue_settings.processing_threads_num.value); - - LOG_TRACE(log, "Set `processing_threads_num` to {}", queue_settings.processing_threads_num); - } } std::shared_ptr getQueueLog(const ObjectStoragePtr & storage, const ContextPtr & context, const ObjectStorageQueueSettings & table_settings) @@ -154,7 +142,7 @@ StorageObjectStorageQueue::StorageObjectStorageQueue( throw Exception(ErrorCodes::BAD_QUERY_PARAMETER, "ObjectStorageQueue url must either end with '/' or contain globs"); } - checkAndAdjustSettings(*queue_settings, engine_args, mode > LoadingStrictnessLevel::CREATE, log); + checkAndAdjustSettings(*queue_settings, mode > LoadingStrictnessLevel::CREATE); object_storage = configuration->createObjectStorage(context_, /* is_readonly */true); FormatFactory::instance().checkFormatName(configuration->format); From d8e670297bcdc1850260a1d60d4ee39b2a895a4d Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 10 Sep 2024 17:42:27 +0200 Subject: [PATCH 179/215] groupConcat consistency --- .../AggregateFunctionGroupConcat.cpp | 13 +++++++++++-- .../03235_groupArray_returns_string.reference | 1 + .../0_stateless/03235_groupArray_returns_string.sql | 10 ++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/03235_groupArray_returns_string.reference create mode 100644 tests/queries/0_stateless/03235_groupArray_returns_string.sql diff --git a/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp b/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp index 636ac80e350..8fb0b645096 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp +++ b/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp @@ -116,15 +116,17 @@ class GroupConcatImpl final SerializationPtr serialization; UInt64 limit; const String delimiter; + const DataTypePtr type; public: GroupConcatImpl(const DataTypePtr & data_type_, const Array & parameters_, UInt64 limit_, const String & delimiter_) : IAggregateFunctionDataHelper, GroupConcatImpl>( {data_type_}, parameters_, std::make_shared()) - , serialization(this->argument_types[0]->getDefaultSerialization()) , limit(limit_) , delimiter(delimiter_) + , type(data_type_) { + serialization = isFixedString(type) ? std::make_shared()->getDefaultSerialization() : this->argument_types[0]->getDefaultSerialization(); } String getName() const override { return name; } @@ -140,7 +142,14 @@ public: if (cur_data.data_size != 0) cur_data.insertChar(delimiter.c_str(), delimiter.size(), arena); - cur_data.insert(columns[0], serialization, row_num, arena); + if (isFixedString(type)) + { + ColumnWithTypeAndName col = {columns[0]->getPtr(), type, "column"}; + const auto & col_str = castColumn(col, std::make_shared()); + cur_data.insert(col_str.get(), serialization, row_num, arena); + } + else + cur_data.insert(columns[0], serialization, row_num, arena); } void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override diff --git a/tests/queries/0_stateless/03235_groupArray_returns_string.reference b/tests/queries/0_stateless/03235_groupArray_returns_string.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/03235_groupArray_returns_string.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/03235_groupArray_returns_string.sql b/tests/queries/0_stateless/03235_groupArray_returns_string.sql new file mode 100644 index 00000000000..618ec6f839b --- /dev/null +++ b/tests/queries/0_stateless/03235_groupArray_returns_string.sql @@ -0,0 +1,10 @@ +CREATE TABLE t (st FixedString(54)) ENGINE=MergeTree ORDER BY (); + +INSERT INTO t VALUES +('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRTUVWXYZ'), +('\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0'), +('IIIIIIIIII\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0'); + +WITH (SELECT groupConcat(',')(st) FROM t) AS a, + (SELECT groupConcat(',')(st :: String) FROM t) AS b +SELECT equals(a, b); From f588e3c31bc2f097f168646f73ed078259265659 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 10 Sep 2024 17:43:50 +0200 Subject: [PATCH 180/215] rename tests --- ...ng.reference => 03235_groupArray_string_consistency.reference} | 0 ...returns_string.sql => 03235_groupArray_string_consistency.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{03235_groupArray_returns_string.reference => 03235_groupArray_string_consistency.reference} (100%) rename tests/queries/0_stateless/{03235_groupArray_returns_string.sql => 03235_groupArray_string_consistency.sql} (100%) diff --git a/tests/queries/0_stateless/03235_groupArray_returns_string.reference b/tests/queries/0_stateless/03235_groupArray_string_consistency.reference similarity index 100% rename from tests/queries/0_stateless/03235_groupArray_returns_string.reference rename to tests/queries/0_stateless/03235_groupArray_string_consistency.reference diff --git a/tests/queries/0_stateless/03235_groupArray_returns_string.sql b/tests/queries/0_stateless/03235_groupArray_string_consistency.sql similarity index 100% rename from tests/queries/0_stateless/03235_groupArray_returns_string.sql rename to tests/queries/0_stateless/03235_groupArray_string_consistency.sql From 080193cfc36909ab9a1016b70edc007c8dd3540f Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Tue, 10 Sep 2024 15:46:21 +0000 Subject: [PATCH 181/215] 14% more aesthetic code --- src/Functions/sqid.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/sqid.cpp b/src/Functions/sqid.cpp index 32434eb5e73..074a34bd083 100644 --- a/src/Functions/sqid.cpp +++ b/src/Functions/sqid.cpp @@ -124,7 +124,7 @@ public: std::string_view sqid = col_non_const->getDataAt(i).toView(); std::vector integers = sqids.decode(String(sqid)); res_nested_data.insert(integers.begin(), integers.end()); - res_offsets_data.push_back(i == 0 ? integers.size() : res_offsets_data.back() + integers.size()); + res_offsets_data.push_back(res_offsets_data.back() + integers.size()); } } else From 8b7a5616a23d6d7a27961a891483557b754fab22 Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Tue, 10 Sep 2024 15:51:31 +0000 Subject: [PATCH 182/215] update test --- src/Coordination/tests/gtest_coordination.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index deb6aaed8c9..b908140f1d7 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -3640,7 +3640,7 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveInMultiRequest) auto ops = prepare_create_tree(); ops.push_back(zkutil::makeRemoveRequest("/A/C", -1)); - ops.push_back(zkutil::makeRemoveRecursiveRequest("/A", 4)); + ops.push_back(zkutil::makeRemoveRecursiveRequest("/A", 3)); const auto request = std::make_shared(ops, ACLs{}); storage.preprocessRequest(request, 1, 0, new_zxid); From dcd1874520ce7934b3bd67e93b4441bf063138a6 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 10 Sep 2024 18:12:56 +0200 Subject: [PATCH 183/215] Update docs/en/engines/table-engines/integrations/azure-queue.md Co-authored-by: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> --- docs/en/engines/table-engines/integrations/azure-queue.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/engines/table-engines/integrations/azure-queue.md b/docs/en/engines/table-engines/integrations/azure-queue.md index 7c3d510ddcc..7e19bc92973 100644 --- a/docs/en/engines/table-engines/integrations/azure-queue.md +++ b/docs/en/engines/table-engines/integrations/azure-queue.md @@ -50,7 +50,7 @@ When the `MATERIALIZED VIEW` joins the engine, it starts collecting data in the Example: ``` sql - CREATE TABLE s3queue_engine_table (name String, value UInt32) + CREATE TABLE azure_queue_engine_table (name String, value UInt32) ENGINE=AzureQueue('', 'CSV', 'gzip') SETTINGS mode = 'unordered'; @@ -59,7 +59,7 @@ Example: ENGINE = MergeTree() ORDER BY name; CREATE MATERIALIZED VIEW consumer TO stats - AS SELECT name, value FROM s3queue_engine_table; + AS SELECT name, value FROM azure_queue_engine_table; SELECT * FROM stats ORDER BY name; ``` From b7e863b0e9b1891c5e4adaec61b5d51db6fc0fcd Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 10 Sep 2024 18:13:04 +0200 Subject: [PATCH 184/215] Update docs/en/engines/table-engines/integrations/azure-queue.md Co-authored-by: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> --- docs/en/engines/table-engines/integrations/azure-queue.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/integrations/azure-queue.md b/docs/en/engines/table-engines/integrations/azure-queue.md index 7e19bc92973..b5259336a8b 100644 --- a/docs/en/engines/table-engines/integrations/azure-queue.md +++ b/docs/en/engines/table-engines/integrations/azure-queue.md @@ -27,7 +27,7 @@ CREATE TABLE test (name String, value UInt32) **Example** ```sql -CREATE TABLE s3queue_engine_table (name String, value UInt32) +CREATE TABLE azure_queue_engine_table (name String, value UInt32) ENGINE=AzureQueue('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite1:10000/devstoreaccount1/data/') SETTINGS mode = 'unordered' From 6a6d26aeeda7c6d480dbaf73658567ed494ca67b Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 10 Sep 2024 18:16:27 +0200 Subject: [PATCH 185/215] Update StorageObjectStorageQueue.cpp --- src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index 55a1d43b26b..c1ef37e1a48 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -118,7 +118,7 @@ StorageObjectStorageQueue::StorageObjectStorageQueue( const String & comment, ContextPtr context_, std::optional format_settings_, - ASTStorage * engine_args, + ASTStorage * /* engine_args */, LoadingStrictnessLevel mode) : IStorage(table_id_) , WithContext(context_) From caab4dd8b903a8caaa45b0e9d811487110ac2560 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 10 Sep 2024 18:20:48 +0200 Subject: [PATCH 186/215] fix --- src/Interpreters/Squashing.cpp | 2 +- .../03236_squashing_high_memory.reference | 0 .../03236_squashing_high_memory.sql | 22 +++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/03236_squashing_high_memory.reference create mode 100644 tests/queries/0_stateless/03236_squashing_high_memory.sql diff --git a/src/Interpreters/Squashing.cpp b/src/Interpreters/Squashing.cpp index 95b76c60063..c656a1a797b 100644 --- a/src/Interpreters/Squashing.cpp +++ b/src/Interpreters/Squashing.cpp @@ -45,7 +45,7 @@ Chunk Squashing::squash(Chunk && input_chunk) Chunk Squashing::add(Chunk && input_chunk) { - if (!input_chunk) + if (!input_chunk || input_chunk.getNumRows() == 0) return {}; /// Just read block is already enough. diff --git a/tests/queries/0_stateless/03236_squashing_high_memory.reference b/tests/queries/0_stateless/03236_squashing_high_memory.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03236_squashing_high_memory.sql b/tests/queries/0_stateless/03236_squashing_high_memory.sql new file mode 100644 index 00000000000..523281fb74a --- /dev/null +++ b/tests/queries/0_stateless/03236_squashing_high_memory.sql @@ -0,0 +1,22 @@ +DROP TABLE IF EXISTS id_values; + +DROP TABLE IF EXISTS test_table; + +CREATE TABLE id_values ENGINE MergeTree ORDER BY id1 AS + SELECT arrayJoin(range(1000000)) AS id1, arrayJoin(range(1000)) AS id2; + +SET max_memory_usage = 1G; + +CREATE TABLE test_table ENGINE MergeTree ORDER BY id AS +SELECT id_values.id1 AS id, + string_values.string_val1 AS string_val1, + string_values.string_val2 AS string_val2 +FROM id_values + JOIN (SELECT arrayJoin(range(10)) AS id1, + 'qwe' AS string_val1, + 'asd' AS string_val2) AS string_values + ON id_values.id1 = string_values.id1 + SETTINGS join_algorithm = 'hash'; + +DROP TABLE IF EXISTS id_values; +DROP TABLE IF EXISTS test_table; From df1821a5798b8d138c3da4b39517196ba06b447a Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 10 Sep 2024 19:32:28 +0200 Subject: [PATCH 187/215] Update tests/queries/0_stateless/03236_squashing_high_memory.sql --- tests/queries/0_stateless/03236_squashing_high_memory.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03236_squashing_high_memory.sql b/tests/queries/0_stateless/03236_squashing_high_memory.sql index 523281fb74a..98b33b80e9f 100644 --- a/tests/queries/0_stateless/03236_squashing_high_memory.sql +++ b/tests/queries/0_stateless/03236_squashing_high_memory.sql @@ -3,7 +3,7 @@ DROP TABLE IF EXISTS id_values; DROP TABLE IF EXISTS test_table; CREATE TABLE id_values ENGINE MergeTree ORDER BY id1 AS - SELECT arrayJoin(range(1000000)) AS id1, arrayJoin(range(1000)) AS id2; + SELECT arrayJoin(range(500000)) AS id1, arrayJoin(range(1000)) AS id2; SET max_memory_usage = 1G; From 3730582d0639bb2e0247b66aa9a17ce9bd6e00d7 Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Tue, 10 Sep 2024 17:43:35 +0000 Subject: [PATCH 188/215] add full tree acl check --- src/Coordination/KeeperStorage.cpp | 66 ++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 87d29e0ff65..2eb9ab30efa 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -1584,7 +1584,7 @@ struct KeeperStorageRemoveRecursiveRequestProcessor final : public KeeperStorage } std::vector - preprocess(Storage & storage, int64_t zxid, int64_t /*session_id*/, int64_t /*time*/, uint64_t & digest, const KeeperContext & keeper_context) const override + preprocess(Storage & storage, int64_t zxid, int64_t session_id, int64_t /*time*/, uint64_t & digest, const KeeperContext & keeper_context) const override { ProfileEvents::increment(ProfileEvents::KeeperRemoveRequest); Coordination::ZooKeeperRemoveRecursiveRequest & request = dynamic_cast(*this->zk_request); @@ -1609,10 +1609,13 @@ struct KeeperStorageRemoveRecursiveRequestProcessor final : public KeeperStorage return {typename Storage::Delta{zxid, Coordination::Error::ZNONODE}}; } - ToDeleteTreeCollector collector(storage, zxid, request.remove_nodes_limit); - bool limit_exceeded = collector.collect(request.path, *node); + ToDeleteTreeCollector collector(storage, zxid, session_id, request.remove_nodes_limit); + auto collect_status = collector.collect(request.path, *node); - if (limit_exceeded) + if (collect_status == ToDeleteTreeCollector::CollectStatus::NoAuth) + return {typename Storage::Delta{zxid, Coordination::Error::ZNOAUTH}}; + + if (collect_status == ToDeleteTreeCollector::CollectStatus::LimitExceeded) return {typename Storage::Delta{zxid, Coordination::Error::ZNOTEMPTY}}; if (request.restored_from_zookeeper_log) @@ -1670,6 +1673,7 @@ private: { Storage & storage; int64_t zxid; + int64_t session_id; uint32_t limit; uint32_t max_level = 0; @@ -1683,20 +1687,30 @@ private: uint32_t level; }; + enum class CollectStatus + { + Ok, + NoAuth, + LimitExceeded, + }; + + friend struct KeeperStorageRemoveRecursiveRequestProcessor; + public: - ToDeleteTreeCollector(Storage & storage_, int64_t zxid_, uint32_t limit_) + ToDeleteTreeCollector(Storage & storage_, int64_t zxid_, int64_t session_id_, uint32_t limit_) : storage(storage_) , zxid(zxid_) + , session_id(session_id_) , limit(limit_) { } - bool collect(StringRef root_path, const SNode & root_node) + CollectStatus collect(StringRef root_path, const SNode & root_node) { std::deque steps; if (checkLimits(&root_node)) - return true; + return CollectStatus::LimitExceeded; steps.push_back(Step{root_path.toString(), &root_node, 0}); @@ -1718,13 +1732,23 @@ private: chassert(node_ptr != nullptr); const auto & node = *node_ptr; - chassert(storage.uncommitted_state.getActualNodeView(path, node) != nullptr); /// explicitly check that node is not deleted + auto actual_node_ptr = storage.uncommitted_state.getActualNodeView(path, node); + chassert(actual_node_ptr != nullptr); /// explicitly check that node is not deleted - if (visitRocksDBNode(steps, path, level) || visitMemNode(steps, path, level) || visitRootAndUncommitted(steps, path, node, level)) - return true; + if (actual_node_ptr->numChildren() > 0 && !storage.checkACL(path, Coordination::ACL::Delete, session_id, /*is_local=*/false)) + return CollectStatus::NoAuth; + + if (auto status = visitRocksDBNode(steps, path, level); status != CollectStatus::Ok) + return status; + + if (auto status = visitMemNode(steps, path, level); status != CollectStatus::Ok) + return status; + + if (auto status = visitRootAndUncommitted(steps, path, node, level); status != CollectStatus::Ok) + return status; } - return false; + return CollectStatus::Ok; } std::vector extractDeltas() @@ -1741,7 +1765,7 @@ private: } private: - bool visitRocksDBNode(std::deque & steps, StringRef root_path, uint32_t level) + CollectStatus visitRocksDBNode(std::deque & steps, StringRef root_path, uint32_t level) { if constexpr (Storage::use_rocksdb) { @@ -1757,22 +1781,22 @@ private: continue; if (checkLimits(actual_child_node_ptr)) - return true; + return CollectStatus::LimitExceeded; steps.push_back(Step{std::move(child_path), std::move(child_node), level + 1}); } } - return false; + return CollectStatus::Ok; } - bool visitMemNode(std::deque & steps, StringRef root_path, uint32_t level) + CollectStatus visitMemNode(std::deque & steps, StringRef root_path, uint32_t level) { if constexpr (!Storage::use_rocksdb) { auto node_it = storage.container.find(root_path); if (node_it == storage.container.end()) - return false; + return CollectStatus::Ok; std::filesystem::path root_fs_path(root_path.toString()); const auto & children = node_it->value.getChildren(); @@ -1791,16 +1815,16 @@ private: continue; if (checkLimits(actual_child_node_ptr)) - return true; + return CollectStatus::LimitExceeded; steps.push_back(Step{std::move(child_path), &child_node, level + 1}); } } - return false; + return CollectStatus::Ok; } - bool visitRootAndUncommitted(std::deque & steps, StringRef root_path, const SNode & root_node, uint32_t level) + CollectStatus visitRootAndUncommitted(std::deque & steps, StringRef root_path, const SNode & root_node, uint32_t level) { const auto & nodes = storage.uncommitted_state.nodes; @@ -1815,7 +1839,7 @@ private: continue; if (checkLimits(actual_child_node_ptr)) - return true; + return CollectStatus::LimitExceeded; const String & child_path = it->first; const SNode & child_node = *it->second.node; @@ -1825,7 +1849,7 @@ private: addDelta(root_path, root_node, level); - return false; + return CollectStatus::Ok; } void addDelta(StringRef root_path, const SNode & root_node, uint32_t level) From 00fe1d12eee2e7a7b328d582861ec37cd5606190 Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Tue, 10 Sep 2024 17:45:39 +0000 Subject: [PATCH 189/215] add acl test for rmr --- src/Coordination/tests/gtest_coordination.cpp | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index b908140f1d7..db113f6dfbe 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -3792,6 +3792,70 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveWatches) ASSERT_EQ(storage.list_watches.size(), 0); } +TYPED_TEST(CoordinationTest, TestRemoveRecursiveAcls) +{ + using namespace DB; + using namespace Coordination; + + using Storage = typename TestFixture::Storage; + + ChangelogDirTest rocks("./rocksdb"); + this->setRocksDBDirectory("./rocksdb"); + + Storage storage{500, "", this->keeper_context}; + int zxid = 0; + + { + int new_zxid = ++zxid; + String user_auth_data = "test_user:test_password"; + + const auto auth_request = std::make_shared(); + auth_request->scheme = "digest"; + auth_request->data = user_auth_data; + + storage.preprocessRequest(auth_request, 1, 0, new_zxid); + auto responses = storage.processRequest(auth_request, 1, new_zxid); + + EXPECT_EQ(responses.size(), 1); + EXPECT_EQ(responses[0].response->error, Coordination::Error::ZOK) << "Failed to add auth to session"; + } + + const auto create = [&](const String & path) + { + int new_zxid = ++zxid; + + const auto create_request = std::make_shared(); + create_request->path = path; + create_request->acls = {{.permissions = ACL::Create, .scheme = "auth", .id = ""}}; + + storage.preprocessRequest(create_request, 1, 0, new_zxid); + auto responses = storage.processRequest(create_request, 1, new_zxid); + + EXPECT_EQ(responses.size(), 1); + EXPECT_EQ(responses[0].response->error, Coordination::Error::ZOK) << "Failed to create " << path; + }; + + /// Add nodes with only Create ACL + create("/A"); + create("/A/B"); + create("/A/C"); + create("/A/B/D"); + + { + int new_zxid = ++zxid; + + auto remove_request = std::make_shared(); + remove_request->path = "/A"; + remove_request->remove_nodes_limit = 4; + + storage.preprocessRequest(remove_request, 1, 0, new_zxid); + auto responses = storage.processRequest(remove_request, 1, new_zxid); + + EXPECT_EQ(responses.size(), 1); + EXPECT_EQ(responses[0].response->error, Coordination::Error::ZNOAUTH); + } +} + /// INSTANTIATE_TEST_SUITE_P(CoordinationTestSuite, /// CoordinationTest, /// ::testing::ValuesIn(std::initializer_list{CompressionParam{true, ".zstd"}, CompressionParam{false, ""}})); From 3e3d6523207ac93e96c6f7434d56cd21fd230947 Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Tue, 10 Sep 2024 17:51:35 +0000 Subject: [PATCH 190/215] refactor tests --- src/Coordination/tests/gtest_coordination.cpp | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index db113f6dfbe..73402af5ec4 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -3475,18 +3475,6 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveRequest) ASSERT_FALSE(exists("/T3")); } - { - SCOPED_TRACE("Recursive Remove Tree"); - create("/T4", zkutil::CreateMode::Persistent); - create("/T4/A", zkutil::CreateMode::Persistent); - - auto responses = remove_recursive("/T4", 100); - ASSERT_EQ(responses.size(), 1); - ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); - ASSERT_FALSE(exists("/T4")); - ASSERT_FALSE(exists("/T4/A")); - } - { SCOPED_TRACE("Recursive Remove Tree Small Limit"); create("/T5", zkutil::CreateMode::Persistent); @@ -3528,7 +3516,7 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveRequest) ASSERT_EQ(responses.size(), 1); ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); ASSERT_EQ(storage.ephemerals.size(), 0); - ASSERT_TRUE(!exists("/T7")); + ASSERT_FALSE(exists("/T7")); } { @@ -3543,10 +3531,10 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveRequest) ASSERT_EQ(responses.size(), 1); ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK); ASSERT_EQ(storage.ephemerals.size(), 0); - ASSERT_TRUE(!exists("/T8")); - ASSERT_TRUE(!exists("/T8/A")); - ASSERT_TRUE(!exists("/T8/B")); - ASSERT_TRUE(!exists("/T8/A/C")); + ASSERT_FALSE(exists("/T8")); + ASSERT_FALSE(exists("/T8/A")); + ASSERT_FALSE(exists("/T8/B")); + ASSERT_FALSE(exists("/T8/A/C")); } } From 9b517a939fce931da57a7d0604dc366ef35bf3fd Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 10 Sep 2024 19:53:16 +0200 Subject: [PATCH 191/215] Update 03236_squashing_high_memory.sql --- tests/queries/0_stateless/03236_squashing_high_memory.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03236_squashing_high_memory.sql b/tests/queries/0_stateless/03236_squashing_high_memory.sql index 98b33b80e9f..78316597430 100644 --- a/tests/queries/0_stateless/03236_squashing_high_memory.sql +++ b/tests/queries/0_stateless/03236_squashing_high_memory.sql @@ -5,7 +5,7 @@ DROP TABLE IF EXISTS test_table; CREATE TABLE id_values ENGINE MergeTree ORDER BY id1 AS SELECT arrayJoin(range(500000)) AS id1, arrayJoin(range(1000)) AS id2; -SET max_memory_usage = 1G; +SET max_memory_usage = '1G'; CREATE TABLE test_table ENGINE MergeTree ORDER BY id AS SELECT id_values.id1 AS id, From 63aebfa5d7d410e1b33a1f914d02a8e75f52d5ea Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 10 Sep 2024 20:15:36 +0200 Subject: [PATCH 192/215] Update 03236_squashing_high_memory.sql --- tests/queries/0_stateless/03236_squashing_high_memory.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/queries/0_stateless/03236_squashing_high_memory.sql b/tests/queries/0_stateless/03236_squashing_high_memory.sql index 78316597430..df296cb6713 100644 --- a/tests/queries/0_stateless/03236_squashing_high_memory.sql +++ b/tests/queries/0_stateless/03236_squashing_high_memory.sql @@ -1,3 +1,6 @@ +-- Tags: no-fasttest +-- reason: test requires too many rows to read + DROP TABLE IF EXISTS id_values; DROP TABLE IF EXISTS test_table; From 2f15fcd23fb069a69d2f1a0caee36ae33f39fe45 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Tue, 10 Sep 2024 20:57:03 +0200 Subject: [PATCH 193/215] Test with sparse serialization, vertical merge and skip indices --- .../03175_sparse_and_skip_index.reference | 4 ++ .../03175_sparse_and_skip_index.sql | 45 +++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 tests/queries/0_stateless/03175_sparse_and_skip_index.reference create mode 100644 tests/queries/0_stateless/03175_sparse_and_skip_index.sql diff --git a/tests/queries/0_stateless/03175_sparse_and_skip_index.reference b/tests/queries/0_stateless/03175_sparse_and_skip_index.reference new file mode 100644 index 00000000000..619e98a152a --- /dev/null +++ b/tests/queries/0_stateless/03175_sparse_and_skip_index.reference @@ -0,0 +1,4 @@ +key Sparse +value Sparse +1000 +1 diff --git a/tests/queries/0_stateless/03175_sparse_and_skip_index.sql b/tests/queries/0_stateless/03175_sparse_and_skip_index.sql new file mode 100644 index 00000000000..4de6d1ac6df --- /dev/null +++ b/tests/queries/0_stateless/03175_sparse_and_skip_index.sql @@ -0,0 +1,45 @@ +DROP TABLE IF EXISTS t_bloom_filter; +CREATE TABLE t_bloom_filter( + key UInt64, + value UInt64, + + INDEX key_bf key TYPE bloom_filter(0.01) GRANULARITY 2147483648, -- bloom filter on sorting key column + INDEX value_bf value TYPE bloom_filter(0.01) GRANULARITY 2147483648 -- bloom filter on no-sorting column +) ENGINE=MergeTree ORDER BY key +SETTINGS + -- settings to trigger sparse serialization and vertical merge + ratio_of_defaults_for_sparse_serialization = 0.0 + ,vertical_merge_algorithm_min_rows_to_activate = 1 + ,vertical_merge_algorithm_min_columns_to_activate = 1 + ,allow_vertical_merges_from_compact_to_wide_parts = 1 + ,min_bytes_for_wide_part=0 +; + +SYSTEM STOP MERGES t_bloom_filter; + +-- Create at least one part +INSERT INTO t_bloom_filter +SELECT + number % 100 as key, -- 100 unique keys + rand() % 100 as value -- 100 unique values +FROM numbers(50_000); + +-- And another part +INSERT INTO t_bloom_filter +SELECT + number % 100 as key, -- 100 unique keys + rand() % 100 as value -- 100 unique values +FROM numbers(50_000, 50_000); + +SYSTEM START MERGES t_bloom_filter; + +-- Merge everything into a single part +OPTIMIZE TABLE t_bloom_filter FINAL; + +-- Check sparse serialization +SELECT column, serialization_kind FROM system.parts_columns WHERE database = currentDatabase() AND table = 't_bloom_filter' AND active ORDER BY column; + +SELECT COUNT() FROM t_bloom_filter WHERE key = 1; + +-- Check bloom filter non-zero size +SELECT COUNT() FROM system.parts WHERE database = currentDatabase() AND table = 't_bloom_filter' AND secondary_indices_uncompressed_bytes > 200 AND active; From d4aa06524b4e65e2bca4ff851ad54dd4468e5cc0 Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Tue, 10 Sep 2024 20:57:55 +0200 Subject: [PATCH 194/215] Add materialization when building indices in vertical merge --- src/Storages/MergeTree/MergeTask.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 9a1e749734c..5c993504245 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -952,6 +952,7 @@ MergeTask::VerticalMergeRuntimeContext::PreparedColumnPipeline MergeTask::Vertic indexes_to_recalc = MergeTreeIndexFactory::instance().getMany(indexes_it->second); auto indices_expression_dag = indexes_it->second.getSingleExpressionForIndices(global_ctx->metadata_snapshot->getColumns(), global_ctx->data->getContext())->getActionsDAG().clone(); + indices_expression_dag.addMaterializingOutputActions(); /// Const columns cannot be written without materialization. auto calculate_indices_expression_step = std::make_unique( merge_column_query_plan.getCurrentDataStream(), std::move(indices_expression_dag)); From 3d36f6dce36595e8796f3d7361ca8ffc1e4bab38 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Tue, 10 Sep 2024 23:48:32 +0200 Subject: [PATCH 195/215] Update 03236_squashing_high_memory.sql --- tests/queries/0_stateless/03236_squashing_high_memory.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03236_squashing_high_memory.sql b/tests/queries/0_stateless/03236_squashing_high_memory.sql index df296cb6713..479ea87f798 100644 --- a/tests/queries/0_stateless/03236_squashing_high_memory.sql +++ b/tests/queries/0_stateless/03236_squashing_high_memory.sql @@ -6,7 +6,7 @@ DROP TABLE IF EXISTS id_values; DROP TABLE IF EXISTS test_table; CREATE TABLE id_values ENGINE MergeTree ORDER BY id1 AS - SELECT arrayJoin(range(500000)) AS id1, arrayJoin(range(1000)) AS id2; + SELECT arrayJoin(range(20000)) AS id1, arrayJoin(range(1000)) AS id2; SET max_memory_usage = '1G'; From fd3bd785b880644862f1437b99557d6643d78504 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 11 Sep 2024 11:17:06 +0200 Subject: [PATCH 196/215] Update 03236_squashing_high_memory.sql --- tests/queries/0_stateless/03236_squashing_high_memory.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03236_squashing_high_memory.sql b/tests/queries/0_stateless/03236_squashing_high_memory.sql index 479ea87f798..e1c193c3ea6 100644 --- a/tests/queries/0_stateless/03236_squashing_high_memory.sql +++ b/tests/queries/0_stateless/03236_squashing_high_memory.sql @@ -1,12 +1,14 @@ -- Tags: no-fasttest -- reason: test requires too many rows to read +SET max_rows_to_read = '501G'; + DROP TABLE IF EXISTS id_values; DROP TABLE IF EXISTS test_table; CREATE TABLE id_values ENGINE MergeTree ORDER BY id1 AS - SELECT arrayJoin(range(20000)) AS id1, arrayJoin(range(1000)) AS id2; + SELECT arrayJoin(range(500000)) AS id1, arrayJoin(range(1000)) AS id2; SET max_memory_usage = '1G'; From 55d6672bc43579dad3edaa1e8c5d595025d50fc8 Mon Sep 17 00:00:00 2001 From: Mikhail Artemenko Date: Wed, 11 Sep 2024 10:00:40 +0000 Subject: [PATCH 197/215] add early break for test keeper impl --- src/Common/ZooKeeper/TestKeeper.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Common/ZooKeeper/TestKeeper.cpp b/src/Common/ZooKeeper/TestKeeper.cpp index 3982971163c..2fbe9110b6b 100644 --- a/src/Common/ZooKeeper/TestKeeper.cpp +++ b/src/Common/ZooKeeper/TestKeeper.cpp @@ -368,6 +368,8 @@ std::pair TestKeeperRemoveRecursiveRequest::process(TestKeepe if (child_path.starts_with(path)) children.emplace_back(child_path, child_node); + else + break; } if (children.size() > remove_nodes_limit) From 3921f910f5d0d7e69581a2cc7033213809af48aa Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 11 Sep 2024 10:33:23 +0200 Subject: [PATCH 198/215] Another attempt to address EAGAIN "Resource unavailable" --- docker/test/base/setup_export_logs.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/test/base/setup_export_logs.sh b/docker/test/base/setup_export_logs.sh index 3df9655701c..7ec1c31f04a 100755 --- a/docker/test/base/setup_export_logs.sh +++ b/docker/test/base/setup_export_logs.sh @@ -187,10 +187,15 @@ function setup_logs_replication ') echo -e "Creating remote destination table ${table}_${hash} with statement:" >&2 + echo "::group::${table}" # there's the only way big "$statement" can be printed without causing EAGAIN error # cat: write error: Resource temporarily unavailable - echo "$statement" | cat + statement_print="${statement}" + if [ "${#statement_print}" -gt 4000 ]; then + statement_print="${statement::1999}\n…\n${statement:${#statement}-1999}" + fi + echo "$statement_print" echo "::endgroup::" echo "$statement" | clickhouse-client --database_replicated_initial_query_timeout_sec=10 \ From af469ffde471ff64e8e0201ceb96ba6e1a0a29c4 Mon Sep 17 00:00:00 2001 From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:32:36 +0200 Subject: [PATCH 199/215] remove sanitizer builds --- tests/queries/0_stateless/03236_squashing_high_memory.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03236_squashing_high_memory.sql b/tests/queries/0_stateless/03236_squashing_high_memory.sql index e1c193c3ea6..f6e5dbdef03 100644 --- a/tests/queries/0_stateless/03236_squashing_high_memory.sql +++ b/tests/queries/0_stateless/03236_squashing_high_memory.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, no-asan, no-tsan, no-msan, no-ubsan -- reason: test requires too many rows to read SET max_rows_to_read = '501G'; From c1830bc041a067d1fae1e8971091c453614eaca7 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 11 Sep 2024 13:08:58 +0200 Subject: [PATCH 200/215] Escape the `\n` in statements --- docker/test/base/setup_export_logs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/base/setup_export_logs.sh b/docker/test/base/setup_export_logs.sh index 7ec1c31f04a..a39f96867be 100755 --- a/docker/test/base/setup_export_logs.sh +++ b/docker/test/base/setup_export_logs.sh @@ -195,7 +195,7 @@ function setup_logs_replication if [ "${#statement_print}" -gt 4000 ]; then statement_print="${statement::1999}\n…\n${statement:${#statement}-1999}" fi - echo "$statement_print" + echo -e "$statement_print" echo "::endgroup::" echo "$statement" | clickhouse-client --database_replicated_initial_query_timeout_sec=10 \ From 6f9d1631e26bbb0263962c6d50c65e9ee2be561a Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 11 Sep 2024 13:11:29 +0200 Subject: [PATCH 201/215] Update aspell-ignore --- utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 796df6f1042..3467f21c812 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -24,6 +24,7 @@ Aggregatefunction AggregatingMergeTree AggregatorThreads AggregatorThreadsActive +AzureQueue Akka AlertManager Alexey From 9e335abe1735eeed100c4b964bf24220b6086bf9 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 11 Sep 2024 13:05:24 +0000 Subject: [PATCH 202/215] CMake: Add comment about ICU data files --- contrib/icu-cmake/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/icu-cmake/CMakeLists.txt b/contrib/icu-cmake/CMakeLists.txt index afaa189701d..5714fef8347 100644 --- a/contrib/icu-cmake/CMakeLists.txt +++ b/contrib/icu-cmake/CMakeLists.txt @@ -481,6 +481,11 @@ if (ARCH_S390X) else() set(ICUDATA_SOURCE_FILE "${ICUDATA_SOURCE_DIR}/icudt75l_dat.S" ) endif() +# ^^ you might be confused how for different little endian platforms (x86, ARM) the same assembly files can be used. +# These files are indeed assembly but they only contain data ('.long' directive), which makes them portable accross CPUs. +# Only the endianness and the character set (ASCII, EBCDIC) makes a difference, also see +# https://unicode-org.github.io/icu/userguide/icu_data/#sharing-icu-data-between-platforms, 'Sharing ICU Data Between Platforms') +# (and as an experiment, try re-generating the data files on x86 vs. ARM, ... you'll get exactly the same files) set(ICUDATA_SOURCES "${ICUDATA_SOURCE_FILE}" From a05610c38fe3b02a9cf66838c15ebc3612a6a795 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 11 Sep 2024 15:25:32 +0200 Subject: [PATCH 203/215] Move creation and validation of table metadata before creating queue metadata & save to factory --- .../ObjectStorageQueueMetadata.cpp | 55 ++++--- .../ObjectStorageQueueMetadata.h | 25 ++- .../ObjectStorageQueueMetadataFactory.cpp | 14 +- .../ObjectStorageQueueMetadataFactory.h | 4 +- .../ObjectStorageQueueTableMetadata.cpp | 154 +++++------------- .../ObjectStorageQueueTableMetadata.h | 29 ++-- .../StorageObjectStorageQueue.cpp | 27 ++- 7 files changed, 133 insertions(+), 175 deletions(-) diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp index 23ac92b667a..e424769b061 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp @@ -33,7 +33,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; extern const int REPLICA_ALREADY_EXISTS; - extern const int INCOMPATIBLE_COLUMNS; } namespace @@ -108,8 +107,12 @@ private: } }; -ObjectStorageQueueMetadata::ObjectStorageQueueMetadata(const fs::path & zookeeper_path_, const ObjectStorageQueueSettings & settings_) +ObjectStorageQueueMetadata::ObjectStorageQueueMetadata( + const fs::path & zookeeper_path_, + const ObjectStorageQueueTableMetadata & table_metadata_, + const ObjectStorageQueueSettings & settings_) : settings(settings_) + , table_metadata(table_metadata_) , zookeeper_path(zookeeper_path_) , buckets_num(getBucketsNum(settings_)) , log(getLogger("StorageObjectStorageQueue(" + zookeeper_path_.string() + ")")) @@ -144,11 +147,6 @@ void ObjectStorageQueueMetadata::shutdown() task->deactivate(); } -void ObjectStorageQueueMetadata::checkSettings(const ObjectStorageQueueSettings & settings_) const -{ - ObjectStorageQueueTableMetadata::checkEquals(settings, settings_); -} - ObjectStorageQueueMetadata::FileStatusPtr ObjectStorageQueueMetadata::getFileStatus(const std::string & path) { return local_file_statuses->get(path, /* create */false); @@ -219,13 +217,14 @@ ObjectStorageQueueMetadata::tryAcquireBucket(const Bucket & bucket, const Proces return ObjectStorageQueueOrderedFileMetadata::tryAcquireBucket(zookeeper_path, bucket, processor, log); } -void ObjectStorageQueueMetadata::initialize( - const ConfigurationPtr & configuration, - const StorageInMemoryMetadata & storage_metadata) +void ObjectStorageQueueMetadata::syncWithKeeper( + const fs::path & zookeeper_path, + const ObjectStorageQueueTableMetadata & table_metadata, + const ObjectStorageQueueSettings & settings, + LoggerPtr log) { - const auto metadata_from_table = ObjectStorageQueueTableMetadata(*configuration, settings, storage_metadata); - const auto & columns_from_table = storage_metadata.getColumns(); const auto table_metadata_path = zookeeper_path / "metadata"; + const auto buckets_num = getBucketsNum(settings); const auto metadata_paths = settings.mode == ObjectStorageQueueMode::ORDERED ? ObjectStorageQueueOrderedFileMetadata::getMetadataPaths(buckets_num) : ObjectStorageQueueUnorderedFileMetadata::getMetadataPaths(); @@ -237,24 +236,17 @@ void ObjectStorageQueueMetadata::initialize( { if (zookeeper->exists(table_metadata_path)) { - const auto metadata_from_zk = ObjectStorageQueueTableMetadata::parse(zookeeper->get(fs::path(zookeeper_path) / "metadata")); - const auto columns_from_zk = ColumnsDescription::parse(metadata_from_zk.columns); + const auto metadata_str = zookeeper->get(fs::path(zookeeper_path) / "metadata"); + const auto metadata_from_zk = ObjectStorageQueueTableMetadata::parse(metadata_str); - metadata_from_table.checkEquals(metadata_from_zk); - if (columns_from_zk != columns_from_table) - { - throw Exception( - ErrorCodes::INCOMPATIBLE_COLUMNS, - "Table columns structure in ZooKeeper is different from local table structure. " - "Local columns:\n{}\nZookeeper columns:\n{}", - columns_from_table.toString(), columns_from_zk.toString()); - } + table_metadata.checkEquals(metadata_from_zk); return; } Coordination::Requests requests; requests.emplace_back(zkutil::makeCreateRequest(zookeeper_path, "", zkutil::CreateMode::Persistent)); - requests.emplace_back(zkutil::makeCreateRequest(table_metadata_path, metadata_from_table.toString(), zkutil::CreateMode::Persistent)); + requests.emplace_back(zkutil::makeCreateRequest( + table_metadata_path, table_metadata.toString(), zkutil::CreateMode::Persistent)); for (const auto & path : metadata_paths) { @@ -263,16 +255,27 @@ void ObjectStorageQueueMetadata::initialize( } if (!settings.last_processed_path.value.empty()) - getFileMetadata(settings.last_processed_path)->setProcessedAtStartRequests(requests, zookeeper); + { + ObjectStorageQueueOrderedFileMetadata( + zookeeper_path, + settings.last_processed_path, + std::make_shared(), + /* bucket_info */nullptr, + buckets_num, + settings.loading_retries, + log).setProcessedAtStartRequests(requests, zookeeper); + } Coordination::Responses responses; auto code = zookeeper->tryMulti(requests, responses); if (code == Coordination::Error::ZNODEEXISTS) { auto exception = zkutil::KeeperMultiException(code, requests, responses); + LOG_INFO(log, "Got code `{}` for path: {}. " "It looks like the table {} was created by another server at the same moment, " - "will retry", code, exception.getPathForFirstFailedOp(), zookeeper_path.string()); + "will retry", + code, exception.getPathForFirstFailedOp(), zookeeper_path.string()); continue; } else if (code != Coordination::Error::ZOK) diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h index e5fae047ac5..71d26ca7c47 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.h @@ -8,6 +8,7 @@ #include #include #include +#include #include namespace fs = std::filesystem; @@ -52,11 +53,19 @@ public: using Bucket = size_t; using Processor = std::string; - ObjectStorageQueueMetadata(const fs::path & zookeeper_path_, const ObjectStorageQueueSettings & settings_); + ObjectStorageQueueMetadata( + const fs::path & zookeeper_path_, + const ObjectStorageQueueTableMetadata & table_metadata_, + const ObjectStorageQueueSettings & settings_); + ~ObjectStorageQueueMetadata(); - void initialize(const ConfigurationPtr & configuration, const StorageInMemoryMetadata & storage_metadata); - void checkSettings(const ObjectStorageQueueSettings & settings) const; + static void syncWithKeeper( + const fs::path & zookeeper_path, + const ObjectStorageQueueTableMetadata & table_metadata, + const ObjectStorageQueueSettings & settings, + LoggerPtr log); + void shutdown(); FileMetadataPtr getFileMetadata(const std::string & path, ObjectStorageQueueOrderedFileMetadata::BucketInfoPtr bucket_info = {}); @@ -72,11 +81,17 @@ public: static size_t getBucketsNum(const ObjectStorageQueueSettings & settings); static size_t getBucketsNum(const ObjectStorageQueueTableMetadata & settings); + void checkTableMetadataEquals(const ObjectStorageQueueMetadata & other); + + const ObjectStorageQueueTableMetadata & getTableMetadata() const { return table_metadata; } + ObjectStorageQueueTableMetadata & getTableMetadata() { return table_metadata; } + private: void cleanupThreadFunc(); void cleanupThreadFuncImpl(); - const ObjectStorageQueueSettings settings; + ObjectStorageQueueSettings settings; + ObjectStorageQueueTableMetadata table_metadata; const fs::path zookeeper_path; const size_t buckets_num; @@ -89,4 +104,6 @@ private: std::shared_ptr local_file_statuses; }; +using ObjectStorageQueueMetadataPtr = std::unique_ptr; + } diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp index ffae33d6f41..ba98711eff9 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.cpp @@ -14,19 +14,23 @@ ObjectStorageQueueMetadataFactory & ObjectStorageQueueMetadataFactory::instance( return ret; } -ObjectStorageQueueMetadataFactory::FilesMetadataPtr -ObjectStorageQueueMetadataFactory::getOrCreate(const std::string & zookeeper_path, const ObjectStorageQueueSettings & settings) +ObjectStorageQueueMetadataFactory::FilesMetadataPtr ObjectStorageQueueMetadataFactory::getOrCreate( + const std::string & zookeeper_path, + ObjectStorageQueueMetadataPtr metadata) { std::lock_guard lock(mutex); auto it = metadata_by_path.find(zookeeper_path); if (it == metadata_by_path.end()) { - auto files_metadata = std::make_shared(zookeeper_path, settings); - it = metadata_by_path.emplace(zookeeper_path, std::move(files_metadata)).first; + it = metadata_by_path.emplace(zookeeper_path, std::move(metadata)).first; } else { - it->second.metadata->checkSettings(settings); + auto & metadata_from_table = metadata->getTableMetadata(); + auto & metadata_from_keeper = it->second.metadata->getTableMetadata(); + + metadata_from_table.checkEquals(metadata_from_keeper); + it->second.ref_count += 1; } return it->second.metadata; diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h index a93f5ee3d83..a9975c526ef 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadataFactory.h @@ -13,7 +13,9 @@ public: static ObjectStorageQueueMetadataFactory & instance(); - FilesMetadataPtr getOrCreate(const std::string & zookeeper_path, const ObjectStorageQueueSettings & settings); + FilesMetadataPtr getOrCreate( + const std::string & zookeeper_path, + ObjectStorageQueueMetadataPtr metadata); void remove(const std::string & zookeeper_path); diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp index cb9cdf8e186..c2f70e3e804 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp @@ -1,6 +1,5 @@ #include -#include #include #include #include @@ -32,18 +31,18 @@ namespace ObjectStorageQueueTableMetadata::ObjectStorageQueueTableMetadata( - const StorageObjectStorage::Configuration & configuration, const ObjectStorageQueueSettings & engine_settings, - const StorageInMemoryMetadata & storage_metadata) + const ColumnsDescription & columns_, + const std::string & format_) + : format_name(format_) + , columns(columns_.toString()) + , after_processing(engine_settings.after_processing.toString()) + , mode(engine_settings.mode.toString()) + , tracked_files_limit(engine_settings.tracked_files_limit) + , tracked_file_ttl_sec(engine_settings.tracked_file_ttl_sec) + , buckets(engine_settings.buckets) + , processing_threads_num(engine_settings.processing_threads_num) { - format_name = configuration.format; - after_processing = engine_settings.after_processing.toString(); - mode = engine_settings.mode.toString(); - tracked_files_limit = engine_settings.tracked_files_limit; - tracked_file_ttl_sec = engine_settings.tracked_file_ttl_sec; - buckets = engine_settings.buckets; - processing_threads_num = engine_settings.processing_threads_num; - columns = storage_metadata.getColumns().toString(); } String ObjectStorageQueueTableMetadata::toString() const @@ -65,48 +64,40 @@ String ObjectStorageQueueTableMetadata::toString() const return oss.str(); } -void ObjectStorageQueueTableMetadata::read(const String & metadata_str) +template +static auto getOrDefault( + const Poco::JSON::Object::Ptr & json, + const std::string & setting, + const std::string & compatibility_prefix, + const T & default_value) { - Poco::JSON::Parser parser; - auto json = parser.parse(metadata_str).extract(); + if (!compatibility_prefix.empty() && json->has(compatibility_prefix + setting)) + return json->getValue(compatibility_prefix + setting); - after_processing = json->getValue("after_processing"); - mode = json->getValue("mode"); + if (json->has(setting)) + return json->getValue(setting); - format_name = json->getValue("format_name"); - columns = json->getValue("columns"); + return default_value; +} - /// Check with "s3queue_" prefix for compatibility. - { - if (json->has("s3queue_tracked_files_limit")) - tracked_files_limit = json->getValue("s3queue_tracked_files_limit"); - if (json->has("s3queue_tracked_file_ttl_sec")) - tracked_file_ttl_sec = json->getValue("s3queue_tracked_file_ttl_sec"); - if (json->has("s3queue_processing_threads_num")) - processing_threads_num = json->getValue("s3queue_processing_threads_num"); - } - - if (json->has("tracked_files_limit")) - tracked_files_limit = json->getValue("tracked_files_limit"); - - if (json->has("tracked_file_ttl_sec")) - tracked_file_ttl_sec = json->getValue("tracked_file_ttl_sec"); - - if (json->has("last_processed_file")) - last_processed_path = json->getValue("last_processed_file"); - - if (json->has("processing_threads_num")) - processing_threads_num = json->getValue("processing_threads_num"); - - if (json->has("buckets")) - buckets = json->getValue("buckets"); +ObjectStorageQueueTableMetadata::ObjectStorageQueueTableMetadata(const Poco::JSON::Object::Ptr & json) + : format_name(json->getValue("format_name")) + , columns(json->getValue("columns")) + , after_processing(json->getValue("after_processing")) + , mode(json->getValue("mode")) + , tracked_files_limit(getOrDefault(json, "tracked_files_limit", "s3queue_", 0)) + , tracked_file_ttl_sec(getOrDefault(json, "tracked_files_ttl_sec", "s3queue_", 0)) + , buckets(getOrDefault(json, "buckets", "", 0)) + , processing_threads_num(getOrDefault(json, "processing_threads_num", "s3queue_", 0)) + , last_processed_path(getOrDefault(json, "last_processed_file", "s3queue_", "")) +{ } ObjectStorageQueueTableMetadata ObjectStorageQueueTableMetadata::parse(const String & metadata_str) { - ObjectStorageQueueTableMetadata metadata; - metadata.read(metadata_str); - return metadata; + Poco::JSON::Parser parser; + auto json = parser.parse(metadata_str).extract(); + return ObjectStorageQueueTableMetadata(json); } void ObjectStorageQueueTableMetadata::checkEquals(const ObjectStorageQueueTableMetadata & from_zk) const @@ -181,72 +172,17 @@ void ObjectStorageQueueTableMetadata::checkImmutableFieldsEquals(const ObjectSto ErrorCodes::METADATA_MISMATCH, "Existing table metadata in ZooKeeper differs in processing buckets. " "Stored in ZooKeeper: {}, local: {}", - ObjectStorageQueueMetadata::getBucketsNum(*this), ObjectStorageQueueMetadata::getBucketsNum(from_zk)); + ObjectStorageQueueMetadata::getBucketsNum(from_zk), ObjectStorageQueueMetadata::getBucketsNum(*this)); } } + + if (columns != from_zk.columns) + throw Exception( + ErrorCodes::METADATA_MISMATCH, + "Existing table metadata in ZooKeeper differs in columns. " + "Stored in ZooKeeper: {}, local: {}", + from_zk.columns, + columns); } -void ObjectStorageQueueTableMetadata::checkEquals(const ObjectStorageQueueSettings & current, const ObjectStorageQueueSettings & expected) -{ - if (current.after_processing != expected.after_processing) - throw Exception( - ErrorCodes::METADATA_MISMATCH, - "Existing table metadata in ZooKeeper differs " - "in action after processing. Stored in ZooKeeper: {}, local: {}", - expected.after_processing.toString(), - current.after_processing.toString()); - - if (current.mode != expected.mode) - throw Exception( - ErrorCodes::METADATA_MISMATCH, - "Existing table metadata in ZooKeeper differs in engine mode. " - "Stored in ZooKeeper: {}, local: {}", - expected.mode.toString(), - current.mode.toString()); - - if (current.tracked_files_limit != expected.tracked_files_limit) - throw Exception( - ErrorCodes::METADATA_MISMATCH, - "Existing table metadata in ZooKeeper differs in max set size. " - "Stored in ZooKeeper: {}, local: {}", - expected.tracked_files_limit, - current.tracked_files_limit); - - if (current.tracked_file_ttl_sec != expected.tracked_file_ttl_sec) - throw Exception( - ErrorCodes::METADATA_MISMATCH, - "Existing table metadata in ZooKeeper differs in max set age. " - "Stored in ZooKeeper: {}, local: {}", - expected.tracked_file_ttl_sec, - current.tracked_file_ttl_sec); - - if (current.last_processed_path.value != expected.last_processed_path.value) - throw Exception( - ErrorCodes::METADATA_MISMATCH, - "Existing table metadata in ZooKeeper differs in last_processed_path. " - "Stored in ZooKeeper: {}, local: {}", - expected.last_processed_path.value, - current.last_processed_path.value); - - if (current.mode == ObjectStorageQueueMode::ORDERED) - { - if (current.buckets != expected.buckets) - { - throw Exception( - ErrorCodes::METADATA_MISMATCH, - "Existing table metadata in ZooKeeper differs in buckets setting. " - "Stored in ZooKeeper: {}, local: {}", - expected.buckets, current.buckets); - } - - if (ObjectStorageQueueMetadata::getBucketsNum(current) != ObjectStorageQueueMetadata::getBucketsNum(expected)) - { - throw Exception( - ErrorCodes::METADATA_MISMATCH, - "Existing table metadata in ZooKeeper differs in processing buckets. " - "Stored in ZooKeeper: {}, local: {}", - ObjectStorageQueueMetadata::getBucketsNum(current), ObjectStorageQueueMetadata::getBucketsNum(expected)); - } - } -} } diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.h index bbae06b66c6..d70b859ae1d 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.h @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include namespace DB @@ -16,29 +18,28 @@ class ReadBuffer; */ struct ObjectStorageQueueTableMetadata { - String format_name; - String columns; - String after_processing; - String mode; - UInt64 tracked_files_limit = 0; - UInt64 tracked_file_ttl_sec = 0; - UInt64 buckets = 0; - UInt64 processing_threads_num = 1; - String last_processed_path; + const String format_name; + const String columns; + const String after_processing; + const String mode; + const UInt64 tracked_files_limit; + const UInt64 tracked_file_ttl_sec; + const UInt64 buckets; + const UInt64 processing_threads_num; + const String last_processed_path; - ObjectStorageQueueTableMetadata() = default; ObjectStorageQueueTableMetadata( - const StorageObjectStorage::Configuration & configuration, const ObjectStorageQueueSettings & engine_settings, - const StorageInMemoryMetadata & storage_metadata); + const ColumnsDescription & columns_, + const std::string & format_); + + explicit ObjectStorageQueueTableMetadata(const Poco::JSON::Object::Ptr & json); - void read(const String & metadata_str); static ObjectStorageQueueTableMetadata parse(const String & metadata_str); String toString() const; void checkEquals(const ObjectStorageQueueTableMetadata & from_zk) const; - static void checkEquals(const ObjectStorageQueueSettings & current, const ObjectStorageQueueSettings & expected); private: void checkImmutableFieldsEquals(const ObjectStorageQueueTableMetadata & from_zk) const; diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index c1ef37e1a48..8f11836a11b 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -85,7 +85,10 @@ namespace } } - std::shared_ptr getQueueLog(const ObjectStoragePtr & storage, const ContextPtr & context, const ObjectStorageQueueSettings & table_settings) + std::shared_ptr getQueueLog( + const ObjectStoragePtr & storage, + const ContextPtr & context, + const ObjectStorageQueueSettings & table_settings) { const auto & settings = context->getSettingsRef(); switch (storage->getType()) @@ -105,7 +108,6 @@ namespace default: throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected object storage type: {}", storage->getType()); } - } } @@ -161,21 +163,14 @@ StorageObjectStorageQueue::StorageObjectStorageQueue( setInMemoryMetadata(storage_metadata); LOG_INFO(log, "Using zookeeper path: {}", zk_path.string()); - task = getContext()->getSchedulePool().createTask("ObjectStorageQueueStreamingTask", [this] { threadFunc(); }); - /// Get metadata manager from ObjectStorageQueueMetadataFactory, - /// it will increase the ref count for the metadata object. - /// The ref count is decreased when StorageObjectStorageQueue::drop() method is called. - files_metadata = ObjectStorageQueueMetadataFactory::instance().getOrCreate(zk_path, *queue_settings); - try - { - files_metadata->initialize(configuration_, storage_metadata); - } - catch (...) - { - ObjectStorageQueueMetadataFactory::instance().remove(zk_path); - throw; - } + ObjectStorageQueueTableMetadata table_metadata(*queue_settings, storage_metadata.getColumns(), configuration_->format); + ObjectStorageQueueMetadata::syncWithKeeper(zk_path, table_metadata, *queue_settings, log); + + auto queue_metadata = std::make_unique(zk_path, std::move(table_metadata), *queue_settings); + files_metadata = ObjectStorageQueueMetadataFactory::instance().getOrCreate(zk_path, std::move(queue_metadata)); + + task = getContext()->getSchedulePool().createTask("ObjectStorageQueueStreamingTask", [this] { threadFunc(); }); } void StorageObjectStorageQueue::startup() From 3aef79b5a265802c536195653469e8d3f88128ec Mon Sep 17 00:00:00 2001 From: Alexander Gololobov Date: Wed, 11 Sep 2024 15:45:55 +0200 Subject: [PATCH 204/215] Backport changes from private to minimize merge conflicts --- src/Storages/MergeTree/MergeTask.cpp | 37 ++++++++++++++-------------- src/Storages/MergeTree/MergeTask.h | 18 +++++--------- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 5c993504245..e04d6afbf7a 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -125,19 +125,19 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::extractMergingAndGatheringColu std::set key_columns(sort_key_columns_vec.cbegin(), sort_key_columns_vec.cend()); /// Force sign column for Collapsing mode - if (ctx->merging_params.mode == MergeTreeData::MergingParams::Collapsing) - key_columns.emplace(ctx->merging_params.sign_column); + if (global_ctx->merging_params.mode == MergeTreeData::MergingParams::Collapsing) + key_columns.emplace(global_ctx->merging_params.sign_column); /// Force version column for Replacing mode - if (ctx->merging_params.mode == MergeTreeData::MergingParams::Replacing) + if (global_ctx->merging_params.mode == MergeTreeData::MergingParams::Replacing) { - key_columns.emplace(ctx->merging_params.is_deleted_column); - key_columns.emplace(ctx->merging_params.version_column); + key_columns.emplace(global_ctx->merging_params.is_deleted_column); + key_columns.emplace(global_ctx->merging_params.version_column); } /// Force sign column for VersionedCollapsing mode. Version is already in primary key. - if (ctx->merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing) - key_columns.emplace(ctx->merging_params.sign_column); + if (global_ctx->merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing) + key_columns.emplace(global_ctx->merging_params.sign_column); /// Force to merge at least one column in case of empty key if (key_columns.empty()) @@ -206,7 +206,8 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const // E.g. `proj_a.proj` for a normal projection merge and `proj_a.tmp_proj` for a projection materialization merge. local_tmp_prefix = global_ctx->parent_part ? "" : "tmp_merge_"; } - const String local_tmp_suffix = global_ctx->parent_part ? ctx->suffix : ""; + + const String local_tmp_suffix = global_ctx->parent_part ? global_ctx->suffix : ""; if (global_ctx->merges_blocker->isCancelled() || global_ctx->merge_list_element_ptr->is_cancelled.load(std::memory_order_relaxed)) throw Exception(ErrorCodes::ABORTED, "Cancelled merging parts"); @@ -231,7 +232,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const LOG_DEBUG(ctx->log, "DEDUPLICATE BY ('{}')", fmt::join(global_ctx->deduplicate_by_columns, "', '")); } - ctx->disk = global_ctx->space_reservation->getDisk(); + global_ctx->disk = global_ctx->space_reservation->getDisk(); auto local_tmp_part_basename = local_tmp_prefix + global_ctx->future_part->name + local_tmp_suffix; std::optional builder; @@ -243,7 +244,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() const } else { - auto local_single_disk_volume = std::make_shared("volume_" + global_ctx->future_part->name, ctx->disk, 0); + auto local_single_disk_volume = std::make_shared("volume_" + global_ctx->future_part->name, global_ctx->disk, 0); builder.emplace(global_ctx->data->getDataPartBuilder(global_ctx->future_part->name, local_single_disk_volume, local_tmp_part_basename)); builder->withPartStorageType(global_ctx->future_part->part_format.storage_type); } @@ -559,9 +560,9 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::prepareProjectionsToMergeAndRe const bool merge_may_reduce_rows = global_ctx->cleanup || global_ctx->deduplicate || - ctx->merging_params.mode == MergeTreeData::MergingParams::Collapsing || - ctx->merging_params.mode == MergeTreeData::MergingParams::Replacing || - ctx->merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing; + global_ctx->merging_params.mode == MergeTreeData::MergingParams::Collapsing || + global_ctx->merging_params.mode == MergeTreeData::MergingParams::Replacing || + global_ctx->merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing; const auto & projections = global_ctx->metadata_snapshot->getProjections(); @@ -1616,7 +1617,7 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const merge_parts_query_plan.getCurrentDataStream(), sort_description, partition_key_columns, - ctx->merging_params, + global_ctx->merging_params, ctx->rows_sources_write_buf.get(), data_settings->merge_max_block_size, data_settings->merge_max_block_size_bytes, @@ -1726,10 +1727,10 @@ MergeAlgorithm MergeTask::ExecuteAndFinalizeHorizontalPart::chooseMergeAlgorithm } bool is_supported_storage = - ctx->merging_params.mode == MergeTreeData::MergingParams::Ordinary || - ctx->merging_params.mode == MergeTreeData::MergingParams::Collapsing || - ctx->merging_params.mode == MergeTreeData::MergingParams::Replacing || - ctx->merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing; + global_ctx->merging_params.mode == MergeTreeData::MergingParams::Ordinary || + global_ctx->merging_params.mode == MergeTreeData::MergingParams::Collapsing || + global_ctx->merging_params.mode == MergeTreeData::MergingParams::Replacing || + global_ctx->merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing; bool enough_ordinary_cols = global_ctx->gathering_columns.size() >= data_settings->vertical_merge_algorithm_min_columns_to_activate; diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index 84f00d521fd..fe12fba8aab 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -100,6 +100,7 @@ public: global_ctx->context = std::move(context_); global_ctx->holder = &holder; global_ctx->space_reservation = std::move(space_reservation_); + global_ctx->disk = global_ctx->space_reservation->getDisk(); global_ctx->deduplicate = std::move(deduplicate_); global_ctx->deduplicate_by_columns = std::move(deduplicate_by_columns_); global_ctx->cleanup = std::move(cleanup_); @@ -110,12 +111,10 @@ public: global_ctx->ttl_merges_blocker = std::move(ttl_merges_blocker_); global_ctx->txn = std::move(txn); global_ctx->need_prefix = need_prefix; + global_ctx->suffix = std::move(suffix_); + global_ctx->merging_params = std::move(merging_params_); auto prepare_stage_ctx = std::make_shared(); - - prepare_stage_ctx->suffix = std::move(suffix_); - prepare_stage_ctx->merging_params = std::move(merging_params_); - (*stages.begin())->setRuntimeContext(std::move(prepare_stage_ctx), global_ctx); } @@ -172,6 +171,7 @@ private: ContextPtr context{nullptr}; time_t time_of_merge{0}; ReservationSharedPtr space_reservation{nullptr}; + DiskPtr disk{nullptr}; bool deduplicate{false}; Names deduplicate_by_columns{}; bool cleanup{false}; @@ -210,6 +210,8 @@ private: MergeTreeTransactionPtr txn; bool need_prefix; + String suffix; + MergeTreeData::MergingParams merging_params{}; scope_guard temporary_directory_lock; UInt64 prev_elapsed_ms{0}; @@ -222,13 +224,7 @@ private: /// Proper initialization is responsibility of the author struct ExecuteAndFinalizeHorizontalPartRuntimeContext : public IStageRuntimeContext { - /// Dependencies - String suffix; - bool need_prefix; - MergeTreeData::MergingParams merging_params{}; - TemporaryDataOnDiskPtr tmp_disk{nullptr}; - DiskPtr disk{nullptr}; bool need_remove_expired_values{false}; bool force_ttl{false}; CompressionCodecPtr compression_codec{nullptr}; @@ -264,7 +260,6 @@ private: using ExecuteAndFinalizeHorizontalPartRuntimeContextPtr = std::shared_ptr; - struct ExecuteAndFinalizeHorizontalPart : public IStage { bool execute() override; @@ -356,7 +351,6 @@ private: using VerticalMergeRuntimeContextPtr = std::shared_ptr; - struct VerticalMergeStage : public IStage { bool execute() override; From 22ceaa00669d19c5f353b4d4b496244ccbeb25ba Mon Sep 17 00:00:00 2001 From: Alex Katsman Date: Wed, 4 Sep 2024 16:50:22 +0000 Subject: [PATCH 205/215] Use another error code for external data in buffer Use CANNOT_DECOMPRESS error code for the compressed external data in CompressedReadBufferBase for checksum validation and decompression. --- src/Compression/CompressedReadBufferBase.cpp | 18 ++++++++++-------- src/Server/TCPHandler.cpp | 4 ++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Compression/CompressedReadBufferBase.cpp b/src/Compression/CompressedReadBufferBase.cpp index e416fadc829..907e87a6d30 100644 --- a/src/Compression/CompressedReadBufferBase.cpp +++ b/src/Compression/CompressedReadBufferBase.cpp @@ -39,7 +39,7 @@ using Checksum = CityHash_v1_0_2::uint128; /// Validate checksum of data, and if it mismatches, find out possible reason and throw exception. -static void validateChecksum(char * data, size_t size, const Checksum expected_checksum) +static void validateChecksum(char * data, size_t size, const Checksum expected_checksum, bool external_data) { auto calculated_checksum = CityHash_v1_0_2::CityHash128(data, size); if (expected_checksum == calculated_checksum) @@ -64,6 +64,8 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c "this can be caused by disk bit rot. This exception protects ClickHouse " "from data corruption due to hardware failures."; + int error_code = external_data ? ErrorCodes::CANNOT_DECOMPRESS : ErrorCodes::CHECKSUM_DOESNT_MATCH; + auto flip_bit = [](char * buf, size_t pos) { buf[pos / 8] ^= 1 << pos % 8; @@ -87,7 +89,7 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c { message << ". The mismatch is caused by single bit flip in data block at byte " << (bit_pos / 8) << ", bit " << (bit_pos % 8) << ". " << message_hardware_failure; - throw Exception::createDeprecated(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); + throw Exception::createDeprecated(message.str(), error_code); } flip_bit(tmp_data, bit_pos); /// Restore @@ -102,10 +104,10 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c { message << ". The mismatch is caused by single bit flip in checksum. " << message_hardware_failure; - throw Exception::createDeprecated(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); + throw Exception::createDeprecated(message.str(), error_code); } - throw Exception::createDeprecated(message.str(), ErrorCodes::CHECKSUM_DOESNT_MATCH); + throw Exception::createDeprecated(message.str(), error_code); } static void readHeaderAndGetCodecAndSize( @@ -151,7 +153,7 @@ static void readHeaderAndGetCodecAndSize( "Most likely corrupted data.", size_compressed_without_checksum); if (size_compressed_without_checksum < header_size) - throw Exception(ErrorCodes::CORRUPTED_DATA, "Can't decompress data: " + throw Exception(external_data ? ErrorCodes::CANNOT_DECOMPRESS : ErrorCodes::CORRUPTED_DATA, "Can't decompress data: " "the compressed data size ({}, this should include header size) is less than the header size ({})", size_compressed_without_checksum, static_cast(header_size)); } @@ -202,7 +204,7 @@ size_t CompressedReadBufferBase::readCompressedData(size_t & size_decompressed, readBinaryLittleEndian(checksum.low64, checksum_in); readBinaryLittleEndian(checksum.high64, checksum_in); - validateChecksum(compressed_buffer, size_compressed_without_checksum, checksum); + validateChecksum(compressed_buffer, size_compressed_without_checksum, checksum, external_data); } ProfileEvents::increment(ProfileEvents::ReadCompressedBytes, size_compressed_without_checksum + sizeof(Checksum)); @@ -247,7 +249,7 @@ size_t CompressedReadBufferBase::readCompressedDataBlockForAsynchronous(size_t & readBinaryLittleEndian(checksum.low64, checksum_in); readBinaryLittleEndian(checksum.high64, checksum_in); - validateChecksum(compressed_buffer, size_compressed_without_checksum, checksum); + validateChecksum(compressed_buffer, size_compressed_without_checksum, checksum, external_data); } ProfileEvents::increment(ProfileEvents::ReadCompressedBytes, size_compressed_without_checksum + sizeof(Checksum)); @@ -307,7 +309,7 @@ void CompressedReadBufferBase::decompress(BufferBase::Buffer & to, size_t size_d UInt8 header_size = ICompressionCodec::getHeaderSize(); if (size_compressed_without_checksum < header_size) - throw Exception(ErrorCodes::CORRUPTED_DATA, + throw Exception(external_data ? ErrorCodes::CANNOT_DECOMPRESS : ErrorCodes::CORRUPTED_DATA, "Can't decompress data: the compressed data size ({}, this should include header size) is less than the header size ({})", size_compressed_without_checksum, static_cast(header_size)); diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 2b9a7295198..8083f7da24d 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -2133,7 +2133,7 @@ bool TCPHandler::receiveUnexpectedData(bool throw_exception) std::shared_ptr maybe_compressed_in; if (last_block_in.compression == Protocol::Compression::Enable) - maybe_compressed_in = std::make_shared(*in, /* allow_different_codecs */ true); + maybe_compressed_in = std::make_shared(*in, /* allow_different_codecs */ true, /* external_data */ query_kind != ClientInfo::QueryKind::SECONDARY_QUERY); else maybe_compressed_in = in; @@ -2157,7 +2157,7 @@ void TCPHandler::initBlockInput() /// with another codec that the rest of the data. Example: data sent by Distributed tables. if (state.compression == Protocol::Compression::Enable) - state.maybe_compressed_in = std::make_shared(*in, /* allow_different_codecs */ true); + state.maybe_compressed_in = std::make_shared(*in, /* allow_different_codecs */ true, /* external_data */ query_kind != ClientInfo::QueryKind::SECONDARY_QUERY); else state.maybe_compressed_in = in; From abe8f29bf8d282951f30d47bf03d3d3fcc024a17 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 11 Sep 2024 17:32:20 +0200 Subject: [PATCH 206/215] Update test --- tests/integration/test_storage_s3_queue/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index 9e3ee19179a..8b492018766 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -663,7 +663,7 @@ def test_multiple_tables_meta_mismatch(started_cluster): ) except QueryRuntimeException as e: assert ( - "Table columns structure in ZooKeeper is different from local table structure" + "Existing table metadata in ZooKeeper differs in columns" in str(e) ) failed = True From 9a2a6d71d2ec577ae3787a3f63cb1c3ee7b37943 Mon Sep 17 00:00:00 2001 From: Alex Katsman Date: Wed, 11 Sep 2024 15:09:03 +0000 Subject: [PATCH 207/215] Fix check in test_broken_projections --- .../test_broken_projections/test.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/integration/test_broken_projections/test.py b/tests/integration/test_broken_projections/test.py index 578ff42369c..d161f3fba78 100644 --- a/tests/integration/test_broken_projections/test.py +++ b/tests/integration/test_broken_projections/test.py @@ -735,11 +735,14 @@ def test_mutation_with_broken_projection(cluster): f"ALTER TABLE {table_name} DELETE WHERE _part == 'all_0_0_0_4' SETTINGS mutations_sync = 1" ) + parts = get_parts(node, table_name) # All parts changes because this is how alter delete works, # but all parts apart from the first have only hardlinks to files in previous part. - assert ["all_0_0_0_5", "all_1_1_0_5", "all_2_2_0_5", "all_3_3_0_5"] == get_parts( - node, table_name - ) or ["all_1_1_0_5", "all_2_2_0_5", "all_3_3_0_5"] == get_parts(node, table_name) + assert ["all_0_0_0_5", "all_1_1_0_5", "all_2_2_0_5", "all_3_3_0_5"] == parts or [ + "all_1_1_0_5", + "all_2_2_0_5", + "all_3_3_0_5", + ] == parts # Still broken because it was hardlinked. broken = get_broken_projections_info(node, table_name) @@ -752,11 +755,13 @@ def test_mutation_with_broken_projection(cluster): f"ALTER TABLE {table_name} DELETE WHERE c == 13 SETTINGS mutations_sync = 1" ) - assert ["all_1_1_0_6", "all_2_2_0_6", "all_3_3_0_6"] == get_parts( - node, table_name - ) or ["all_0_0_0_6", "all_1_1_0_6", "all_2_2_0_6", "all_3_3_0_6"] == get_parts( - node, table_name - ) + parts = get_parts(node, table_name) + assert ["all_1_1_0_6", "all_2_2_0_6", "all_3_3_0_6"] == parts or [ + "all_0_0_0_6", + "all_1_1_0_6", + "all_2_2_0_6", + "all_3_3_0_6", + ] == parts # Not broken anymore. assert not get_broken_projections_info(node, table_name) From c0a42ef877c7abf93db444c95f39a59cefe93d32 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 11 Sep 2024 15:42:04 +0000 Subject: [PATCH 208/215] Automatic style fix --- tests/integration/test_storage_s3_queue/test.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index 8b492018766..b75ad21f002 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -662,10 +662,7 @@ def test_multiple_tables_meta_mismatch(started_cluster): }, ) except QueryRuntimeException as e: - assert ( - "Existing table metadata in ZooKeeper differs in columns" - in str(e) - ) + assert "Existing table metadata in ZooKeeper differs in columns" in str(e) failed = True assert failed is True From 370ce5ef6347addae66a4fd29b5b0d2b10c8ff76 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 11 Sep 2024 22:37:30 +0200 Subject: [PATCH 209/215] Get rid of broken `get_commits().reversed` --- tests/ci/pr_info.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 5c051b093e0..d3c4ffb1e68 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -9,6 +9,7 @@ from urllib.parse import quote from unidiff import PatchSet # type: ignore from build_download_helper import get_gh_api +from ci_config import Labels from env_helper import ( GITHUB_EVENT_PATH, GITHUB_REPOSITORY, @@ -16,7 +17,6 @@ from env_helper import ( GITHUB_SERVER_URL, GITHUB_UPSTREAM_REPOSITORY, ) -from ci_config import Labels from get_robot_token import get_best_robot_token from github_helper import GitHub @@ -459,16 +459,18 @@ class PRInfo: sync_repo = gh.get_repo(GITHUB_REPOSITORY) sync_pr = sync_repo.get_pull(self.number) # Find the commit that is in both repos, upstream and cloud - sync_commits = sync_pr.get_commits().reversed - upstream_commits = upstream_pr.get_commits().reversed + # Do not ever use `reversed` here, otherwise the list of commits is not full + sync_commits = list(sync_pr.get_commits()) + upstream_commits = list(upstream_pr.get_commits()) # Github objects are compared by _url attribute. We can't compare them directly and # should compare commits by SHA1 upstream_shas = [c.sha for c in upstream_commits] logging.info("Commits in upstream PR:\n %s", ", ".join(upstream_shas)) sync_shas = [c.sha for c in sync_commits] - logging.info("Commits in sync PR:\n %s", ", ".join(reversed(sync_shas))) + logging.info("Commits in sync PR:\n %s", ", ".join(sync_shas)) - # find latest synced commit + # find latest synced commit, search from the latest + upstream_commits.reverse() last_synced_upstream_commit = None for commit in upstream_commits: if commit.sha in sync_shas: From 98c0f82f28e429681120a2c842184d1e038f1529 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com> Date: Wed, 11 Sep 2024 20:45:54 -0400 Subject: [PATCH 210/215] make it compatible with multiple runs --- tests/integration/test_ssl_cert_authentication/test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 26ee33f3f6f..7db28b46f74 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -334,6 +334,8 @@ def test_create_user(): 'lucy\tssl_certificate\t{"common_names":["client2","client3"]}\n' ) + instance.query("DROP USER emma") + def test_x509_san_support(): assert ( @@ -370,6 +372,8 @@ def test_x509_san_support(): == "CREATE USER jemma IDENTIFIED WITH ssl_certificate SAN \\'URI:spiffe://foo.com/bar\\', \\'URI:spiffe://foo.com/baz\\'\n" ) + instance.query("DROP USER jemma") + def test_x509_san_wildcard_support(): assert ( @@ -404,3 +408,5 @@ def test_x509_san_wildcard_support(): instance.query("SHOW CREATE USER brian") == "CREATE USER brian IDENTIFIED WITH ssl_certificate SAN \\'URI:spiffe://bar.com/foo/*/far\\'\n" ) + + instance.query("DROP USER brian") From c124533ef90b2b28373b7da1ae27ee8438684a21 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com> Date: Thu, 12 Sep 2024 00:08:53 -0400 Subject: [PATCH 211/215] fix --- tests/integration/test_ssl_cert_authentication/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 7db28b46f74..4b5f3332a48 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -334,7 +334,7 @@ def test_create_user(): 'lucy\tssl_certificate\t{"common_names":["client2","client3"]}\n' ) - instance.query("DROP USER emma") + instance.query("DROP USER emma") def test_x509_san_support(): From fd53b39352073fdb7e5940a02bd7b449ef7da5b0 Mon Sep 17 00:00:00 2001 From: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com> Date: Thu, 12 Sep 2024 02:33:15 -0400 Subject: [PATCH 212/215] bump From 5fe5cc8611ba5c6ddea22538f47a5f650c28bf88 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 12 Sep 2024 09:21:27 +0000 Subject: [PATCH 213/215] Disallow query cache for queries with non-throw overflow mode --- src/Interpreters/executeQuery.cpp | 13 +++++++++ .../02494_query_cache_bugs.reference | 21 ++++++++++++++ .../0_stateless/02494_query_cache_bugs.sql | 28 +++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 6c22e71bccf..be9423852c1 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -1118,6 +1118,19 @@ static std::tuple executeQueryImpl( && settings.use_query_cache && !internal && client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY + /// Bug 67476: Avoid that the query cache stores truncated results if the query ran with a non-THROW overflow mode and hit a limit. + /// This is more workaround than a fix ... unfortunately it is hard to detect from the perspective of the query cache that the + /// query result is truncated. + && (settings.read_overflow_mode == OverflowMode::THROW + && settings.read_overflow_mode_leaf == OverflowMode::THROW + && settings.group_by_overflow_mode == OverflowMode::THROW + && settings.sort_overflow_mode == OverflowMode::THROW + && settings.result_overflow_mode == OverflowMode::THROW + && settings.timeout_overflow_mode == OverflowMode::THROW + && settings.set_overflow_mode == OverflowMode::THROW + && settings.join_overflow_mode == OverflowMode::THROW + && settings.transfer_overflow_mode == OverflowMode::THROW + && settings.distinct_overflow_mode == OverflowMode::THROW) && (ast->as() || ast->as()); QueryCache::Usage query_cache_usage = QueryCache::Usage::None; diff --git a/tests/queries/0_stateless/02494_query_cache_bugs.reference b/tests/queries/0_stateless/02494_query_cache_bugs.reference index 448e1366ea7..ea9017d5394 100644 --- a/tests/queries/0_stateless/02494_query_cache_bugs.reference +++ b/tests/queries/0_stateless/02494_query_cache_bugs.reference @@ -22,3 +22,24 @@ Row 1: ────── x: 1 2 +-- Bug 67476: Queries with overflow mode != throw must not be cached by the query cache +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 +0 diff --git a/tests/queries/0_stateless/02494_query_cache_bugs.sql b/tests/queries/0_stateless/02494_query_cache_bugs.sql index 74496e0f77a..423068aa646 100644 --- a/tests/queries/0_stateless/02494_query_cache_bugs.sql +++ b/tests/queries/0_stateless/02494_query_cache_bugs.sql @@ -36,4 +36,32 @@ SELECT count(*) FROM system.query_cache; DROP TABLE tab; +SELECT '-- Bug 67476: Queries with overflow mode != throw must not be cached by the query cache'; + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab(c UInt64) ENGINE = Memory; + +SYSTEM DROP QUERY CACHE; +SELECT sum(c) FROM tab SETTINGS read_overflow_mode = 'break', use_query_cache = 1; +SELECT count(*) from system.query_cache; +SELECT sum(c) FROM tab SETTINGS read_overflow_mode_leaf = 'break', use_query_cache = 1; +SELECT count(*) from system.query_cache; +SELECT sum(c) FROM tab SETTINGS group_by_overflow_mode = 'break', use_query_cache = 1; +SELECT count(*) from system.query_cache; +SELECT sum(c) FROM tab SETTINGS sort_overflow_mode = 'break', use_query_cache = 1; +SELECT count(*) from system.query_cache; +SELECT sum(c) FROM tab SETTINGS result_overflow_mode = 'break', use_query_cache = 1; +SELECT count(*) from system.query_cache; +SELECT sum(c) FROM tab SETTINGS timeout_overflow_mode = 'break', use_query_cache = 1; +SELECT count(*) from system.query_cache; +SELECT sum(c) FROM tab SETTINGS set_overflow_mode = 'break', use_query_cache = 1; +SELECT count(*) from system.query_cache; +SELECT sum(c) FROM tab SETTINGS join_overflow_mode = 'break', use_query_cache = 1; +SELECT count(*) from system.query_cache; +SELECT sum(c) FROM tab SETTINGS transfer_overflow_mode = 'break', use_query_cache = 1; +SELECT count(*) from system.query_cache; +SELECT sum(c) FROM tab SETTINGS distinct_overflow_mode = 'break', use_query_cache = 1; +SELECT count(*) from system.query_cache; + SYSTEM DROP QUERY CACHE; From 6e94c9f617a5f821b9a22e762904cc92dab8ef01 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 12 Sep 2024 12:34:56 +0200 Subject: [PATCH 214/215] Fix --- src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp | 2 ++ .../ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp index e424769b061..2da4aa6b665 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMetadata.cpp @@ -239,6 +239,8 @@ void ObjectStorageQueueMetadata::syncWithKeeper( const auto metadata_str = zookeeper->get(fs::path(zookeeper_path) / "metadata"); const auto metadata_from_zk = ObjectStorageQueueTableMetadata::parse(metadata_str); + LOG_TRACE(log, "Metadata in keeper: {}", metadata_str); + table_metadata.checkEquals(metadata_from_zk); return; } diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp index c2f70e3e804..926d5aacda4 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueTableMetadata.cpp @@ -88,7 +88,7 @@ ObjectStorageQueueTableMetadata::ObjectStorageQueueTableMetadata(const Poco::JSO , tracked_files_limit(getOrDefault(json, "tracked_files_limit", "s3queue_", 0)) , tracked_file_ttl_sec(getOrDefault(json, "tracked_files_ttl_sec", "s3queue_", 0)) , buckets(getOrDefault(json, "buckets", "", 0)) - , processing_threads_num(getOrDefault(json, "processing_threads_num", "s3queue_", 0)) + , processing_threads_num(getOrDefault(json, "processing_threads_num", "s3queue_", 1)) , last_processed_path(getOrDefault(json, "last_processed_file", "s3queue_", "")) { } From 2225389474b799efcf653d45c6d7f955ebe1e509 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Thu, 12 Sep 2024 20:10:42 +0800 Subject: [PATCH 215/215] Fix upgrade check failed for join_to_sort settings --- src/Core/SettingsChangesHistory.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index fae8c25f5ed..6bd354ce05b 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -80,6 +80,9 @@ static std::initializer_list