mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 08:40:50 +00:00
Add serialization for SortingStep.
This commit is contained in:
parent
1ed4e1b847
commit
7ee29b6254
@ -9,9 +9,23 @@ namespace DB
|
|||||||
{
|
{
|
||||||
|
|
||||||
#define PLAN_SERIALIZATION_SETTINGS(M, ALIAS) \
|
#define PLAN_SERIALIZATION_SETTINGS(M, ALIAS) \
|
||||||
|
M(UInt64, max_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size in rows for reading", 0) \
|
||||||
|
\
|
||||||
M(UInt64, max_rows_in_distinct, 0, "Maximum number of elements during execution of DISTINCT.", 0) \
|
M(UInt64, max_rows_in_distinct, 0, "Maximum number of elements during execution of DISTINCT.", 0) \
|
||||||
M(UInt64, max_bytes_in_distinct, 0, "Maximum total size of state (in uncompressed bytes) in memory for the execution of DISTINCT.", 0) \
|
M(UInt64, max_bytes_in_distinct, 0, "Maximum total size of state (in uncompressed bytes) in memory for the execution of DISTINCT.", 0) \
|
||||||
M(OverflowMode, distinct_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \
|
M(OverflowMode, distinct_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \
|
||||||
|
\
|
||||||
|
M(UInt64, max_rows_to_sort, 0, "If more than the specified amount of records have to be processed for ORDER BY operation, the behavior will be determined by the 'sort_overflow_mode' which by default is - throw an exception", 0) \
|
||||||
|
M(UInt64, max_bytes_to_sort, 0, "If more than the specified amount of (uncompressed) bytes have to be processed for ORDER BY operation, the behavior will be determined by the 'sort_overflow_mode' which by default is - throw an exception", 0) \
|
||||||
|
M(OverflowMode, sort_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \
|
||||||
|
\
|
||||||
|
M(UInt64, prefer_external_sort_block_bytes, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging.", 0) \
|
||||||
|
M(UInt64, max_bytes_before_external_sort, 0, "If memory usage during ORDER BY operation is exceeding this threshold in bytes, activate the 'external sorting' mode (spill data to disk). Recommended value is half of available system memory.", 0) \
|
||||||
|
M(UInt64, max_bytes_before_remerge_sort, 1000000000, "In case of ORDER BY with LIMIT, when memory usage is higher than specified threshold, perform additional steps of merging blocks before final merge to keep just top LIMIT rows.", 0) \
|
||||||
|
M(Float, remerge_sort_lowered_memory_bytes_ratio, 2., "If memory usage after remerge does not reduced by this ratio, remerge will be disabled.", 0) \
|
||||||
|
M(UInt64, min_free_disk_space_for_temporary_data, 0, "The minimum disk space to keep while writing temporary data used in external sorting and aggregation.", 0) \
|
||||||
|
\
|
||||||
|
|
||||||
|
|
||||||
DECLARE_SETTINGS_TRAITS(QueryPlanSerializationSettingsTraits, PLAN_SERIALIZATION_SETTINGS)
|
DECLARE_SETTINGS_TRAITS(QueryPlanSerializationSettingsTraits, PLAN_SERIALIZATION_SETTINGS)
|
||||||
|
|
||||||
|
@ -42,6 +42,8 @@ QueryPlanStepPtr QueryPlanStepRegistry::createStep(
|
|||||||
void registerExpressionStep(QueryPlanStepRegistry & registry);
|
void registerExpressionStep(QueryPlanStepRegistry & registry);
|
||||||
void registerUnionStep(QueryPlanStepRegistry & registry);
|
void registerUnionStep(QueryPlanStepRegistry & registry);
|
||||||
void registerDistinctStep(QueryPlanStepRegistry & registry);
|
void registerDistinctStep(QueryPlanStepRegistry & registry);
|
||||||
|
void registerSortingStep(QueryPlanStepRegistry & registry);
|
||||||
|
|
||||||
void registerReadFromTableStep(QueryPlanStepRegistry & registry);
|
void registerReadFromTableStep(QueryPlanStepRegistry & registry);
|
||||||
|
|
||||||
void QueryPlanStepRegistry::registerPlanSteps()
|
void QueryPlanStepRegistry::registerPlanSteps()
|
||||||
@ -51,6 +53,7 @@ void QueryPlanStepRegistry::registerPlanSteps()
|
|||||||
registerExpressionStep(registry);
|
registerExpressionStep(registry);
|
||||||
registerUnionStep(registry);
|
registerUnionStep(registry);
|
||||||
registerDistinctStep(registry);
|
registerDistinctStep(registry);
|
||||||
|
registerSortingStep(registry);
|
||||||
|
|
||||||
registerReadFromTableStep(registry);
|
registerReadFromTableStep(registry);
|
||||||
}
|
}
|
||||||
|
@ -2,6 +2,8 @@
|
|||||||
#include <Interpreters/Context.h>
|
#include <Interpreters/Context.h>
|
||||||
#include <Processors/Merges/MergingSortedTransform.h>
|
#include <Processors/Merges/MergingSortedTransform.h>
|
||||||
#include <Processors/QueryPlan/SortingStep.h>
|
#include <Processors/QueryPlan/SortingStep.h>
|
||||||
|
#include <Processors/QueryPlan/QueryPlanSerializationSettings.h>
|
||||||
|
#include <Processors/QueryPlan/QueryPlanStepRegistry.h>
|
||||||
#include <Processors/Transforms/FinishSortingTransform.h>
|
#include <Processors/Transforms/FinishSortingTransform.h>
|
||||||
#include <Processors/Transforms/LimitsCheckingTransform.h>
|
#include <Processors/Transforms/LimitsCheckingTransform.h>
|
||||||
#include <Processors/Transforms/MergeSortingTransform.h>
|
#include <Processors/Transforms/MergeSortingTransform.h>
|
||||||
@ -48,6 +50,33 @@ SortingStep::Settings::Settings(size_t max_block_size_)
|
|||||||
max_block_size = max_block_size_;
|
max_block_size = max_block_size_;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
SortingStep::Settings::Settings(const QueryPlanSerializationSettings & settings)
|
||||||
|
{
|
||||||
|
max_block_size = settings.max_block_size;
|
||||||
|
size_limits = SizeLimits(settings.max_rows_to_sort, settings.max_bytes_to_sort, settings.sort_overflow_mode);
|
||||||
|
max_bytes_before_remerge = settings.max_bytes_before_remerge_sort;
|
||||||
|
remerge_lowered_memory_bytes_ratio = settings.remerge_sort_lowered_memory_bytes_ratio;
|
||||||
|
max_bytes_before_external_sort = settings.max_bytes_before_external_sort;
|
||||||
|
tmp_data = Context::getGlobalContextInstance()->getTempDataOnDisk();
|
||||||
|
min_free_disk_space = settings.min_free_disk_space_for_temporary_data;
|
||||||
|
max_block_bytes = settings.prefer_external_sort_block_bytes;
|
||||||
|
read_in_order_use_buffering = false; //settings.read_in_order_use_buffering;
|
||||||
|
}
|
||||||
|
|
||||||
|
void SortingStep::Settings::updatePlanSettings(QueryPlanSerializationSettings & settings) const
|
||||||
|
{
|
||||||
|
settings.max_block_size = max_block_size;
|
||||||
|
settings.max_rows_to_sort = size_limits.max_rows;
|
||||||
|
settings.max_bytes_to_sort = size_limits.max_bytes;
|
||||||
|
settings.sort_overflow_mode = size_limits.overflow_mode;
|
||||||
|
|
||||||
|
settings.max_bytes_before_remerge_sort = max_bytes_before_remerge;
|
||||||
|
settings.remerge_sort_lowered_memory_bytes_ratio = remerge_lowered_memory_bytes_ratio;
|
||||||
|
settings.max_bytes_before_external_sort = max_bytes_before_external_sort;
|
||||||
|
settings.min_free_disk_space_for_temporary_data = min_free_disk_space;
|
||||||
|
settings.prefer_external_sort_block_bytes = max_block_bytes;
|
||||||
|
}
|
||||||
|
|
||||||
static ITransformingStep::Traits getTraits(size_t limit)
|
static ITransformingStep::Traits getTraits(size_t limit)
|
||||||
{
|
{
|
||||||
return ITransformingStep::Traits
|
return ITransformingStep::Traits
|
||||||
@ -465,4 +494,53 @@ void SortingStep::describeActions(JSONBuilder::JSONMap & map) const
|
|||||||
map.add("Limit", limit);
|
map.add("Limit", limit);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void SortingStep::serializeSettings(QueryPlanSerializationSettings & settings) const
|
||||||
|
{
|
||||||
|
sort_settings.updatePlanSettings(settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
void SortingStep::serialize(WriteBuffer & out) const
|
||||||
|
{
|
||||||
|
if (type != Type::Full)
|
||||||
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Serialization of SortingStep is implemented only for Full sorting");
|
||||||
|
|
||||||
|
/// Do not serialize type here; Later we can use different names if needed.\
|
||||||
|
|
||||||
|
/// Do not serialize limit for now; it is expected to be pushed down from plan optimization.
|
||||||
|
|
||||||
|
serializeSortDescription(result_description, out);
|
||||||
|
|
||||||
|
/// Later
|
||||||
|
if (!partition_by_description.empty())
|
||||||
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Serialization of partitioned sorting is not implemented for SortingStep");
|
||||||
|
|
||||||
|
writeVarUInt(partition_by_description.size(), out);
|
||||||
|
}
|
||||||
|
|
||||||
|
std::unique_ptr<IQueryPlanStep> SortingStep::deserialize(
|
||||||
|
ReadBuffer & in, const DataStreams & input_streams_, const DataStream *, QueryPlanSerializationSettings & settings)
|
||||||
|
{
|
||||||
|
if (input_streams_.size() != 1)
|
||||||
|
throw Exception(ErrorCodes::INCORRECT_DATA, "SortingStep must have one input stream");
|
||||||
|
|
||||||
|
SortingStep::Settings sort_settings(settings);
|
||||||
|
|
||||||
|
SortDescription result_description;
|
||||||
|
deserializeSortDescription(result_description, in);
|
||||||
|
|
||||||
|
UInt64 partition_desc_size;
|
||||||
|
readVarUInt(partition_desc_size, in);
|
||||||
|
|
||||||
|
if (partition_desc_size)
|
||||||
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Deserialization of partitioned sorting is not implemented for SortingStep");
|
||||||
|
|
||||||
|
return std::make_unique<SortingStep>(
|
||||||
|
input_streams_.front(), std::move(result_description), 0, std::move(sort_settings), true);
|
||||||
|
}
|
||||||
|
|
||||||
|
void registerSortingStep(QueryPlanStepRegistry & registry)
|
||||||
|
{
|
||||||
|
registry.registerStep("Sorting", SortingStep::deserialize);
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -23,7 +23,7 @@ public:
|
|||||||
size_t max_block_size;
|
size_t max_block_size;
|
||||||
SizeLimits size_limits;
|
SizeLimits size_limits;
|
||||||
size_t max_bytes_before_remerge = 0;
|
size_t max_bytes_before_remerge = 0;
|
||||||
double remerge_lowered_memory_bytes_ratio = 0;
|
float remerge_lowered_memory_bytes_ratio = 0;
|
||||||
size_t max_bytes_before_external_sort = 0;
|
size_t max_bytes_before_external_sort = 0;
|
||||||
TemporaryDataOnDiskScopePtr tmp_data = nullptr;
|
TemporaryDataOnDiskScopePtr tmp_data = nullptr;
|
||||||
size_t min_free_disk_space = 0;
|
size_t min_free_disk_space = 0;
|
||||||
@ -32,6 +32,9 @@ public:
|
|||||||
|
|
||||||
explicit Settings(const Context & context);
|
explicit Settings(const Context & context);
|
||||||
explicit Settings(size_t max_block_size_);
|
explicit Settings(size_t max_block_size_);
|
||||||
|
explicit Settings(const QueryPlanSerializationSettings & settings);
|
||||||
|
|
||||||
|
void updatePlanSettings(QueryPlanSerializationSettings & settings) const;
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Full
|
/// Full
|
||||||
@ -93,6 +96,11 @@ public:
|
|||||||
UInt64 limit_,
|
UInt64 limit_,
|
||||||
bool skip_partial_sort = false);
|
bool skip_partial_sort = false);
|
||||||
|
|
||||||
|
void serializeSettings(QueryPlanSerializationSettings & settings) const override;
|
||||||
|
void serialize(WriteBuffer & out) const override;
|
||||||
|
|
||||||
|
static std::unique_ptr<IQueryPlanStep> deserialize(ReadBuffer & in, const DataStreams & input_streams_, const DataStream *, QueryPlanSerializationSettings & settings);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
void scatterByPartitionIfNeeded(QueryPipelineBuilder& pipeline);
|
void scatterByPartitionIfNeeded(QueryPipelineBuilder& pipeline);
|
||||||
void updateOutputStream() override;
|
void updateOutputStream() override;
|
||||||
|
Loading…
Reference in New Issue
Block a user