2017-04-01 09:19:00 +00:00
|
|
|
#include <Storages/MergeTree/MergeList.h>
|
2019-01-13 22:02:33 +00:00
|
|
|
#include <Storages/MergeTree/MergeTreeDataMergerMutator.h>
|
2021-09-16 21:19:58 +00:00
|
|
|
#include <Storages/MergeTree/FutureMergedMutatedPart.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Common/CurrentMetrics.h>
|
2018-06-01 15:32:27 +00:00
|
|
|
#include <Common/CurrentThread.h>
|
2022-01-10 19:39:10 +00:00
|
|
|
#include <Common/MemoryTracker.h>
|
|
|
|
#include <base/getThreadId.h>
|
2018-02-01 17:55:08 +00:00
|
|
|
|
2016-12-23 20:23:46 +00:00
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2019-01-11 19:14:50 +00:00
|
|
|
|
2021-10-13 20:47:28 +00:00
|
|
|
MemoryTrackerThreadSwitcher::MemoryTrackerThreadSwitcher(MergeListEntry & merge_list_entry_)
|
|
|
|
: merge_list_entry(merge_list_entry_)
|
2021-09-16 21:19:58 +00:00
|
|
|
{
|
|
|
|
// Each merge is executed into separate background processing pool thread
|
2019-03-14 18:03:35 +00:00
|
|
|
background_thread_memory_tracker = CurrentThread::getMemoryTracker();
|
Fix possible memory_tracker use-after-free for merges/mutations
There are two possible cases for execution merges/mutations:
1) from background thread
2) from OPTIMIZE TABLE query
1) is pretty simple, it's memory tracking structure is as follow:
current_thread::memory_tracker = level=Thread / description="(for thread)" ==
background_thread_memory_tracker = level=Thread / description="(for thread)"
current_thread::memory_tracker.parent = level=Global / description="(total)"
So as you can see it is pretty simple and MemoryTrackerThreadSwitcher
does not do anything icky for this case.
2) is complex, it's memory tracking structure is as follow:
current_thread::memory_tracker = level=Thread / description="(for thread)"
current_thread::memory_tracker.parent = level=Process / description="(for query)" ==
background_thread_memory_tracker = level=Process / description="(for query)"
Before this patch to track memory (and related things, like sampling,
profiling and so on) for OPTIMIZE TABLE query dirty hacks was done to
do this, since current_thread memory_tracker was of Thread scope, that
does not have any limits.
And so if will change parent for it to Merge/Mutate memory tracker
(which also does not have some of settings) it will not be correctly
tracked.
To address this Merge/Mutate was set as parent not to the
current_thread memory_tracker but to it's parent, since it's scope is
Process with all settings.
But that parent's memory_tracker is the memory_tracker of the
thread_group, and so if you will have nested ThreadPool inside
merge/mutate (this is the case for s3 async writes, which has been
added in #33291) you may get use-after-free of memory_tracker.
Consider the following example:
MemoryTrackerThreadSwitcher()
thread_group.memory_tracker.parent = merge_list_entry->memory_tracker
(see also background_thread_memory_tracker above)
CurrentThread::attachTo()
current_thread.memory_tracker.parent = thread_group.memory_tracker
CurrentThread::detachQuery()
current_thread.memory_tracker.parent = thread_group.memory_tracker.parent
# and this is equal to merge_list_entry->memory_tracker
~MemoryTrackerThreadSwitcher()
thread_group.memory_tracker = thread_group.memory_tracker.parent
So after the following we will get incorrect memory_tracker (from the
mege_list_entry) when the next job in that ThreadPool will not have
thread_group, since in this case it will not try to update the
current_thread.memory_tracker.parent and use-after-free will happens.
So to address the (2) issue, settings from the parent memory_tracker
should be copied to the merge_list_entry->memory_tracker, to avoid
playing with parent memory tracker.
Note, that settings from the query (OPTIMIZE TABLE) is not available at
that time, so it cannot be used (instead of parent's memory tracker
settings).
v2: remove memory_tracker.setOrRaiseHardLimit() from settings
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
2022-02-18 07:45:29 +00:00
|
|
|
background_thread_memory_tracker_prev_parent = background_thread_memory_tracker->getParent();
|
|
|
|
background_thread_memory_tracker->setParent(&merge_list_entry->memory_tracker);
|
2021-10-03 08:21:54 +00:00
|
|
|
|
|
|
|
prev_untracked_memory_limit = current_thread->untracked_memory_limit;
|
2021-10-13 20:47:28 +00:00
|
|
|
current_thread->untracked_memory_limit = merge_list_entry->max_untracked_memory;
|
2021-10-13 20:47:28 +00:00
|
|
|
|
|
|
|
/// Avoid accounting memory from another mutation/merge
|
|
|
|
/// (NOTE: consider moving such code to ThreadFromGlobalPool and related places)
|
|
|
|
prev_untracked_memory = current_thread->untracked_memory;
|
2021-10-13 20:47:28 +00:00
|
|
|
current_thread->untracked_memory = merge_list_entry->untracked_memory;
|
2021-10-13 20:47:28 +00:00
|
|
|
|
2022-07-17 15:22:12 +00:00
|
|
|
prev_query_id = std::string(current_thread->getQueryId());
|
2021-10-13 20:47:28 +00:00
|
|
|
current_thread->setQueryId(merge_list_entry->query_id);
|
2021-09-16 21:19:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
MemoryTrackerThreadSwitcher::~MemoryTrackerThreadSwitcher()
|
|
|
|
{
|
|
|
|
// Unplug memory_tracker from current background processing pool thread
|
Fix possible memory_tracker use-after-free for merges/mutations
There are two possible cases for execution merges/mutations:
1) from background thread
2) from OPTIMIZE TABLE query
1) is pretty simple, it's memory tracking structure is as follow:
current_thread::memory_tracker = level=Thread / description="(for thread)" ==
background_thread_memory_tracker = level=Thread / description="(for thread)"
current_thread::memory_tracker.parent = level=Global / description="(total)"
So as you can see it is pretty simple and MemoryTrackerThreadSwitcher
does not do anything icky for this case.
2) is complex, it's memory tracking structure is as follow:
current_thread::memory_tracker = level=Thread / description="(for thread)"
current_thread::memory_tracker.parent = level=Process / description="(for query)" ==
background_thread_memory_tracker = level=Process / description="(for query)"
Before this patch to track memory (and related things, like sampling,
profiling and so on) for OPTIMIZE TABLE query dirty hacks was done to
do this, since current_thread memory_tracker was of Thread scope, that
does not have any limits.
And so if will change parent for it to Merge/Mutate memory tracker
(which also does not have some of settings) it will not be correctly
tracked.
To address this Merge/Mutate was set as parent not to the
current_thread memory_tracker but to it's parent, since it's scope is
Process with all settings.
But that parent's memory_tracker is the memory_tracker of the
thread_group, and so if you will have nested ThreadPool inside
merge/mutate (this is the case for s3 async writes, which has been
added in #33291) you may get use-after-free of memory_tracker.
Consider the following example:
MemoryTrackerThreadSwitcher()
thread_group.memory_tracker.parent = merge_list_entry->memory_tracker
(see also background_thread_memory_tracker above)
CurrentThread::attachTo()
current_thread.memory_tracker.parent = thread_group.memory_tracker
CurrentThread::detachQuery()
current_thread.memory_tracker.parent = thread_group.memory_tracker.parent
# and this is equal to merge_list_entry->memory_tracker
~MemoryTrackerThreadSwitcher()
thread_group.memory_tracker = thread_group.memory_tracker.parent
So after the following we will get incorrect memory_tracker (from the
mege_list_entry) when the next job in that ThreadPool will not have
thread_group, since in this case it will not try to update the
current_thread.memory_tracker.parent and use-after-free will happens.
So to address the (2) issue, settings from the parent memory_tracker
should be copied to the merge_list_entry->memory_tracker, to avoid
playing with parent memory tracker.
Note, that settings from the query (OPTIMIZE TABLE) is not available at
that time, so it cannot be used (instead of parent's memory tracker
settings).
v2: remove memory_tracker.setOrRaiseHardLimit() from settings
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
2022-02-18 07:45:29 +00:00
|
|
|
background_thread_memory_tracker->setParent(background_thread_memory_tracker_prev_parent);
|
2021-10-03 08:21:54 +00:00
|
|
|
|
|
|
|
current_thread->untracked_memory_limit = prev_untracked_memory_limit;
|
2021-10-13 20:47:28 +00:00
|
|
|
|
|
|
|
merge_list_entry->untracked_memory = current_thread->untracked_memory;
|
2021-10-13 20:47:28 +00:00
|
|
|
current_thread->untracked_memory = prev_untracked_memory;
|
2021-10-13 20:47:28 +00:00
|
|
|
|
2021-10-13 20:47:28 +00:00
|
|
|
current_thread->setQueryId(prev_query_id);
|
2021-09-16 21:19:58 +00:00
|
|
|
}
|
|
|
|
|
2021-10-03 08:21:54 +00:00
|
|
|
MergeListElement::MergeListElement(
|
|
|
|
const StorageID & table_id_,
|
|
|
|
FutureMergedMutatedPartPtr future_part,
|
Fix possible memory_tracker use-after-free for merges/mutations
There are two possible cases for execution merges/mutations:
1) from background thread
2) from OPTIMIZE TABLE query
1) is pretty simple, it's memory tracking structure is as follow:
current_thread::memory_tracker = level=Thread / description="(for thread)" ==
background_thread_memory_tracker = level=Thread / description="(for thread)"
current_thread::memory_tracker.parent = level=Global / description="(total)"
So as you can see it is pretty simple and MemoryTrackerThreadSwitcher
does not do anything icky for this case.
2) is complex, it's memory tracking structure is as follow:
current_thread::memory_tracker = level=Thread / description="(for thread)"
current_thread::memory_tracker.parent = level=Process / description="(for query)" ==
background_thread_memory_tracker = level=Process / description="(for query)"
Before this patch to track memory (and related things, like sampling,
profiling and so on) for OPTIMIZE TABLE query dirty hacks was done to
do this, since current_thread memory_tracker was of Thread scope, that
does not have any limits.
And so if will change parent for it to Merge/Mutate memory tracker
(which also does not have some of settings) it will not be correctly
tracked.
To address this Merge/Mutate was set as parent not to the
current_thread memory_tracker but to it's parent, since it's scope is
Process with all settings.
But that parent's memory_tracker is the memory_tracker of the
thread_group, and so if you will have nested ThreadPool inside
merge/mutate (this is the case for s3 async writes, which has been
added in #33291) you may get use-after-free of memory_tracker.
Consider the following example:
MemoryTrackerThreadSwitcher()
thread_group.memory_tracker.parent = merge_list_entry->memory_tracker
(see also background_thread_memory_tracker above)
CurrentThread::attachTo()
current_thread.memory_tracker.parent = thread_group.memory_tracker
CurrentThread::detachQuery()
current_thread.memory_tracker.parent = thread_group.memory_tracker.parent
# and this is equal to merge_list_entry->memory_tracker
~MemoryTrackerThreadSwitcher()
thread_group.memory_tracker = thread_group.memory_tracker.parent
So after the following we will get incorrect memory_tracker (from the
mege_list_entry) when the next job in that ThreadPool will not have
thread_group, since in this case it will not try to update the
current_thread.memory_tracker.parent and use-after-free will happens.
So to address the (2) issue, settings from the parent memory_tracker
should be copied to the merge_list_entry->memory_tracker, to avoid
playing with parent memory tracker.
Note, that settings from the query (OPTIMIZE TABLE) is not available at
that time, so it cannot be used (instead of parent's memory tracker
settings).
v2: remove memory_tracker.setOrRaiseHardLimit() from settings
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
2022-02-18 07:45:29 +00:00
|
|
|
const Settings & settings)
|
2021-09-16 21:19:58 +00:00
|
|
|
: table_id{table_id_}
|
|
|
|
, partition_id{future_part->part_info.partition_id}
|
|
|
|
, result_part_name{future_part->name}
|
|
|
|
, result_part_path{future_part->path}
|
|
|
|
, result_part_info{future_part->part_info}
|
|
|
|
, num_parts{future_part->parts.size()}
|
Fix possible memory_tracker use-after-free for merges/mutations
There are two possible cases for execution merges/mutations:
1) from background thread
2) from OPTIMIZE TABLE query
1) is pretty simple, it's memory tracking structure is as follow:
current_thread::memory_tracker = level=Thread / description="(for thread)" ==
background_thread_memory_tracker = level=Thread / description="(for thread)"
current_thread::memory_tracker.parent = level=Global / description="(total)"
So as you can see it is pretty simple and MemoryTrackerThreadSwitcher
does not do anything icky for this case.
2) is complex, it's memory tracking structure is as follow:
current_thread::memory_tracker = level=Thread / description="(for thread)"
current_thread::memory_tracker.parent = level=Process / description="(for query)" ==
background_thread_memory_tracker = level=Process / description="(for query)"
Before this patch to track memory (and related things, like sampling,
profiling and so on) for OPTIMIZE TABLE query dirty hacks was done to
do this, since current_thread memory_tracker was of Thread scope, that
does not have any limits.
And so if will change parent for it to Merge/Mutate memory tracker
(which also does not have some of settings) it will not be correctly
tracked.
To address this Merge/Mutate was set as parent not to the
current_thread memory_tracker but to it's parent, since it's scope is
Process with all settings.
But that parent's memory_tracker is the memory_tracker of the
thread_group, and so if you will have nested ThreadPool inside
merge/mutate (this is the case for s3 async writes, which has been
added in #33291) you may get use-after-free of memory_tracker.
Consider the following example:
MemoryTrackerThreadSwitcher()
thread_group.memory_tracker.parent = merge_list_entry->memory_tracker
(see also background_thread_memory_tracker above)
CurrentThread::attachTo()
current_thread.memory_tracker.parent = thread_group.memory_tracker
CurrentThread::detachQuery()
current_thread.memory_tracker.parent = thread_group.memory_tracker.parent
# and this is equal to merge_list_entry->memory_tracker
~MemoryTrackerThreadSwitcher()
thread_group.memory_tracker = thread_group.memory_tracker.parent
So after the following we will get incorrect memory_tracker (from the
mege_list_entry) when the next job in that ThreadPool will not have
thread_group, since in this case it will not try to update the
current_thread.memory_tracker.parent and use-after-free will happens.
So to address the (2) issue, settings from the parent memory_tracker
should be copied to the merge_list_entry->memory_tracker, to avoid
playing with parent memory tracker.
Note, that settings from the query (OPTIMIZE TABLE) is not available at
that time, so it cannot be used (instead of parent's memory tracker
settings).
v2: remove memory_tracker.setOrRaiseHardLimit() from settings
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
2022-02-18 07:45:29 +00:00
|
|
|
, max_untracked_memory(settings.max_untracked_memory)
|
2021-10-13 20:47:28 +00:00
|
|
|
, query_id(table_id.getShortName() + "::" + result_part_name)
|
2021-09-16 21:19:58 +00:00
|
|
|
, thread_id{getThreadId()}
|
|
|
|
, merge_type{future_part->merge_type}
|
|
|
|
, merge_algorithm{MergeAlgorithm::Undecided}
|
2022-07-14 14:22:06 +00:00
|
|
|
, description{"to apply mutate/merge in " + query_id}
|
2021-09-16 21:19:58 +00:00
|
|
|
{
|
|
|
|
for (const auto & source_part : future_part->parts)
|
|
|
|
{
|
|
|
|
source_part_names.emplace_back(source_part->name);
|
2022-10-23 03:29:26 +00:00
|
|
|
source_part_paths.emplace_back(source_part->getDataPartStorage().getFullPath());
|
2021-09-16 21:19:58 +00:00
|
|
|
|
|
|
|
total_size_bytes_compressed += source_part->getBytesOnDisk();
|
|
|
|
total_size_marks += source_part->getMarksCount();
|
|
|
|
total_rows_count += source_part->index_granularity.getTotalRows();
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!future_part->parts.empty())
|
|
|
|
{
|
|
|
|
source_data_version = future_part->parts[0]->info.getDataVersion();
|
|
|
|
is_mutation = (result_part_info.getDataVersion() != source_data_version);
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2021-10-03 08:21:54 +00:00
|
|
|
|
2022-07-14 14:22:06 +00:00
|
|
|
memory_tracker.setDescription(description.c_str());
|
Fix possible memory_tracker use-after-free for merges/mutations
There are two possible cases for execution merges/mutations:
1) from background thread
2) from OPTIMIZE TABLE query
1) is pretty simple, it's memory tracking structure is as follow:
current_thread::memory_tracker = level=Thread / description="(for thread)" ==
background_thread_memory_tracker = level=Thread / description="(for thread)"
current_thread::memory_tracker.parent = level=Global / description="(total)"
So as you can see it is pretty simple and MemoryTrackerThreadSwitcher
does not do anything icky for this case.
2) is complex, it's memory tracking structure is as follow:
current_thread::memory_tracker = level=Thread / description="(for thread)"
current_thread::memory_tracker.parent = level=Process / description="(for query)" ==
background_thread_memory_tracker = level=Process / description="(for query)"
Before this patch to track memory (and related things, like sampling,
profiling and so on) for OPTIMIZE TABLE query dirty hacks was done to
do this, since current_thread memory_tracker was of Thread scope, that
does not have any limits.
And so if will change parent for it to Merge/Mutate memory tracker
(which also does not have some of settings) it will not be correctly
tracked.
To address this Merge/Mutate was set as parent not to the
current_thread memory_tracker but to it's parent, since it's scope is
Process with all settings.
But that parent's memory_tracker is the memory_tracker of the
thread_group, and so if you will have nested ThreadPool inside
merge/mutate (this is the case for s3 async writes, which has been
added in #33291) you may get use-after-free of memory_tracker.
Consider the following example:
MemoryTrackerThreadSwitcher()
thread_group.memory_tracker.parent = merge_list_entry->memory_tracker
(see also background_thread_memory_tracker above)
CurrentThread::attachTo()
current_thread.memory_tracker.parent = thread_group.memory_tracker
CurrentThread::detachQuery()
current_thread.memory_tracker.parent = thread_group.memory_tracker.parent
# and this is equal to merge_list_entry->memory_tracker
~MemoryTrackerThreadSwitcher()
thread_group.memory_tracker = thread_group.memory_tracker.parent
So after the following we will get incorrect memory_tracker (from the
mege_list_entry) when the next job in that ThreadPool will not have
thread_group, since in this case it will not try to update the
current_thread.memory_tracker.parent and use-after-free will happens.
So to address the (2) issue, settings from the parent memory_tracker
should be copied to the merge_list_entry->memory_tracker, to avoid
playing with parent memory tracker.
Note, that settings from the query (OPTIMIZE TABLE) is not available at
that time, so it cannot be used (instead of parent's memory tracker
settings).
v2: remove memory_tracker.setOrRaiseHardLimit() from settings
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
2022-02-18 07:45:29 +00:00
|
|
|
/// MemoryTracker settings should be set here, because
|
|
|
|
/// later (see MemoryTrackerThreadSwitcher)
|
|
|
|
/// parent memory tracker will be changed, and if merge executed from the
|
|
|
|
/// query (OPTIMIZE TABLE), all settings will be lost (since
|
|
|
|
/// current_thread::memory_tracker will have Thread level MemoryTracker,
|
|
|
|
/// which does not have any settings itself, it relies on the settings of the
|
|
|
|
/// thread_group::memory_tracker, but MemoryTrackerThreadSwitcher will reset parent).
|
|
|
|
memory_tracker.setProfilerStep(settings.memory_profiler_step);
|
|
|
|
memory_tracker.setSampleProbability(settings.memory_profiler_sample_probability);
|
2022-05-11 16:18:41 +00:00
|
|
|
memory_tracker.setSoftLimit(settings.memory_overcommit_ratio_denominator);
|
2022-09-11 01:21:34 +00:00
|
|
|
if (settings.memory_tracker_fault_probability > 0.0)
|
Fix possible memory_tracker use-after-free for merges/mutations
There are two possible cases for execution merges/mutations:
1) from background thread
2) from OPTIMIZE TABLE query
1) is pretty simple, it's memory tracking structure is as follow:
current_thread::memory_tracker = level=Thread / description="(for thread)" ==
background_thread_memory_tracker = level=Thread / description="(for thread)"
current_thread::memory_tracker.parent = level=Global / description="(total)"
So as you can see it is pretty simple and MemoryTrackerThreadSwitcher
does not do anything icky for this case.
2) is complex, it's memory tracking structure is as follow:
current_thread::memory_tracker = level=Thread / description="(for thread)"
current_thread::memory_tracker.parent = level=Process / description="(for query)" ==
background_thread_memory_tracker = level=Process / description="(for query)"
Before this patch to track memory (and related things, like sampling,
profiling and so on) for OPTIMIZE TABLE query dirty hacks was done to
do this, since current_thread memory_tracker was of Thread scope, that
does not have any limits.
And so if will change parent for it to Merge/Mutate memory tracker
(which also does not have some of settings) it will not be correctly
tracked.
To address this Merge/Mutate was set as parent not to the
current_thread memory_tracker but to it's parent, since it's scope is
Process with all settings.
But that parent's memory_tracker is the memory_tracker of the
thread_group, and so if you will have nested ThreadPool inside
merge/mutate (this is the case for s3 async writes, which has been
added in #33291) you may get use-after-free of memory_tracker.
Consider the following example:
MemoryTrackerThreadSwitcher()
thread_group.memory_tracker.parent = merge_list_entry->memory_tracker
(see also background_thread_memory_tracker above)
CurrentThread::attachTo()
current_thread.memory_tracker.parent = thread_group.memory_tracker
CurrentThread::detachQuery()
current_thread.memory_tracker.parent = thread_group.memory_tracker.parent
# and this is equal to merge_list_entry->memory_tracker
~MemoryTrackerThreadSwitcher()
thread_group.memory_tracker = thread_group.memory_tracker.parent
So after the following we will get incorrect memory_tracker (from the
mege_list_entry) when the next job in that ThreadPool will not have
thread_group, since in this case it will not try to update the
current_thread.memory_tracker.parent and use-after-free will happens.
So to address the (2) issue, settings from the parent memory_tracker
should be copied to the merge_list_entry->memory_tracker, to avoid
playing with parent memory tracker.
Note, that settings from the query (OPTIMIZE TABLE) is not available at
that time, so it cannot be used (instead of parent's memory tracker
settings).
v2: remove memory_tracker.setOrRaiseHardLimit() from settings
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
2022-02-18 07:45:29 +00:00
|
|
|
memory_tracker.setFaultProbability(settings.memory_tracker_fault_probability);
|
|
|
|
|
|
|
|
/// Let's try to copy memory related settings from the query,
|
|
|
|
/// since settings that we have here is not from query, but global, from the table.
|
|
|
|
///
|
|
|
|
/// NOTE: Remember, that Thread level MemoryTracker does not have any settings,
|
|
|
|
/// so it's parent is required.
|
|
|
|
MemoryTracker * query_memory_tracker = CurrentThread::getMemoryTracker();
|
|
|
|
MemoryTracker * parent_query_memory_tracker;
|
|
|
|
if (query_memory_tracker->level == VariableContext::Thread &&
|
|
|
|
(parent_query_memory_tracker = query_memory_tracker->getParent()) &&
|
|
|
|
parent_query_memory_tracker != &total_memory_tracker)
|
|
|
|
{
|
|
|
|
memory_tracker.setOrRaiseHardLimit(parent_query_memory_tracker->getHardLimit());
|
|
|
|
}
|
|
|
|
|
2016-12-23 20:23:46 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
MergeInfo MergeListElement::getInfo() const
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
MergeInfo res;
|
2021-06-06 12:24:49 +00:00
|
|
|
res.database = table_id.getDatabaseName();
|
|
|
|
res.table = table_id.getTableName();
|
2017-04-01 07:20:54 +00:00
|
|
|
res.result_part_name = result_part_name;
|
2019-11-20 08:51:52 +00:00
|
|
|
res.result_part_path = result_part_path;
|
2018-09-11 11:16:40 +00:00
|
|
|
res.partition_id = partition_id;
|
2019-01-13 22:02:33 +00:00
|
|
|
res.is_mutation = is_mutation;
|
2017-04-01 07:20:54 +00:00
|
|
|
res.elapsed = watch.elapsedSeconds();
|
2018-03-03 18:00:46 +00:00
|
|
|
res.progress = progress.load(std::memory_order_relaxed);
|
2017-04-01 07:20:54 +00:00
|
|
|
res.num_parts = num_parts;
|
|
|
|
res.total_size_bytes_compressed = total_size_bytes_compressed;
|
|
|
|
res.total_size_marks = total_size_marks;
|
2019-03-26 12:37:42 +00:00
|
|
|
res.total_rows_count = total_rows_count;
|
2017-04-01 07:20:54 +00:00
|
|
|
res.bytes_read_uncompressed = bytes_read_uncompressed.load(std::memory_order_relaxed);
|
|
|
|
res.bytes_written_uncompressed = bytes_written_uncompressed.load(std::memory_order_relaxed);
|
|
|
|
res.rows_read = rows_read.load(std::memory_order_relaxed);
|
|
|
|
res.rows_written = rows_written.load(std::memory_order_relaxed);
|
|
|
|
res.columns_written = columns_written.load(std::memory_order_relaxed);
|
|
|
|
res.memory_usage = memory_tracker.get();
|
2020-02-02 02:27:15 +00:00
|
|
|
res.thread_id = thread_id;
|
2020-09-03 13:00:13 +00:00
|
|
|
res.merge_type = toString(merge_type);
|
2020-10-27 15:27:12 +00:00
|
|
|
res.merge_algorithm = toString(merge_algorithm.load(std::memory_order_relaxed));
|
2017-04-01 07:20:54 +00:00
|
|
|
|
|
|
|
for (const auto & source_part_name : source_part_names)
|
|
|
|
res.source_part_names.emplace_back(source_part_name);
|
|
|
|
|
2019-11-20 08:51:52 +00:00
|
|
|
for (const auto & source_part_path : source_part_paths)
|
|
|
|
res.source_part_paths.emplace_back(source_part_path);
|
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
return res;
|
2016-12-23 20:23:46 +00:00
|
|
|
}
|
|
|
|
|
2021-09-16 21:19:58 +00:00
|
|
|
MergeListElement::~MergeListElement() = default;
|
|
|
|
|
2016-12-23 20:23:46 +00:00
|
|
|
|
|
|
|
}
|