mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 00:30:49 +00:00
Use ThreadPoolCallbackRunnerLocal
This commit is contained in:
parent
16eb12a321
commit
3d5c8db1e0
@ -4,6 +4,7 @@
|
|||||||
#include <IO/copyData.h>
|
#include <IO/copyData.h>
|
||||||
#include <Poco/Logger.h>
|
#include <Poco/Logger.h>
|
||||||
#include <Interpreters/Context.h>
|
#include <Interpreters/Context.h>
|
||||||
|
#include <Common/threadPoolCallbackRunner.h>
|
||||||
#include <Common/logger_useful.h>
|
#include <Common/logger_useful.h>
|
||||||
#include <Common/setThreadName.h>
|
#include <Common/setThreadName.h>
|
||||||
#include <Core/ServerUUID.h>
|
#include <Core/ServerUUID.h>
|
||||||
@ -84,16 +85,12 @@ UInt128 IDisk::getEncryptedFileIV(const String &) const
|
|||||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "File encryption is not implemented for disk of type {}", getDataSourceDescription().type);
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "File encryption is not implemented for disk of type {}", getDataSourceDescription().type);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
using ResultsCollector = std::vector<std::future<void>>;
|
|
||||||
|
|
||||||
void asyncCopy(
|
void asyncCopy(
|
||||||
IDisk & from_disk,
|
IDisk & from_disk,
|
||||||
String from_path,
|
String from_path,
|
||||||
IDisk & to_disk,
|
IDisk & to_disk,
|
||||||
String to_path,
|
String to_path,
|
||||||
ThreadPool & pool,
|
ThreadPoolCallbackRunnerLocal<void> & runner,
|
||||||
ResultsCollector & results,
|
|
||||||
bool copy_root_dir,
|
bool copy_root_dir,
|
||||||
const ReadSettings & read_settings,
|
const ReadSettings & read_settings,
|
||||||
const WriteSettings & write_settings,
|
const WriteSettings & write_settings,
|
||||||
@ -101,29 +98,11 @@ void asyncCopy(
|
|||||||
{
|
{
|
||||||
if (from_disk.isFile(from_path))
|
if (from_disk.isFile(from_path))
|
||||||
{
|
{
|
||||||
auto promise = std::make_shared<std::promise<void>>();
|
runner(
|
||||||
auto future = promise->get_future();
|
[&from_disk, from_path, &to_disk, to_path, &read_settings, &write_settings, &cancellation_hook] {
|
||||||
|
from_disk.copyFile(
|
||||||
pool.scheduleOrThrowOnError(
|
from_path, to_disk, fs::path(to_path) / fileName(from_path), read_settings, write_settings, cancellation_hook);
|
||||||
[&from_disk, from_path, &to_disk, to_path, &read_settings, &write_settings, promise, thread_group = CurrentThread::getGroup(), &cancellation_hook]()
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
SCOPE_EXIT_SAFE(if (thread_group) CurrentThread::detachFromGroupIfNotDetached(););
|
|
||||||
|
|
||||||
if (thread_group)
|
|
||||||
CurrentThread::attachToGroup(thread_group);
|
|
||||||
|
|
||||||
from_disk.copyFile(from_path, to_disk, fs::path(to_path) / fileName(from_path), read_settings, write_settings, cancellation_hook);
|
|
||||||
promise->set_value();
|
|
||||||
}
|
|
||||||
catch (...)
|
|
||||||
{
|
|
||||||
promise->set_exception(std::current_exception());
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
|
|
||||||
results.push_back(std::move(future));
|
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -136,7 +115,7 @@ void asyncCopy(
|
|||||||
}
|
}
|
||||||
|
|
||||||
for (auto it = from_disk.iterateDirectory(from_path); it->isValid(); it->next())
|
for (auto it = from_disk.iterateDirectory(from_path); it->isValid(); it->next())
|
||||||
asyncCopy(from_disk, it->path(), to_disk, dest, pool, results, true, read_settings, write_settings, cancellation_hook);
|
asyncCopy(from_disk, it->path(), to_disk, dest, runner, true, read_settings, write_settings, cancellation_hook);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -149,21 +128,16 @@ void IDisk::copyThroughBuffers(
|
|||||||
WriteSettings write_settings,
|
WriteSettings write_settings,
|
||||||
const std::function<void()> & cancellation_hook)
|
const std::function<void()> & cancellation_hook)
|
||||||
{
|
{
|
||||||
ResultsCollector results;
|
ThreadPoolCallbackRunnerLocal<void> runner(copying_thread_pool, "AsyncCopy");
|
||||||
|
|
||||||
SCOPE_EXIT_SAFE(
|
|
||||||
for (auto & result : results)
|
|
||||||
result.wait();
|
|
||||||
for (auto & result : results)
|
|
||||||
result.get(); /// May rethrow an exception
|
|
||||||
);
|
|
||||||
|
|
||||||
/// Disable parallel write. We already copy in parallel.
|
/// Disable parallel write. We already copy in parallel.
|
||||||
/// Avoid high memory usage. See test_s3_zero_copy_ttl/test.py::test_move_and_s3_memory_usage
|
/// Avoid high memory usage. See test_s3_zero_copy_ttl/test.py::test_move_and_s3_memory_usage
|
||||||
write_settings.s3_allow_parallel_part_upload = false;
|
write_settings.s3_allow_parallel_part_upload = false;
|
||||||
write_settings.azure_allow_parallel_part_upload = false;
|
write_settings.azure_allow_parallel_part_upload = false;
|
||||||
|
|
||||||
asyncCopy(*this, from_path, *to_disk, to_path, copying_thread_pool, results, copy_root_dir, read_settings, write_settings, cancellation_hook);
|
asyncCopy(*this, from_path, *to_disk, to_path, runner, copy_root_dir, read_settings, write_settings, cancellation_hook);
|
||||||
|
|
||||||
|
runner.waitForAllToFinishAndRethrowFirstError();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user