2016-01-28 01:00:27 +00:00
|
|
|
#pragma once
|
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Interpreters/InterserverIOHandler.h>
|
|
|
|
#include <Storages/MergeTree/MergeTreeData.h>
|
2019-05-17 14:34:25 +00:00
|
|
|
#include <Storages/IStorage_fwd.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <IO/HashingWriteBuffer.h>
|
|
|
|
#include <IO/copyData.h>
|
2017-12-27 17:58:52 +00:00
|
|
|
#include <IO/ConnectionTimeouts.h>
|
2019-05-12 14:57:23 +00:00
|
|
|
#include <IO/ReadWriteBufferFromHTTP.h>
|
2021-05-26 20:37:44 +00:00
|
|
|
#include <Common/Throttler.h>
|
2016-01-28 01:00:27 +00:00
|
|
|
|
|
|
|
|
2020-10-08 15:45:10 +00:00
|
|
|
namespace zkutil
|
|
|
|
{
|
|
|
|
class ZooKeeper;
|
|
|
|
using ZooKeeperPtr = std::shared_ptr<ZooKeeper>;
|
|
|
|
}
|
|
|
|
|
2016-01-28 01:00:27 +00:00
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2021-05-26 20:37:44 +00:00
|
|
|
class StorageReplicatedMergeTree;
|
|
|
|
|
2016-01-28 01:00:27 +00:00
|
|
|
namespace DataPartsExchange
|
|
|
|
{
|
|
|
|
|
2021-05-26 20:37:44 +00:00
|
|
|
/** Service for sending parts from the table *ReplicatedMergeTree.
|
2016-01-28 01:00:27 +00:00
|
|
|
*/
|
|
|
|
class Service final : public InterserverIOEndpoint
|
|
|
|
{
|
|
|
|
public:
|
2021-05-26 20:37:44 +00:00
|
|
|
explicit Service(StorageReplicatedMergeTree & data_);
|
2016-01-28 01:00:27 +00:00
|
|
|
|
|
|
|
Service(const Service &) = delete;
|
|
|
|
Service & operator=(const Service &) = delete;
|
|
|
|
|
|
|
|
std::string getId(const std::string & node_id) const override;
|
2021-02-19 12:51:26 +00:00
|
|
|
void processQuery(const HTMLForm & params, ReadBuffer & body, WriteBuffer & out, HTTPServerResponse & response) override;
|
2016-01-28 01:00:27 +00:00
|
|
|
|
|
|
|
private:
|
|
|
|
MergeTreeData::DataPartPtr findPart(const String & name);
|
2021-02-10 14:12:49 +00:00
|
|
|
void sendPartFromMemory(
|
|
|
|
const MergeTreeData::DataPartPtr & part,
|
|
|
|
WriteBuffer & out,
|
|
|
|
const std::map<String, std::shared_ptr<IMergeTreeDataPart>> & projections = {});
|
|
|
|
|
|
|
|
MergeTreeData::DataPart::Checksums sendPartFromDisk(
|
|
|
|
const MergeTreeData::DataPartPtr & part,
|
|
|
|
WriteBuffer & out,
|
|
|
|
int client_protocol_version,
|
|
|
|
const std::map<String, std::shared_ptr<IMergeTreeDataPart>> & projections = {});
|
|
|
|
|
2021-06-24 08:25:05 +00:00
|
|
|
void sendPartFromDiskRemoteMeta(const MergeTreeData::DataPartPtr & part, WriteBuffer & out);
|
2016-01-28 01:00:27 +00:00
|
|
|
|
2020-01-14 14:27:48 +00:00
|
|
|
/// StorageReplicatedMergeTree::shutdown() waits for all parts exchange handlers to finish,
|
|
|
|
/// so Service will never access dangling reference to storage
|
2021-05-26 20:37:44 +00:00
|
|
|
StorageReplicatedMergeTree & data;
|
2020-05-30 21:57:37 +00:00
|
|
|
Poco::Logger * log;
|
2016-01-28 01:00:27 +00:00
|
|
|
};
|
|
|
|
|
2017-04-16 15:00:33 +00:00
|
|
|
/** Client for getting the parts from the table *MergeTree.
|
2016-01-28 01:00:27 +00:00
|
|
|
*/
|
2021-02-19 12:51:26 +00:00
|
|
|
class Fetcher final : private boost::noncopyable
|
2016-01-28 01:00:27 +00:00
|
|
|
{
|
|
|
|
public:
|
2022-02-14 09:20:27 +00:00
|
|
|
explicit Fetcher(StorageReplicatedMergeTree & data_) : data(data_), log(&Poco::Logger::get("Fetcher")) {}
|
2016-01-28 01:00:27 +00:00
|
|
|
|
2017-04-16 15:00:33 +00:00
|
|
|
/// Downloads a part to tmp_directory. If to_detached - downloads to the `detached` directory.
|
2022-09-08 14:18:21 +00:00
|
|
|
MergeTreeData::MutableDataPartPtr fetchSelectedPart(
|
2020-06-26 11:30:23 +00:00
|
|
|
const StorageMetadataPtr & metadata_snapshot,
|
2021-05-21 16:14:01 +00:00
|
|
|
ContextPtr context,
|
2016-01-28 01:00:27 +00:00
|
|
|
const String & part_name,
|
|
|
|
const String & replica_path,
|
|
|
|
const String & host,
|
|
|
|
int port,
|
2017-12-27 17:58:52 +00:00
|
|
|
const ConnectionTimeouts & timeouts,
|
2018-07-26 15:10:57 +00:00
|
|
|
const String & user,
|
|
|
|
const String & password,
|
2018-07-30 18:32:21 +00:00
|
|
|
const String & interserver_scheme,
|
2021-05-26 20:37:44 +00:00
|
|
|
ThrottlerPtr throttler,
|
2018-05-21 13:49:54 +00:00
|
|
|
bool to_detached = false,
|
2020-10-08 15:45:10 +00:00
|
|
|
const String & tmp_prefix_ = "",
|
2021-03-12 09:58:32 +00:00
|
|
|
std::optional<CurrentlySubmergingEmergingTagger> * tagger_ptr = nullptr,
|
2021-06-24 08:25:05 +00:00
|
|
|
bool try_zero_copy = true,
|
2021-07-05 03:32:56 +00:00
|
|
|
DiskPtr dest_disk = nullptr);
|
2016-01-28 01:00:27 +00:00
|
|
|
|
2017-10-06 16:53:55 +00:00
|
|
|
/// You need to stop the data transfer.
|
|
|
|
ActionBlocker blocker;
|
2016-01-28 01:00:27 +00:00
|
|
|
|
|
|
|
private:
|
2021-02-10 14:12:49 +00:00
|
|
|
void downloadBaseOrProjectionPartToDisk(
|
|
|
|
const String & replica_path,
|
2022-04-21 19:19:13 +00:00
|
|
|
DataPartStorageBuilderPtr & data_part_storage_builder,
|
2021-02-10 14:12:49 +00:00
|
|
|
bool sync,
|
|
|
|
PooledReadWriteBufferFromHTTP & in,
|
2021-05-26 20:37:44 +00:00
|
|
|
MergeTreeData::DataPart::Checksums & checksums,
|
|
|
|
ThrottlerPtr throttler) const;
|
|
|
|
|
2021-02-10 14:12:49 +00:00
|
|
|
|
2020-04-29 17:14:49 +00:00
|
|
|
MergeTreeData::MutableDataPartPtr downloadPartToDisk(
|
2019-05-12 14:57:23 +00:00
|
|
|
const String & part_name,
|
|
|
|
const String & replica_path,
|
|
|
|
bool to_detached,
|
|
|
|
const String & tmp_prefix_,
|
2020-06-26 21:55:48 +00:00
|
|
|
bool sync,
|
2021-02-10 14:12:49 +00:00
|
|
|
DiskPtr disk,
|
|
|
|
PooledReadWriteBufferFromHTTP & in,
|
|
|
|
size_t projections,
|
2021-05-26 20:37:44 +00:00
|
|
|
MergeTreeData::DataPart::Checksums & checksums,
|
|
|
|
ThrottlerPtr throttler);
|
2019-05-12 14:57:23 +00:00
|
|
|
|
2020-04-29 17:14:49 +00:00
|
|
|
MergeTreeData::MutableDataPartPtr downloadPartToMemory(
|
|
|
|
const String & part_name,
|
2020-10-29 16:18:25 +00:00
|
|
|
const UUID & part_uuid,
|
2020-06-26 11:30:23 +00:00
|
|
|
const StorageMetadataPtr & metadata_snapshot,
|
2021-05-21 16:14:01 +00:00
|
|
|
ContextPtr context,
|
2021-07-05 03:32:56 +00:00
|
|
|
DiskPtr disk,
|
2021-02-10 14:12:49 +00:00
|
|
|
PooledReadWriteBufferFromHTTP & in,
|
2021-05-26 20:37:44 +00:00
|
|
|
size_t projections,
|
|
|
|
ThrottlerPtr throttler);
|
2020-04-29 17:14:49 +00:00
|
|
|
|
2021-06-24 08:25:05 +00:00
|
|
|
MergeTreeData::MutableDataPartPtr downloadPartToDiskRemoteMeta(
|
2020-10-08 15:45:10 +00:00
|
|
|
const String & part_name,
|
|
|
|
const String & replica_path,
|
|
|
|
bool to_detached,
|
|
|
|
const String & tmp_prefix_,
|
2021-07-05 03:32:56 +00:00
|
|
|
DiskPtr disk,
|
2021-05-26 20:37:44 +00:00
|
|
|
PooledReadWriteBufferFromHTTP & in,
|
|
|
|
ThrottlerPtr throttler);
|
2020-10-08 15:45:10 +00:00
|
|
|
|
2022-02-14 09:20:27 +00:00
|
|
|
StorageReplicatedMergeTree & data;
|
2020-05-30 21:57:37 +00:00
|
|
|
Poco::Logger * log;
|
2016-01-28 01:00:27 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|