2019-05-23 09:03:39 +00:00
|
|
|
#pragma once
|
|
|
|
|
2019-12-06 14:37:21 +00:00
|
|
|
#include <Common/config.h>
|
|
|
|
|
|
|
|
#if USE_AWS_S3
|
|
|
|
|
2019-05-23 09:03:39 +00:00
|
|
|
#include <Storages/IStorage.h>
|
2019-05-29 12:54:31 +00:00
|
|
|
#include <Poco/URI.h>
|
2019-05-23 09:03:39 +00:00
|
|
|
#include <common/logger_useful.h>
|
|
|
|
#include <ext/shared_ptr_helper.h>
|
2019-12-11 14:21:48 +00:00
|
|
|
|
|
|
|
namespace Aws::S3
|
|
|
|
{
|
|
|
|
class S3Client;
|
|
|
|
}
|
2019-09-22 22:13:42 +00:00
|
|
|
|
2019-05-23 09:03:39 +00:00
|
|
|
namespace DB
|
|
|
|
{
|
2019-12-03 16:23:24 +00:00
|
|
|
|
2019-05-31 07:27:14 +00:00
|
|
|
/**
|
2019-06-01 21:18:20 +00:00
|
|
|
* This class represents table engine for external S3 urls.
|
2019-05-31 07:27:14 +00:00
|
|
|
* It sends HTTP GET to server when select is called and
|
2019-06-01 21:18:20 +00:00
|
|
|
* HTTP PUT when insert is called.
|
2019-05-31 07:27:14 +00:00
|
|
|
*/
|
2020-07-10 07:26:55 +00:00
|
|
|
class StorageS3 : public ext::shared_ptr_helper<StorageS3>, public IStorage
|
2019-05-23 09:03:39 +00:00
|
|
|
{
|
|
|
|
public:
|
2019-12-06 14:37:21 +00:00
|
|
|
StorageS3(const S3::URI & uri,
|
2019-11-04 19:57:03 +00:00
|
|
|
const String & access_key_id,
|
|
|
|
const String & secret_access_key,
|
2019-12-04 16:06:55 +00:00
|
|
|
const StorageID & table_id_,
|
2019-06-01 21:18:20 +00:00
|
|
|
const String & format_name_,
|
2019-09-23 12:41:59 +00:00
|
|
|
UInt64 min_upload_part_size_,
|
2019-06-01 21:18:20 +00:00
|
|
|
const ColumnsDescription & columns_,
|
2019-09-22 22:13:42 +00:00
|
|
|
const ConstraintsDescription & constraints_,
|
2020-11-06 14:07:56 +00:00
|
|
|
const Context & context_,
|
2020-09-30 12:04:21 +00:00
|
|
|
const String & compression_method_ = "");
|
2019-06-01 21:18:20 +00:00
|
|
|
|
|
|
|
String getName() const override
|
|
|
|
{
|
2020-07-13 14:13:30 +00:00
|
|
|
return name;
|
2019-06-01 21:18:20 +00:00
|
|
|
}
|
|
|
|
|
2020-08-03 13:54:14 +00:00
|
|
|
Pipe read(
|
2019-09-22 22:13:42 +00:00
|
|
|
const Names & column_names,
|
2020-06-15 19:08:58 +00:00
|
|
|
const StorageMetadataPtr & /*metadata_snapshot*/,
|
2020-09-20 17:52:17 +00:00
|
|
|
SelectQueryInfo & query_info,
|
2019-05-23 09:03:39 +00:00
|
|
|
const Context & context,
|
|
|
|
QueryProcessingStage::Enum processed_stage,
|
|
|
|
size_t max_block_size,
|
|
|
|
unsigned num_streams) override;
|
|
|
|
|
2020-06-15 19:08:58 +00:00
|
|
|
BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, const Context & context) override;
|
2019-05-23 09:03:39 +00:00
|
|
|
|
2020-04-28 10:38:57 +00:00
|
|
|
NamesAndTypesList getVirtuals() const override;
|
2020-04-27 13:55:30 +00:00
|
|
|
|
2019-09-22 22:13:42 +00:00
|
|
|
private:
|
2019-12-06 14:37:21 +00:00
|
|
|
S3::URI uri;
|
2020-11-06 14:07:56 +00:00
|
|
|
const Context & global_context;
|
2019-05-23 09:03:39 +00:00
|
|
|
|
2019-05-31 07:27:14 +00:00
|
|
|
String format_name;
|
2019-09-23 12:41:59 +00:00
|
|
|
UInt64 min_upload_part_size;
|
2019-11-19 12:46:07 +00:00
|
|
|
String compression_method;
|
2019-12-03 16:23:24 +00:00
|
|
|
std::shared_ptr<Aws::S3::S3Client> client;
|
2020-07-13 14:13:30 +00:00
|
|
|
String name;
|
2019-05-23 09:03:39 +00:00
|
|
|
};
|
2019-12-11 14:21:48 +00:00
|
|
|
|
2019-05-23 09:03:39 +00:00
|
|
|
}
|
2019-12-11 14:21:48 +00:00
|
|
|
|
2019-12-09 12:36:06 +00:00
|
|
|
#endif
|