2019-05-23 09:03:39 +00:00
|
|
|
#pragma once
|
|
|
|
|
2019-12-06 14:37:21 +00:00
|
|
|
#include <Common/config.h>
|
|
|
|
|
|
|
|
#if USE_AWS_S3
|
|
|
|
|
2019-05-23 09:03:39 +00:00
|
|
|
#include <Storages/IStorage.h>
|
2019-05-29 12:54:31 +00:00
|
|
|
#include <Poco/URI.h>
|
2019-05-23 09:03:39 +00:00
|
|
|
#include <common/logger_useful.h>
|
|
|
|
#include <ext/shared_ptr_helper.h>
|
2019-12-11 14:21:48 +00:00
|
|
|
|
|
|
|
namespace Aws::S3
|
|
|
|
{
|
|
|
|
class S3Client;
|
|
|
|
}
|
2019-09-22 22:13:42 +00:00
|
|
|
|
2019-05-23 09:03:39 +00:00
|
|
|
namespace DB
|
|
|
|
{
|
2019-12-03 16:23:24 +00:00
|
|
|
|
2019-05-31 07:27:14 +00:00
|
|
|
/**
|
2019-06-01 21:18:20 +00:00
|
|
|
* This class represents table engine for external S3 urls.
|
2019-05-31 07:27:14 +00:00
|
|
|
* It sends HTTP GET to server when select is called and
|
2019-06-01 21:18:20 +00:00
|
|
|
* HTTP PUT when insert is called.
|
2019-05-31 07:27:14 +00:00
|
|
|
*/
|
2020-03-19 23:48:53 +00:00
|
|
|
class StorageS3 final : public ext::shared_ptr_helper<StorageS3>, public IStorage
|
2019-05-23 09:03:39 +00:00
|
|
|
{
|
|
|
|
public:
|
2019-12-06 14:37:21 +00:00
|
|
|
StorageS3(const S3::URI & uri,
|
2019-11-04 19:57:03 +00:00
|
|
|
const String & access_key_id,
|
|
|
|
const String & secret_access_key,
|
2019-12-04 16:06:55 +00:00
|
|
|
const StorageID & table_id_,
|
2019-06-01 21:18:20 +00:00
|
|
|
const String & format_name_,
|
2019-09-23 12:41:59 +00:00
|
|
|
UInt64 min_upload_part_size_,
|
2019-06-01 21:18:20 +00:00
|
|
|
const ColumnsDescription & columns_,
|
2019-09-22 22:13:42 +00:00
|
|
|
const ConstraintsDescription & constraints_,
|
2019-11-19 12:46:07 +00:00
|
|
|
Context & context_,
|
|
|
|
const String & compression_method_);
|
2019-06-01 21:18:20 +00:00
|
|
|
|
|
|
|
String getName() const override
|
|
|
|
{
|
|
|
|
return "S3";
|
|
|
|
}
|
|
|
|
|
|
|
|
Block getHeaderBlock(const Names & /*column_names*/) const
|
|
|
|
{
|
|
|
|
return getSampleBlock();
|
|
|
|
}
|
|
|
|
|
2020-02-19 16:07:28 +00:00
|
|
|
Pipes read(
|
2019-09-22 22:13:42 +00:00
|
|
|
const Names & column_names,
|
2019-05-23 09:03:39 +00:00
|
|
|
const SelectQueryInfo & query_info,
|
|
|
|
const Context & context,
|
|
|
|
QueryProcessingStage::Enum processed_stage,
|
|
|
|
size_t max_block_size,
|
|
|
|
unsigned num_streams) override;
|
|
|
|
|
2019-05-31 07:27:14 +00:00
|
|
|
BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override;
|
2019-05-23 09:03:39 +00:00
|
|
|
|
2019-09-22 22:13:42 +00:00
|
|
|
private:
|
2019-12-06 14:37:21 +00:00
|
|
|
S3::URI uri;
|
2019-05-31 07:27:14 +00:00
|
|
|
const Context & context_global;
|
2019-05-23 09:03:39 +00:00
|
|
|
|
2019-05-31 07:27:14 +00:00
|
|
|
String format_name;
|
2019-09-23 12:41:59 +00:00
|
|
|
UInt64 min_upload_part_size;
|
2019-11-19 12:46:07 +00:00
|
|
|
String compression_method;
|
2019-12-03 16:23:24 +00:00
|
|
|
std::shared_ptr<Aws::S3::S3Client> client;
|
2019-05-23 09:03:39 +00:00
|
|
|
};
|
2019-12-11 14:21:48 +00:00
|
|
|
|
2019-05-23 09:03:39 +00:00
|
|
|
}
|
2019-12-11 14:21:48 +00:00
|
|
|
|
2019-12-09 12:36:06 +00:00
|
|
|
#endif
|