2017-10-03 23:39:24 +00:00
|
|
|
#pragma once
|
2018-12-27 13:27:01 +00:00
|
|
|
|
2018-09-26 05:12:32 +00:00
|
|
|
#include <Core/BackgroundSchedulePool.h>
|
2018-12-27 13:27:01 +00:00
|
|
|
#include <Core/NamesAndTypes.h>
|
2017-10-03 23:39:24 +00:00
|
|
|
#include <DataStreams/IBlockOutputStream.h>
|
2018-12-27 13:27:01 +00:00
|
|
|
#include <Storages/IStorage.h>
|
2017-10-03 23:39:24 +00:00
|
|
|
#include <Poco/Event.h>
|
2017-12-16 01:45:11 +00:00
|
|
|
#include <Poco/Semaphore.h>
|
2018-12-27 13:27:01 +00:00
|
|
|
#include <ext/shared_ptr_helper.h>
|
|
|
|
|
2018-12-27 14:18:20 +00:00
|
|
|
#include <cppkafka/cppkafka.h>
|
2018-12-27 13:27:01 +00:00
|
|
|
#include <mutex>
|
2017-10-03 23:39:24 +00:00
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2018-12-27 14:18:20 +00:00
|
|
|
using ConsumerPtr = std::shared_ptr<cppkafka::Consumer>;
|
2017-10-03 23:39:24 +00:00
|
|
|
|
|
|
|
/** Implements a Kafka queue table engine that can be used as a persistent queue / buffer,
|
|
|
|
* or as a basic building block for creating pipelines with a continuous insertion / ETL.
|
|
|
|
*/
|
|
|
|
class StorageKafka : public ext::shared_ptr_helper<StorageKafka>, public IStorage
|
|
|
|
{
|
|
|
|
friend class KafkaBlockInputStream;
|
|
|
|
friend class KafkaBlockOutputStream;
|
|
|
|
|
|
|
|
public:
|
|
|
|
std::string getName() const override { return "Kafka"; }
|
|
|
|
std::string getTableName() const override { return table_name; }
|
2019-03-29 20:31:06 +00:00
|
|
|
std::string getDatabaseName() const override { return database_name; }
|
2017-10-03 23:39:24 +00:00
|
|
|
|
|
|
|
void startup() override;
|
|
|
|
void shutdown() override;
|
|
|
|
|
|
|
|
BlockInputStreams read(
|
|
|
|
const Names & column_names,
|
|
|
|
const SelectQueryInfo & query_info,
|
|
|
|
const Context & context,
|
2018-04-19 14:47:09 +00:00
|
|
|
QueryProcessingStage::Enum processed_stage,
|
2019-02-18 23:38:44 +00:00
|
|
|
size_t max_block_size,
|
2017-10-03 23:39:24 +00:00
|
|
|
unsigned num_streams) override;
|
|
|
|
|
2017-12-01 21:13:25 +00:00
|
|
|
void rename(const String & /*new_path_to_db*/, const String & new_database_name, const String & new_table_name) override
|
2017-10-03 23:39:24 +00:00
|
|
|
{
|
|
|
|
table_name = new_table_name;
|
|
|
|
database_name = new_database_name;
|
|
|
|
}
|
|
|
|
|
|
|
|
void updateDependencies() override;
|
|
|
|
|
|
|
|
private:
|
2017-12-16 01:45:11 +00:00
|
|
|
// Configuration and state
|
2017-10-03 23:39:24 +00:00
|
|
|
String table_name;
|
|
|
|
String database_name;
|
2019-01-04 12:10:00 +00:00
|
|
|
Context global_context;
|
2017-10-03 23:39:24 +00:00
|
|
|
Names topics;
|
2017-12-16 01:45:11 +00:00
|
|
|
const String brokers;
|
|
|
|
const String group;
|
2017-10-03 23:39:24 +00:00
|
|
|
const String format_name;
|
2018-07-18 05:22:01 +00:00
|
|
|
// Optional row delimiter for generating char delimited stream
|
|
|
|
// in order to make various input stream parsers happy.
|
|
|
|
char row_delimiter;
|
2017-10-03 23:39:24 +00:00
|
|
|
const String schema_name;
|
2018-10-16 18:21:27 +00:00
|
|
|
/// Total number of consumers
|
2018-03-19 17:52:48 +00:00
|
|
|
size_t num_consumers;
|
2018-10-16 18:21:27 +00:00
|
|
|
/// Maximum block size for insertion into this table
|
2019-02-10 16:55:12 +00:00
|
|
|
UInt64 max_block_size;
|
2018-03-19 17:52:48 +00:00
|
|
|
/// Number of actually created consumers.
|
|
|
|
/// Can differ from num_consumers in case of exception in startup() (or if startup() hasn't been called).
|
|
|
|
/// In this case we still need to be able to shutdown() properly.
|
|
|
|
size_t num_created_consumers = 0;
|
2017-12-16 01:45:11 +00:00
|
|
|
Poco::Logger * log;
|
2017-10-03 23:39:24 +00:00
|
|
|
|
2017-12-16 01:45:11 +00:00
|
|
|
// Consumer list
|
|
|
|
Poco::Semaphore semaphore;
|
2017-10-03 23:39:24 +00:00
|
|
|
std::mutex mutex;
|
2017-12-16 01:45:11 +00:00
|
|
|
std::vector<ConsumerPtr> consumers; /// Available consumers
|
|
|
|
|
2019-01-18 12:48:38 +00:00
|
|
|
size_t skip_broken;
|
|
|
|
|
2017-12-16 01:45:11 +00:00
|
|
|
// Stream thread
|
2018-09-26 05:12:32 +00:00
|
|
|
BackgroundSchedulePool::TaskHolder task;
|
2017-12-16 01:45:11 +00:00
|
|
|
std::atomic<bool> stream_cancelled{false};
|
|
|
|
|
2018-12-27 14:18:20 +00:00
|
|
|
cppkafka::Configuration createConsumerConfiguration();
|
2017-12-20 19:09:13 +00:00
|
|
|
ConsumerPtr claimConsumer();
|
|
|
|
ConsumerPtr tryClaimConsumer(long wait_ms);
|
2017-12-16 01:45:11 +00:00
|
|
|
void pushConsumer(ConsumerPtr c);
|
2017-10-03 23:39:24 +00:00
|
|
|
|
2017-11-03 21:50:22 +00:00
|
|
|
void streamThread();
|
2018-09-26 05:12:32 +00:00
|
|
|
bool streamToViews();
|
2018-10-16 18:21:27 +00:00
|
|
|
bool checkDependencies(const String & database_name, const String & table_name);
|
2017-11-03 21:50:22 +00:00
|
|
|
|
|
|
|
protected:
|
2017-10-03 23:39:24 +00:00
|
|
|
StorageKafka(
|
|
|
|
const std::string & table_name_,
|
|
|
|
const std::string & database_name_,
|
|
|
|
Context & context_,
|
2018-03-06 20:18:34 +00:00
|
|
|
const ColumnsDescription & columns_,
|
2017-10-03 23:39:24 +00:00
|
|
|
const String & brokers_, const String & group_, const Names & topics_,
|
2018-10-16 18:21:27 +00:00
|
|
|
const String & format_name_, char row_delimiter_, const String & schema_name_,
|
2019-02-10 16:55:12 +00:00
|
|
|
size_t num_consumers_, UInt64 max_block_size_, size_t skip_broken);
|
2017-10-03 23:39:24 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
}
|