thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-25 17:12:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Freeze the Kafka buffer after first empty response (#5283)

Browse Source 
* Check inside inferior streams for cancellation while reading.
* Stop reading from Kafka buffer after first empty read.
This commit is contained in:
Ivan 2019-05-15 19:11:50 +03:00 committed by GitHub
parent 98359a6a09
commit 1ea9e3019d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dbms/src/Formats/BlockInputStreamFromRowInputStream.cpp
View File

@ -63,7 +63,7 @@ Block BlockInputStreamFromRowInputStream::readImpl()
if (rows_portion_size && batch == rows_portion_size)
{
batch = 0;
if (!checkTimeLimit())
if (!checkTimeLimit() || isCancelled())
break;
}

11
dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp
View File

@ -27,6 +27,8 @@ void ReadBufferFromKafkaConsumer::subscribe(const Names & topics)
consumer->poll(5s);
consumer->resume();
}
stalled = false;
}
void ReadBufferFromKafkaConsumer::unsubscribe()
@ -38,6 +40,12 @@ void ReadBufferFromKafkaConsumer::unsubscribe()
/// Do commit messages implicitly after we processed the previous batch.
bool ReadBufferFromKafkaConsumer::nextImpl()
{
/// NOTE: ReadBuffer was implemented with a immutable buffer contents in mind.
/// If we failed to poll any message once - don't try again.
/// Otherwise, the |poll_timeout| expectations get flawn.
if (stalled)
return false;
if (current == messages.end())
{
commit();
@ -48,7 +56,10 @@ bool ReadBufferFromKafkaConsumer::nextImpl()
}
if (messages.empty() || current == messages.end())
{
stalled = true;
return false;
}
if (auto err = current->get_error())
{

1
dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.h
View File

@ -38,6 +38,7 @@ private:
Poco::Logger * log;
const size_t batch_size = 1;
const size_t poll_timeout = 0;
bool stalled = false;
Messages messages;
Messages::const_iterator current;