mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-10-04 07:30:52 +00:00
Use Port::Data instead of Chunk in LazyOutputFormat.
This commit is contained in:
parent
3fd0b69fab
commit
7ab38d5007
@ -5,6 +5,11 @@
|
|||||||
namespace DB
|
namespace DB
|
||||||
{
|
{
|
||||||
|
|
||||||
|
namespace ErrorCodes
|
||||||
|
{
|
||||||
|
extern const int NOT_IMPLEMENTED;
|
||||||
|
}
|
||||||
|
|
||||||
IOutputFormat::IOutputFormat(const Block & header_, WriteBuffer & out_)
|
IOutputFormat::IOutputFormat(const Block & header_, WriteBuffer & out_)
|
||||||
: IProcessor({header_, header_, header_}, {}), out(out_)
|
: IProcessor({header_, header_, header_}, {}), out(out_)
|
||||||
{
|
{
|
||||||
@ -30,7 +35,7 @@ IOutputFormat::Status IOutputFormat::prepare()
|
|||||||
if (!input.hasData())
|
if (!input.hasData())
|
||||||
return Status::NeedData;
|
return Status::NeedData;
|
||||||
|
|
||||||
current_chunk = input.pull(true);
|
current_chunk = input.pullData(true);
|
||||||
current_block_kind = kind;
|
current_block_kind = kind;
|
||||||
has_input = true;
|
has_input = true;
|
||||||
return Status::Ready;
|
return Status::Ready;
|
||||||
@ -44,23 +49,31 @@ IOutputFormat::Status IOutputFormat::prepare()
|
|||||||
return Status::Finished;
|
return Status::Finished;
|
||||||
}
|
}
|
||||||
|
|
||||||
static Chunk prepareTotals(Chunk chunk)
|
static Port::Data prepareTotals(Port::Data data)
|
||||||
{
|
{
|
||||||
if (!chunk.hasRows())
|
if (data.exception)
|
||||||
|
return data;
|
||||||
|
|
||||||
|
if (!data.chunk.hasRows())
|
||||||
return {};
|
return {};
|
||||||
|
|
||||||
if (chunk.getNumRows() > 1)
|
if (data.chunk.getNumRows() > 1)
|
||||||
{
|
{
|
||||||
/// This may happen if something like ARRAY JOIN was executed on totals.
|
/// This may happen if something like ARRAY JOIN was executed on totals.
|
||||||
/// Skip rows except the first one.
|
/// Skip rows except the first one.
|
||||||
auto columns = chunk.detachColumns();
|
auto columns = data.chunk.detachColumns();
|
||||||
for (auto & column : columns)
|
for (auto & column : columns)
|
||||||
column = column->cut(0, 1);
|
column = column->cut(0, 1);
|
||||||
|
|
||||||
chunk.setColumns(std::move(columns), 1);
|
data.chunk.setColumns(std::move(columns), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
return chunk;
|
return data;
|
||||||
|
}
|
||||||
|
|
||||||
|
void IOutputFormat::consume(Chunk)
|
||||||
|
{
|
||||||
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method consume s not implemented for {}", getName());
|
||||||
}
|
}
|
||||||
|
|
||||||
void IOutputFormat::work()
|
void IOutputFormat::work()
|
||||||
@ -84,18 +97,25 @@ void IOutputFormat::work()
|
|||||||
switch (current_block_kind)
|
switch (current_block_kind)
|
||||||
{
|
{
|
||||||
case Main:
|
case Main:
|
||||||
result_rows += current_chunk.getNumRows();
|
{
|
||||||
result_bytes += current_chunk.allocatedBytes();
|
result_rows += current_chunk.chunk.getNumRows();
|
||||||
|
result_bytes += current_chunk.chunk.allocatedBytes();
|
||||||
consume(std::move(current_chunk));
|
consume(std::move(current_chunk));
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
case Totals:
|
case Totals:
|
||||||
if (auto totals = prepareTotals(std::move(current_chunk)))
|
{
|
||||||
|
auto totals = prepareTotals(std::move(current_chunk));
|
||||||
|
if (totals.exception || totals.chunk)
|
||||||
consumeTotals(std::move(totals));
|
consumeTotals(std::move(totals));
|
||||||
break;
|
break;
|
||||||
|
}
|
||||||
case Extremes:
|
case Extremes:
|
||||||
|
{
|
||||||
consumeExtremes(std::move(current_chunk));
|
consumeExtremes(std::move(current_chunk));
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (auto_flush)
|
if (auto_flush)
|
||||||
flush();
|
flush();
|
||||||
|
@ -28,7 +28,7 @@ public:
|
|||||||
protected:
|
protected:
|
||||||
WriteBuffer & out;
|
WriteBuffer & out;
|
||||||
|
|
||||||
Chunk current_chunk;
|
Port::Data current_chunk;
|
||||||
PortKind current_block_kind = PortKind::Main;
|
PortKind current_block_kind = PortKind::Main;
|
||||||
bool has_input = false;
|
bool has_input = false;
|
||||||
bool finished = false;
|
bool finished = false;
|
||||||
@ -39,9 +39,14 @@ protected:
|
|||||||
|
|
||||||
RowsBeforeLimitCounterPtr rows_before_limit_counter;
|
RowsBeforeLimitCounterPtr rows_before_limit_counter;
|
||||||
|
|
||||||
virtual void consume(Chunk) = 0;
|
virtual void consume(Chunk);
|
||||||
virtual void consumeTotals(Chunk) {}
|
virtual void consumeTotals(Chunk) {}
|
||||||
virtual void consumeExtremes(Chunk) {}
|
virtual void consumeExtremes(Chunk) {}
|
||||||
|
|
||||||
|
virtual void consume(Port::Data data) { consume(data.getChunkOrTrow()); }
|
||||||
|
virtual void consumeTotals(Port::Data data) { consumeTotals(data.getChunkOrTrow()); }
|
||||||
|
virtual void consumeExtremes(Port::Data data) { consumeExtremes(data.getChunkOrTrow()); }
|
||||||
|
|
||||||
virtual void finalize() {}
|
virtual void finalize() {}
|
||||||
|
|
||||||
public:
|
public:
|
||||||
@ -77,8 +82,8 @@ public:
|
|||||||
virtual void doWritePrefix() {}
|
virtual void doWritePrefix() {}
|
||||||
virtual void doWriteSuffix() { finalize(); }
|
virtual void doWriteSuffix() { finalize(); }
|
||||||
|
|
||||||
void setTotals(const Block & totals) { consumeTotals(Chunk(totals.getColumns(), totals.rows())); }
|
void setTotals(const Block & totals) { consumeTotals(Port::Data{.chunk = Chunk(totals.getColumns(), totals.rows())}); }
|
||||||
void setExtremes(const Block & extremes) { consumeExtremes(Chunk(extremes.getColumns(), extremes.rows())); }
|
void setExtremes(const Block & extremes) { consumeExtremes(Port::Data{.chunk = Chunk(extremes.getColumns(), extremes.rows())}); }
|
||||||
|
|
||||||
size_t getResultRows() const { return result_rows; }
|
size_t getResultRows() const { return result_rows; }
|
||||||
size_t getResultBytes() const { return result_bytes; }
|
size_t getResultBytes() const { return result_bytes; }
|
||||||
|
@ -15,24 +15,24 @@ Chunk LazyOutputFormat::getChunk(UInt64 milliseconds)
|
|||||||
return {};
|
return {};
|
||||||
}
|
}
|
||||||
|
|
||||||
Chunk chunk;
|
Port::Data data;
|
||||||
if (!queue.tryPop(chunk, milliseconds))
|
if (!queue.tryPop(data, milliseconds))
|
||||||
return {};
|
return {};
|
||||||
|
|
||||||
if (chunk)
|
if (!data.exception)
|
||||||
info.update(chunk.getNumRows(), chunk.allocatedBytes());
|
info.update(data.chunk.getNumRows(), data.chunk.allocatedBytes());
|
||||||
|
|
||||||
return chunk;
|
return data.getChunkOrTrow();
|
||||||
}
|
}
|
||||||
|
|
||||||
Chunk LazyOutputFormat::getTotals()
|
Chunk LazyOutputFormat::getTotals()
|
||||||
{
|
{
|
||||||
return std::move(totals);
|
return totals.getChunkOrTrow();
|
||||||
}
|
}
|
||||||
|
|
||||||
Chunk LazyOutputFormat::getExtremes()
|
Chunk LazyOutputFormat::getExtremes()
|
||||||
{
|
{
|
||||||
return std::move(extremes);
|
return extremes.getChunkOrTrow();
|
||||||
}
|
}
|
||||||
|
|
||||||
void LazyOutputFormat::setRowsBeforeLimit(size_t rows_before_limit)
|
void LazyOutputFormat::setRowsBeforeLimit(size_t rows_before_limit)
|
||||||
|
@ -37,28 +37,28 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
void consume(Chunk chunk) override
|
void consume(Port::Data data) override
|
||||||
{
|
{
|
||||||
if (!finished_processing)
|
if (!finished_processing)
|
||||||
queue.emplace(std::move(chunk));
|
queue.emplace(std::move(data));
|
||||||
}
|
}
|
||||||
|
|
||||||
void consumeTotals(Chunk chunk) override { totals = std::move(chunk); }
|
void consumeTotals(Port::Data data) override { totals = std::move(data); }
|
||||||
void consumeExtremes(Chunk chunk) override { extremes = std::move(chunk); }
|
void consumeExtremes(Port::Data data) override { extremes = std::move(data); }
|
||||||
|
|
||||||
void finalize() override
|
void finalize() override
|
||||||
{
|
{
|
||||||
finished_processing = true;
|
finished_processing = true;
|
||||||
|
|
||||||
/// In case we are waiting for result.
|
/// In case we are waiting for result.
|
||||||
queue.emplace(Chunk());
|
queue.emplace(Port::Data{});
|
||||||
}
|
}
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
ConcurrentBoundedQueue<Chunk> queue;
|
ConcurrentBoundedQueue<Port::Data> queue;
|
||||||
Chunk totals;
|
Port::Data totals;
|
||||||
Chunk extremes;
|
Port::Data extremes;
|
||||||
|
|
||||||
/// Is not used.
|
/// Is not used.
|
||||||
static WriteBuffer out;
|
static WriteBuffer out;
|
||||||
|
@ -60,6 +60,14 @@ protected:
|
|||||||
/// Note: std::variant can be used. But move constructor for it can't be inlined.
|
/// Note: std::variant can be used. But move constructor for it can't be inlined.
|
||||||
Chunk chunk;
|
Chunk chunk;
|
||||||
std::exception_ptr exception;
|
std::exception_ptr exception;
|
||||||
|
|
||||||
|
Chunk getChunkOrTrow()
|
||||||
|
{
|
||||||
|
if (exception)
|
||||||
|
std::rethrow_exception(std::move(exception));
|
||||||
|
|
||||||
|
return std::move(chunk);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
private:
|
private:
|
||||||
@ -303,12 +311,7 @@ public:
|
|||||||
|
|
||||||
Chunk ALWAYS_INLINE pull(bool set_not_needed = false)
|
Chunk ALWAYS_INLINE pull(bool set_not_needed = false)
|
||||||
{
|
{
|
||||||
auto data_ = pullData(set_not_needed);
|
return pullData(set_not_needed).getChunkOrTrow();
|
||||||
|
|
||||||
if (data_.exception)
|
|
||||||
std::rethrow_exception(data_.exception);
|
|
||||||
|
|
||||||
return std::move(data_.chunk);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
bool ALWAYS_INLINE isFinished() const
|
bool ALWAYS_INLINE isFinished() const
|
||||||
|
Loading…
Reference in New Issue
Block a user