thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-18 20:32:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

log duration while reading parquet

Browse Source 
Change-Id: If79741b7456667a8dde3e355d9dc684c2dd84f4f
This commit is contained in:
copperybean 2024-02-21 00:17:30 +08:00
parent dbdff6c038
commit 8172f6cec0
2 changed files with 15 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp
View File

@ -673,6 +673,15 @@ void ParquetBlockInputFormat::scheduleMoreWorkIfNeeded(std::optional<size_t> row
}
}
Chunk ParquetBlockInputFormat::generate()
{
auto res = IInputFormat::generate();
if (!res)
LOG_INFO(&Poco::Logger::get("ParquetBlockInputFormat"), "{} ms consumed by reading parquet file", consumed_nanosecs / 1e6);
return res;
}
Chunk ParquetBlockInputFormat::read()
{
initializeIfNeeded();
@ -683,6 +692,8 @@ Chunk ParquetBlockInputFormat::read()
if (need_only_count)
return getChunkForCount(row_group_batches[row_group_batches_completed++].total_rows);
Stopwatch watch(CLOCK_MONOTONIC);
SCOPE_EXIT({ consumed_nanosecs += watch.elapsedNanoseconds(); });
std::unique_lock lock(mutex);
while (true)

4
src/Processors/Formats/Impl/ParquetBlockInputFormat.h
View File

@ -65,6 +65,8 @@ public:
size_t getApproxBytesReadForChunk() const override { return previous_approx_bytes_read_for_chunk; }
Chunk generate() override;
private:
Chunk read() override;
@ -286,6 +288,8 @@ private:
std::exception_ptr background_exception = nullptr;
std::atomic<int> is_stopped{0};
bool is_initialized = false;
UInt64 consumed_nanosecs = 0;
};
class ParquetSchemaReader : public ISchemaReader