thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 01:25:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #35296 from Avogar/fix-lz4

Browse Source 
Fix possible segfault while using lz4 compression
This commit is contained in:
Kruglov Pavel 2022-03-15 22:17:14 +01:00 committed by GitHub
commit 20e17a6d3e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 46 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
src/IO/Lz4InflatingReadBuffer.cpp
View File

@ -35,26 +35,39 @@ bool Lz4InflatingReadBuffer::nextImpl()
if (eof_flag)
return false;
if (!in_available)
bool need_more_input = false;
size_t ret;
do
{
in->nextIfAtEnd();
in_available = in->buffer().end() - in->position();
if (!in_available)
{
in->nextIfAtEnd();
in_available = in->buffer().end() - in->position();
}
in_data = reinterpret_cast<void *>(in->position());
out_data = reinterpret_cast<void *>(internal_buffer.begin());
out_available = internal_buffer.size();
size_t bytes_read = in_available;
size_t bytes_written = out_available;
ret = LZ4F_decompress(dctx, out_data, &bytes_written, in_data, &bytes_read, /* LZ4F_decompressOptions_t */ nullptr);
in_available -= bytes_read;
out_available -= bytes_written;
/// It may happen that we didn't get new uncompressed data
/// (for example if we read the end of frame). Load new data
/// in this case.
need_more_input = bytes_written == 0;
in->position() = in->buffer().end() - in_available;
}
while (need_more_input && !LZ4F_isError(ret) && !in->eof());
in_data = reinterpret_cast<void *>(in->position());
out_data = reinterpret_cast<void *>(internal_buffer.begin());
out_available = internal_buffer.size();
size_t bytes_read = in_available;
size_t bytes_written = out_available;
size_t ret = LZ4F_decompress(dctx, out_data, &bytes_written, in_data, &bytes_read, /* LZ4F_decompressOptions_t */ nullptr);
in_available -= bytes_read;
out_available -= bytes_written;
in->position() = in->buffer().end() - in_available;
working_buffer.resize(internal_buffer.size() - out_available);
if (LZ4F_isError(ret))
@ -70,12 +83,6 @@ bool Lz4InflatingReadBuffer::nextImpl()
return !working_buffer.empty();
}
/// It may happen that we didn't get new uncompressed data
/// (for example if we read the end of frame). Load new data
/// in this case.
if (working_buffer.empty())
return nextImpl();
return true;
}
}

1
tests/queries/0_stateless/02238_lz4_bug.reference Normal file
View File

@ -0,0 +1 @@
1000000 999999

15
tests/queries/0_stateless/02238_lz4_bug.sh Executable file
View File

@ -0,0 +1,15 @@
#!/usr/bin/env bash
# Tags: no-fasttest
# Tag no-fasttest: depends on brotli and bzip2
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS file"
${CLICKHOUSE_CLIENT} --query "CREATE TABLE file (x UInt64) ENGINE = File(TSV, '${CLICKHOUSE_DATABASE}/data.tsv.lz4')"
${CLICKHOUSE_CLIENT} --query "TRUNCATE TABLE file"
${CLICKHOUSE_CLIENT} --query "INSERT INTO file SELECT * FROM numbers(1000000)"
${CLICKHOUSE_CLIENT} --max_read_buffer_size=2 --query "SELECT count(), max(x) FROM file"
${CLICKHOUSE_CLIENT} --query "DROP TABLE file"