thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-18 21:51:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix single carriage return processing in TSV file segmentation engine

Browse Source
This commit is contained in:
avogar 2023-08-14 11:33:41 +00:00
parent 47b551180b
commit 4c12914406
3 changed files with 18 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/Processors/Formats/Impl/TabSeparatedRowInputFormat.cpp
View File

@ -411,11 +411,6 @@ static std::pair<bool, size_t> fileSegmentationEngineTabSeparatedImpl(ReadBuffer
continue;
}
++number_of_rows;
if ((number_of_rows >= min_rows)
&& ((memory.size() + static_cast<size_t>(pos - in.position()) >= min_bytes) || (number_of_rows == max_rows)))
need_more_data = false;
if (*pos == '\n')
{
++pos;
@ -427,7 +422,14 @@ static std::pair<bool, size_t> fileSegmentationEngineTabSeparatedImpl(ReadBuffer
++pos;
if (loadAtPosition(in, memory, pos) && *pos == '\n')
++pos;
else
continue;
}
++number_of_rows;
if ((number_of_rows >= min_rows)
&& ((memory.size() + static_cast<size_t>(pos - in.position()) >= min_bytes) || (number_of_rows == max_rows)))
need_more_data = false;
}
saveUpToPosition(in, memory, pos);

1
tests/queries/0_stateless/02844_tsv_carriage_return_parallel_parsing.reference Normal file
View File

@ -0,0 +1 @@
1000000

10
tests/queries/0_stateless/02844_tsv_carriage_return_parallel_parsing.sh Executable file
View File

@ -0,0 +1,10 @@
#!/usr/bin/env bash
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_LOCAL -q "select 'Hello\rWorld' from numbers(1000000) format TSVRaw" > $CLICKHOUSE_TEST_UNIQUE_NAME.tsv
$CLICKHOUSE_LOCAL -q "select count() from file('$CLICKHOUSE_TEST_UNIQUE_NAME.tsv')"
rm $CLICKHOUSE_TEST_UNIQUE_NAME.tsv