thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-23 08:02:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improvement; added a test

Browse Source
This commit is contained in:
Alexey Milovidov 2020-05-17 10:17:54 +03:00
parent 20c9c2d3ec
commit de8120d69a
5 changed files with 80 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/Processors/ISource.cpp
View File

@ -56,11 +56,6 @@ void ISource::work()
finished = true;
throw;
}
// {
// current_chunk = std::current_exception();
// has_input = true;
// got_exception = true;
// }
}
}

14
src/Processors/Sources/SourceWithProgress.cpp
View File

@ -15,7 +15,9 @@ namespace ErrorCodes
void SourceWithProgress::work()
{
if (!limits.speed_limits.checkTimeLimit(total_stopwatch.elapsed(), limits.timeout_overflow_mode))
{
cancel();
}
else
{
was_progress_called = false;
@ -57,7 +59,13 @@ void SourceWithProgress::progress(const Progress & value)
/// The total amount of data processed or intended for processing in all sources, possibly on remote servers.
ProgressValues progress = process_list_elem->getProgressIn();
size_t total_rows_estimate = std::max(progress.read_rows, progress.total_rows_to_read);
/// If the mode is "throw" and estimate of total rows is known, then throw early if an estimate is too high.
/// If the mode is "break", then allow to read before limit even if estimate is very high.
size_t rows_to_check_limit = progress.read_rows;
if (limits.size_limits.overflow_mode == OverflowMode::THROW && progress.total_rows_to_read > progress.read_rows)
rows_to_check_limit = progress.total_rows_to_read;
/// Check the restrictions on the
/// * amount of data to read
@ -67,9 +75,11 @@ void SourceWithProgress::progress(const Progress & value)
if (limits.mode == LimitsMode::LIMITS_TOTAL)
{
if (!limits.size_limits.check(total_rows_estimate, progress.read_bytes, "rows to read",
if (!limits.size_limits.check(rows_to_check_limit, progress.read_bytes, "rows or bytes to read",
ErrorCodes::TOO_MANY_ROWS, ErrorCodes::TOO_MANY_BYTES))
{
cancel();
}
}
size_t total_rows = progress.total_rows_to_read;

2
src/Processors/Transforms/LimitsCheckingTransform.cpp
View File

@ -58,7 +58,7 @@ void LimitsCheckingTransform::checkQuota(Chunk & chunk)
switch (limits.mode)
{
case LimitsMode::LIMITS_TOTAL:
/// Checked in `progress` method.
/// Checked in SourceWithProgress::progress method.
break;
case LimitsMode::LIMITS_CURRENT:

55
tests/queries/0_stateless/01132_max_rows_to_read.reference
View File

@ -1,6 +1,57 @@
19
20
30
19
20
21
20
20
20
20
0
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
0
1
2
3
4
5
6
7
8
9
10
0
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

15
tests/queries/0_stateless/01132_max_rows_to_read.sql
View File

@ -7,11 +7,24 @@ SELECT count() FROM numbers(19);
SELECT count() FROM numbers(20);
SELECT count() FROM numbers(21); -- { serverError 158 }
-- check early exception if the estimated number of rows is high
SELECT * FROM numbers(30); -- { serverError 158 }
SET read_overflow_mode = 'break';
SELECT count() FROM numbers(19);
SELECT count() FROM numbers(20);
SELECT count() FROM numbers(21);
SELECT count() FROM numbers(29); -- one extra block is read and it is Ok.
SELECT count() FROM numbers(29);
SELECT count() FROM numbers(30);
SELECT count() FROM numbers(31);
-- check that partial result is returned even if the estimated number of rows is high
SELECT * FROM numbers(30);
-- the same for uneven block sizes
-- NOTE: currently it outputs less amount of data; it will be better to output the latest block also
SET max_block_size = 11;
SELECT * FROM numbers(30);
SET max_block_size = 9;
SELECT * FROM numbers(30);