mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 08:40:50 +00:00
Merge pull request #20301 from azat/numbers-numeric-limit
Accept arbitrary numeric types for numbers() arguments (for scientific notation)
This commit is contained in:
commit
d15d340691
@ -6,6 +6,7 @@
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Storages/System/StorageSystemNumbers.h>
|
||||
#include <Interpreters/evaluateConstantExpression.h>
|
||||
#include <Interpreters/convertFieldToType.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include "registerTableFunctions.h"
|
||||
@ -17,6 +18,7 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
|
||||
@ -56,7 +58,16 @@ void registerTableFunctionNumbers(TableFunctionFactory & factory)
|
||||
template <bool multithreaded>
|
||||
UInt64 TableFunctionNumbers<multithreaded>::evaluateArgument(const Context & context, ASTPtr & argument) const
|
||||
{
|
||||
return evaluateConstantExpressionOrIdentifierAsLiteral(argument, context)->as<ASTLiteral &>().value.safeGet<UInt64>();
|
||||
const auto & [field, type] = evaluateConstantExpression(argument, context);
|
||||
|
||||
if (!isNativeNumber(type))
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} expression, must be numeric type", type->getName());
|
||||
|
||||
Field converted = convertFieldToType(field, DataTypeUInt64());
|
||||
if (converted.isNull())
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The value {} is not representable as UInt64", applyVisitor(FieldVisitorToString(), field));
|
||||
|
||||
return converted.safeGet<UInt64>();
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -6,7 +6,7 @@
|
||||
SETTINGS index_granularity = 8192
|
||||
AS
|
||||
SELECT CAST(reinterpretAsString(number), 'SimpleAggregateFunction(any, String)') AS key
|
||||
FROM numbers_mt(toUInt64(5e6))
|
||||
FROM numbers_mt(5e6)
|
||||
SETTINGS max_insert_threads = 16
|
||||
</create_query>
|
||||
|
||||
|
@ -5,5 +5,5 @@
|
||||
-- test is just to ensure that the result is different (and to document the
|
||||
-- outcome).
|
||||
|
||||
SELECT uniqCombined(number) FROM numbers(toUInt64(1e7));
|
||||
SELECT uniqCombined64(number) FROM numbers(toUInt64(1e7));
|
||||
SELECT uniqCombined(number) FROM numbers(1e7);
|
||||
SELECT uniqCombined64(number) FROM numbers(1e7);
|
||||
|
@ -5,45 +5,45 @@
|
||||
-- HashTable for UInt32 (used until (1<<13) elements), hence 8192 elements
|
||||
SELECT 'UInt32';
|
||||
SET max_memory_usage = 4000000;
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 8192) AS k, uniqCombined(number % 8192) u FROM numbers(toUInt64(8192 * 100)) GROUP BY k); -- { serverError 241 }
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 8192) AS k, uniqCombined(number % 8192) u FROM numbers(8192 * 100) GROUP BY k); -- { serverError 241 }
|
||||
SET max_memory_usage = 9830400;
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 8192) AS k, uniqCombined(number % 8192) u FROM numbers(toUInt64(8192 * 100)) GROUP BY k);
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 8192) AS k, uniqCombined(number % 8192) u FROM numbers(8192 * 100) GROUP BY k);
|
||||
|
||||
-- HashTable for UInt64 (used until (1<<12) elements), hence 4096 elements
|
||||
SELECT 'UInt64';
|
||||
SET max_memory_usage = 4000000;
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 4096) AS k, uniqCombined(reinterpretAsString(number % 4096)) u FROM numbers(toUInt64(4096 * 100)) GROUP BY k); -- { serverError 241 }
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 4096) AS k, uniqCombined(reinterpretAsString(number % 4096)) u FROM numbers(4096 * 100) GROUP BY k); -- { serverError 241 }
|
||||
SET max_memory_usage = 9830400;
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 4096) AS k, uniqCombined(reinterpretAsString(number % 4096)) u FROM numbers(toUInt64(4096 * 100)) GROUP BY k);
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 4096) AS k, uniqCombined(reinterpretAsString(number % 4096)) u FROM numbers(4096 * 100) GROUP BY k);
|
||||
|
||||
SELECT 'K=16';
|
||||
|
||||
-- HashTable for UInt32 (used until (1<<12) elements), hence 4096 elements
|
||||
SELECT 'UInt32';
|
||||
SET max_memory_usage = 2000000;
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 4096) AS k, uniqCombined(16)(number % 4096) u FROM numbers(toUInt64(4096 * 100)) GROUP BY k); -- { serverError 241 }
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 4096) AS k, uniqCombined(16)(number % 4096) u FROM numbers(4096 * 100) GROUP BY k); -- { serverError 241 }
|
||||
SET max_memory_usage = 4915200;
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 4096) AS k, uniqCombined(16)(number % 4096) u FROM numbers(toUInt64(4096 * 100)) GROUP BY k);
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 4096) AS k, uniqCombined(16)(number % 4096) u FROM numbers(4096 * 100) GROUP BY k);
|
||||
|
||||
-- HashTable for UInt64 (used until (1<<11) elements), hence 2048 elements
|
||||
SELECT 'UInt64';
|
||||
SET max_memory_usage = 2000000;
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 2048) AS k, uniqCombined(16)(reinterpretAsString(number % 2048)) u FROM numbers(toUInt64(2048 * 100)) GROUP BY k); -- { serverError 241 }
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 2048) AS k, uniqCombined(16)(reinterpretAsString(number % 2048)) u FROM numbers(2048 * 100) GROUP BY k); -- { serverError 241 }
|
||||
SET max_memory_usage = 4915200;
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 2048) AS k, uniqCombined(16)(reinterpretAsString(number % 2048)) u FROM numbers(toUInt64(2048 * 100)) GROUP BY k);
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 2048) AS k, uniqCombined(16)(reinterpretAsString(number % 2048)) u FROM numbers(2048 * 100) GROUP BY k);
|
||||
|
||||
SELECT 'K=18';
|
||||
|
||||
-- HashTable for UInt32 (used until (1<<14) elements), hence 16384 elements
|
||||
SELECT 'UInt32';
|
||||
SET max_memory_usage = 8000000;
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 16384) AS k, uniqCombined(18)(number % 16384) u FROM numbers(toUInt64(16384 * 100)) GROUP BY k); -- { serverError 241 }
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 16384) AS k, uniqCombined(18)(number % 16384) u FROM numbers(16384 * 100) GROUP BY k); -- { serverError 241 }
|
||||
SET max_memory_usage = 19660800;
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 16384) AS k, uniqCombined(18)(number % 16384) u FROM numbers(toUInt64(16384 * 100)) GROUP BY k);
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 16384) AS k, uniqCombined(18)(number % 16384) u FROM numbers(16384 * 100) GROUP BY k);
|
||||
|
||||
-- HashTable for UInt64 (used until (1<<13) elements), hence 8192 elements
|
||||
SELECT 'UInt64';
|
||||
SET max_memory_usage = 8000000;
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 8192) AS k, uniqCombined(18)(reinterpretAsString(number % 8192)) u FROM numbers(toUInt64(8192 * 100)) GROUP BY k); -- { serverError 241 }
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 8192) AS k, uniqCombined(18)(reinterpretAsString(number % 8192)) u FROM numbers(8192 * 100) GROUP BY k); -- { serverError 241 }
|
||||
SET max_memory_usage = 19660800;
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 8192) AS k, uniqCombined(18)(reinterpretAsString(number % 8192)) u FROM numbers(toUInt64(8192 * 100)) GROUP BY k);
|
||||
SELECT sum(u) FROM (SELECT intDiv(number, 8192) AS k, uniqCombined(18)(reinterpretAsString(number % 8192)) u FROM numbers(8192 * 100) GROUP BY k);
|
||||
|
@ -33,7 +33,7 @@ function execute_group_by()
|
||||
"--max_memory_usage_for_user="$((150<<20))
|
||||
"--max_threads=2"
|
||||
)
|
||||
execute_null "${opts[@]}" <<<'SELECT uniq(number) FROM numbers_mt(toUInt64(1e6)) GROUP BY number % 5e5'
|
||||
execute_null "${opts[@]}" <<<'SELECT uniq(number) FROM numbers_mt(1e6) GROUP BY number % 5e5'
|
||||
}
|
||||
|
||||
# This is needed to keep at least one running query for user for the time of test.
|
||||
|
@ -1,7 +1,7 @@
|
||||
drop table if exists data_01513;
|
||||
create table data_01513 (key String) engine=MergeTree() order by key;
|
||||
-- 10e3 groups, 1e3 keys each
|
||||
insert into data_01513 select number%10e3 from numbers(toUInt64(2e6));
|
||||
insert into data_01513 select number%10e3 from numbers(2e6);
|
||||
-- reduce number of parts to 1
|
||||
optimize table data_01513 final;
|
||||
|
@ -10,8 +10,8 @@ set max_block_size=40960;
|
||||
-- MergeSortingTransform: Re-merging intermediate ORDER BY data (20 blocks with 819200 rows) to save memory consumption
|
||||
-- MergeSortingTransform: Memory usage is lowered from 186.25 MiB to 95.00 MiB
|
||||
-- MergeSortingTransform: Re-merging is not useful (memory usage was not lowered by remerge_sort_lowered_memory_bytes_ratio=2.0)
|
||||
select number k, repeat(toString(number), 11) v1, repeat(toString(number), 12) v2 from numbers(toUInt64(3e6)) order by k limit 400e3 format Null; -- { serverError 241 }
|
||||
select number k, repeat(toString(number), 11) v1, repeat(toString(number), 12) v2 from numbers(toUInt64(3e6)) order by k limit 400e3 settings remerge_sort_lowered_memory_bytes_ratio=2. format Null; -- { serverError 241 }
|
||||
select number k, repeat(toString(number), 11) v1, repeat(toString(number), 12) v2 from numbers(3e6) order by k limit 400e3 format Null; -- { serverError 241 }
|
||||
select number k, repeat(toString(number), 11) v1, repeat(toString(number), 12) v2 from numbers(3e6) order by k limit 400e3 settings remerge_sort_lowered_memory_bytes_ratio=2. format Null; -- { serverError 241 }
|
||||
|
||||
-- remerge_sort_lowered_memory_bytes_ratio 1.9 is good (need at least 1.91/0.98=1.94)
|
||||
-- MergeSortingTransform: Re-merging intermediate ORDER BY data (20 blocks with 819200 rows) to save memory consumption
|
||||
@ -26,4 +26,4 @@ select number k, repeat(toString(number), 11) v1, repeat(toString(number), 12) v
|
||||
-- MergeSortingTransform: Memory usage is lowered from 188.13 MiB to 95.00 MiB
|
||||
-- MergeSortingTransform: Re-merging intermediate ORDER BY data (20 blocks with 809600 rows) to save memory consumption
|
||||
-- MergeSortingTransform: Memory usage is lowered from 188.13 MiB to 95.00 MiB
|
||||
select number k, repeat(toString(number), 11) v1, repeat(toString(number), 12) v2 from numbers(toUInt64(3e6)) order by k limit 400e3 settings remerge_sort_lowered_memory_bytes_ratio=1.9 format Null;
|
||||
select number k, repeat(toString(number), 11) v1, repeat(toString(number), 12) v2 from numbers(3e6) order by k limit 400e3 settings remerge_sort_lowered_memory_bytes_ratio=1.9 format Null;
|
||||
|
@ -5,7 +5,7 @@ create table data_01641 (key Int, value String) engine=MergeTree order by (key,
|
||||
-- peak memory usage is 170MiB
|
||||
set max_memory_usage='200Mi';
|
||||
system stop merges data_01641;
|
||||
insert into data_01641 select number, toString(number) from numbers(toUInt64(120e6));
|
||||
insert into data_01641 select number, toString(number) from numbers(120e6);
|
||||
|
||||
-- peak:
|
||||
-- - is 21MiB if background merges already scheduled
|
@ -0,0 +1,5 @@
|
||||
select * from numbers(1e2) format Null;
|
||||
select * from numbers_mt(1e2) format Null;
|
||||
select * from numbers_mt('100') format Null; -- { serverError 43 }
|
||||
select * from numbers_mt(inf) format Null; -- { serverError 43 }
|
||||
select * from numbers_mt(nan) format Null; -- { serverError 43 }
|
Loading…
Reference in New Issue
Block a user