From 57cd49b8d552eb71ac12839cdd01ed58425fb2f1 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Wed, 16 Feb 2022 17:19:10 -0400 Subject: [PATCH 001/139] test that insert_deduplication_token allow to do idempotent insertion with materialized_views --- ...ication_token_materialized_views.reference | 8 ++ ...deduplication_token_materialized_views.sql | 136 ++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.reference create mode 100644 tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.reference b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.reference new file mode 100644 index 00000000000..e0cc8f0ce63 --- /dev/null +++ b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.reference @@ -0,0 +1,8 @@ +deduplicate_blocks_in_dependent_materialized_views=0, insert_deduplication_token = no, results inconsitent +18 18 9 18 +deduplicate_blocks_in_dependent_materialized_views=1, insert_deduplication_token = no, results inconsitent +18 9 9 9 +deduplicate_blocks_in_dependent_materialized_views=0, insert_deduplication_token = yes, results inconsitent +18 18 9 18 +deduplicate_blocks_in_dependent_materialized_views=1, insert_deduplication_token = yes, results consitent +18 18 18 18 diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql new file mode 100644 index 00000000000..f5307fe0dee --- /dev/null +++ b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql @@ -0,0 +1,136 @@ +select 'deduplicate_blocks_in_dependent_materialized_views=0, insert_deduplication_token = no, results inconsitent'; + +drop table if exists test sync; +drop table if exists test_mv_a sync; +drop table if exists test_mv_b sync; +drop table if exists test_mv_c sync; + +set deduplicate_blocks_in_dependent_materialized_views=0; + +CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree ('/clickhouse/test/tables/test','1') ORDER BY tuple() ; + +CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_a','1') +order by tuple() AS SELECT A, count() c FROM test group by A; + +CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_b','1') +partition by A order by tuple() AS SELECT A, count() c FROM test group by A; + +CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_c','1') +order by tuple() AS SELECT A, count() c FROM test group by A; + +SET max_partitions_per_insert_block = 1; +INSERT INTO test SELECT number%3, 1 FROM numbers(9); -- { serverError 252 } +SET max_partitions_per_insert_block = 0; +INSERT INTO test SELECT number%3, 1 FROM numbers(9); +INSERT INTO test SELECT number%3, 2 FROM numbers(9); + +select + (select count() from test), + (select sum(c) from test_mv_a), + (select sum(c) from test_mv_b), + (select sum(c) from test_mv_c); + + +select 'deduplicate_blocks_in_dependent_materialized_views=1, insert_deduplication_token = no, results inconsitent'; + +drop table if exists test sync; +drop table if exists test_mv_a sync; +drop table if exists test_mv_b sync; +drop table if exists test_mv_c sync; + +set deduplicate_blocks_in_dependent_materialized_views=1; + +CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree ('/clickhouse/test/tables/test','1') ORDER BY tuple() ; + +CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_a','1') +order by tuple() AS SELECT A, count() c FROM test group by A; + +CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_b','1') +partition by A order by tuple() AS SELECT A, count() c FROM test group by A; + +CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_c','1') +order by tuple() AS SELECT A, count() c FROM test group by A; + +SET max_partitions_per_insert_block = 1; +INSERT INTO test SELECT number%3, 1 FROM numbers(9) ; -- { serverError 252 } +SET max_partitions_per_insert_block = 0; +INSERT INTO test SELECT number%3, 1 FROM numbers(9); +INSERT INTO test SELECT number%3, 2 FROM numbers(9); + +select + (select count() from test), + (select sum(c) from test_mv_a), + (select sum(c) from test_mv_b), + (select sum(c) from test_mv_c); + + +select 'deduplicate_blocks_in_dependent_materialized_views=0, insert_deduplication_token = yes, results inconsitent'; + +drop table if exists test sync; +drop table if exists test_mv_a sync; +drop table if exists test_mv_b sync; +drop table if exists test_mv_c sync; + +set deduplicate_blocks_in_dependent_materialized_views=0; + +CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree ('/clickhouse/test/tables/test','1') ORDER BY tuple() ; + +CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_a','1') +order by tuple() AS SELECT A, count() c FROM test group by A; + +CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_b','1') +partition by A order by tuple() AS SELECT A, count() c FROM test group by A; + +CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_c','1') +order by tuple() AS SELECT A, count() c FROM test group by A; + +SET max_partitions_per_insert_block = 1; +INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplication_token = 'test1'; -- { serverError 252 } +SET max_partitions_per_insert_block = 0; +INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplication_token = 'test1'; +INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; + +select + (select count() from test), + (select sum(c) from test_mv_a), + (select sum(c) from test_mv_b), + (select sum(c) from test_mv_c); + + +select 'deduplicate_blocks_in_dependent_materialized_views=1, insert_deduplication_token = yes, results consitent'; + +drop table if exists test sync; +drop table if exists test_mv_a sync; +drop table if exists test_mv_b sync; +drop table if exists test_mv_c sync; + +set deduplicate_blocks_in_dependent_materialized_views=1; + +CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree ('/clickhouse/test/tables/test','1') ORDER BY tuple() ; + +CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_a','1') +order by tuple() AS SELECT A, count() c FROM test group by A; + +CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_b','1') +partition by A order by tuple() AS SELECT A, count() c FROM test group by A; + +CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_c','1') +order by tuple() AS SELECT A, count() c FROM test group by A; + + +SET max_partitions_per_insert_block = 1; +INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplication_token = 'test1' ; -- { serverError 252 } +SET max_partitions_per_insert_block = 0; +INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplication_token = 'test1'; +INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; + +select + (select count() from test), + (select sum(c) from test_mv_a), + (select sum(c) from test_mv_b), + (select sum(c) from test_mv_c); + +drop table if exists test sync; +drop table if exists test_mv_a sync; +drop table if exists test_mv_b sync; +drop table if exists test_mv_c sync; From 314168bd5b25b397de5755391cee1fc78163804c Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Wed, 16 Feb 2022 17:21:03 -0400 Subject: [PATCH 002/139] test that insert_deduplication_token allow to do idempotent insertion with materialized_views --- .../02124_insert_deduplication_token_materialized_views.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql index f5307fe0dee..ebde1e1959b 100644 --- a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql +++ b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql @@ -23,6 +23,7 @@ INSERT INTO test SELECT number%3, 1 FROM numbers(9); -- { serverError 252 } SET max_partitions_per_insert_block = 0; INSERT INTO test SELECT number%3, 1 FROM numbers(9); INSERT INTO test SELECT number%3, 2 FROM numbers(9); +INSERT INTO test SELECT number%3, 2 FROM numbers(9); select (select count() from test), @@ -56,6 +57,7 @@ INSERT INTO test SELECT number%3, 1 FROM numbers(9) ; -- { serverError 252 } SET max_partitions_per_insert_block = 0; INSERT INTO test SELECT number%3, 1 FROM numbers(9); INSERT INTO test SELECT number%3, 2 FROM numbers(9); +INSERT INTO test SELECT number%3, 2 FROM numbers(9); select (select count() from test), @@ -89,6 +91,7 @@ INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplicatio SET max_partitions_per_insert_block = 0; INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplication_token = 'test1'; INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; +INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; select (select count() from test), @@ -96,7 +99,6 @@ select (select sum(c) from test_mv_b), (select sum(c) from test_mv_c); - select 'deduplicate_blocks_in_dependent_materialized_views=1, insert_deduplication_token = yes, results consitent'; drop table if exists test sync; @@ -123,6 +125,7 @@ INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplicatio SET max_partitions_per_insert_block = 0; INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplication_token = 'test1'; INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; +INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; select (select count() from test), From fd380a6e7f04573dbd0f7da184b5f830d7d01f8e Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Wed, 16 Feb 2022 17:25:13 -0400 Subject: [PATCH 003/139] test that insert_deduplication_token allow to do idempotent insertion with materialized_views --- ...deduplication_token_materialized_views.sql | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql index ebde1e1959b..4724245a67a 100644 --- a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql +++ b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql @@ -7,15 +7,15 @@ drop table if exists test_mv_c sync; set deduplicate_blocks_in_dependent_materialized_views=0; -CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree ('/clickhouse/test/tables/test','1') ORDER BY tuple() ; +CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree() ORDER BY tuple() ; -CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_a','1') +CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree() order by tuple() AS SELECT A, count() c FROM test group by A; -CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_b','1') +CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree() partition by A order by tuple() AS SELECT A, count() c FROM test group by A; -CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_c','1') +CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree() order by tuple() AS SELECT A, count() c FROM test group by A; SET max_partitions_per_insert_block = 1; @@ -41,15 +41,15 @@ drop table if exists test_mv_c sync; set deduplicate_blocks_in_dependent_materialized_views=1; -CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree ('/clickhouse/test/tables/test','1') ORDER BY tuple() ; +CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree() ORDER BY tuple() ; -CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_a','1') +CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree() order by tuple() AS SELECT A, count() c FROM test group by A; -CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_b','1') +CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree() partition by A order by tuple() AS SELECT A, count() c FROM test group by A; -CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_c','1') +CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree() order by tuple() AS SELECT A, count() c FROM test group by A; SET max_partitions_per_insert_block = 1; @@ -75,15 +75,15 @@ drop table if exists test_mv_c sync; set deduplicate_blocks_in_dependent_materialized_views=0; -CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree ('/clickhouse/test/tables/test','1') ORDER BY tuple() ; +CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree() ORDER BY tuple() ; -CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_a','1') +CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree() order by tuple() AS SELECT A, count() c FROM test group by A; -CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_b','1') +CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree() partition by A order by tuple() AS SELECT A, count() c FROM test group by A; -CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_c','1') +CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree() order by tuple() AS SELECT A, count() c FROM test group by A; SET max_partitions_per_insert_block = 1; @@ -93,7 +93,7 @@ INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplicatio INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; -select +select (select count() from test), (select sum(c) from test_mv_a), (select sum(c) from test_mv_b), @@ -108,15 +108,15 @@ drop table if exists test_mv_c sync; set deduplicate_blocks_in_dependent_materialized_views=1; -CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree ('/clickhouse/test/tables/test','1') ORDER BY tuple() ; +CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree() ORDER BY tuple() ; -CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_a','1') +CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree() order by tuple() AS SELECT A, count() c FROM test group by A; -CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_b','1') +CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree() partition by A order by tuple() AS SELECT A, count() c FROM test group by A; -CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_c','1') +CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree() order by tuple() AS SELECT A, count() c FROM test group by A; @@ -127,7 +127,7 @@ INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplicatio INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; -select +select (select count() from test), (select sum(c) from test_mv_a), (select sum(c) from test_mv_b), From e6b7624976aff39f1fa63330329a4b837b77da17 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Wed, 16 Feb 2022 17:38:50 -0400 Subject: [PATCH 004/139] test that insert_deduplication_token allow to do idempotent insertion with materialized_views --- ...deduplication_token_materialized_views.sql | 81 +++++-------------- 1 file changed, 22 insertions(+), 59 deletions(-) diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql index 4724245a67a..bbcf19463d9 100644 --- a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql +++ b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql @@ -7,16 +7,13 @@ drop table if exists test_mv_c sync; set deduplicate_blocks_in_dependent_materialized_views=0; -CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree() ORDER BY tuple() ; +CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree ('/clickhouse/test/tables/test','1') ORDER BY tuple() ; -CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree() -order by tuple() AS SELECT A, count() c FROM test group by A; +CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_a','1') order by tuple() AS SELECT A, count() c FROM test group by A; -CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree() -partition by A order by tuple() AS SELECT A, count() c FROM test group by A; +CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_b','1') partition by A order by tuple() AS SELECT A, count() c FROM test group by A; -CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree() -order by tuple() AS SELECT A, count() c FROM test group by A; +CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_c','1') order by tuple() AS SELECT A, count() c FROM test group by A; SET max_partitions_per_insert_block = 1; INSERT INTO test SELECT number%3, 1 FROM numbers(9); -- { serverError 252 } @@ -34,24 +31,13 @@ select select 'deduplicate_blocks_in_dependent_materialized_views=1, insert_deduplication_token = no, results inconsitent'; -drop table if exists test sync; -drop table if exists test_mv_a sync; -drop table if exists test_mv_b sync; -drop table if exists test_mv_c sync; +truncate test; +truncate test_mv_a; +truncate test_mv_b; +truncate test_mv_c; set deduplicate_blocks_in_dependent_materialized_views=1; -CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree() ORDER BY tuple() ; - -CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree() -order by tuple() AS SELECT A, count() c FROM test group by A; - -CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree() -partition by A order by tuple() AS SELECT A, count() c FROM test group by A; - -CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree() -order by tuple() AS SELECT A, count() c FROM test group by A; - SET max_partitions_per_insert_block = 1; INSERT INTO test SELECT number%3, 1 FROM numbers(9) ; -- { serverError 252 } SET max_partitions_per_insert_block = 0; @@ -68,24 +54,13 @@ select select 'deduplicate_blocks_in_dependent_materialized_views=0, insert_deduplication_token = yes, results inconsitent'; -drop table if exists test sync; -drop table if exists test_mv_a sync; -drop table if exists test_mv_b sync; -drop table if exists test_mv_c sync; +truncate test; +truncate test_mv_a; +truncate test_mv_b; +truncate test_mv_c; set deduplicate_blocks_in_dependent_materialized_views=0; -CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree() ORDER BY tuple() ; - -CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree() -order by tuple() AS SELECT A, count() c FROM test group by A; - -CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree() -partition by A order by tuple() AS SELECT A, count() c FROM test group by A; - -CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree() -order by tuple() AS SELECT A, count() c FROM test group by A; - SET max_partitions_per_insert_block = 1; INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplication_token = 'test1'; -- { serverError 252 } SET max_partitions_per_insert_block = 0; @@ -93,7 +68,7 @@ INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplicatio INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; -select +select (select count() from test), (select sum(c) from test_mv_a), (select sum(c) from test_mv_b), @@ -101,25 +76,13 @@ select select 'deduplicate_blocks_in_dependent_materialized_views=1, insert_deduplication_token = yes, results consitent'; -drop table if exists test sync; -drop table if exists test_mv_a sync; -drop table if exists test_mv_b sync; -drop table if exists test_mv_c sync; +truncate test; +truncate test_mv_a; +truncate test_mv_b; +truncate test_mv_c; set deduplicate_blocks_in_dependent_materialized_views=1; -CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree() ORDER BY tuple() ; - -CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree() -order by tuple() AS SELECT A, count() c FROM test group by A; - -CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree() -partition by A order by tuple() AS SELECT A, count() c FROM test group by A; - -CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree() -order by tuple() AS SELECT A, count() c FROM test group by A; - - SET max_partitions_per_insert_block = 1; INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplication_token = 'test1' ; -- { serverError 252 } SET max_partitions_per_insert_block = 0; @@ -127,13 +90,13 @@ INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplicatio INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; -select +select (select count() from test), (select sum(c) from test_mv_a), (select sum(c) from test_mv_b), (select sum(c) from test_mv_c); -drop table if exists test sync; -drop table if exists test_mv_a sync; -drop table if exists test_mv_b sync; -drop table if exists test_mv_c sync; +drop table test sync; +drop table test_mv_a sync; +drop table test_mv_b sync; +drop table test_mv_c sync; From 60473bf0d1890ed041578fe7efebf201ae3e06f2 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Thu, 17 Feb 2022 00:03:34 -0400 Subject: [PATCH 005/139] Update 02124_insert_deduplication_token_materialized_views.sql --- ...2124_insert_deduplication_token_materialized_views.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql index bbcf19463d9..cc03cc3084a 100644 --- a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql +++ b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql @@ -7,13 +7,13 @@ drop table if exists test_mv_c sync; set deduplicate_blocks_in_dependent_materialized_views=0; -CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree ('/clickhouse/test/tables/test','1') ORDER BY tuple() ; +CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree () ORDER BY tuple() ; -CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_a','1') order by tuple() AS SELECT A, count() c FROM test group by A; +CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree () order by tuple() AS SELECT A, count() c FROM test group by A; -CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_b','1') partition by A order by tuple() AS SELECT A, count() c FROM test group by A; +CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree () partition by A order by tuple() AS SELECT A, count() c FROM test group by A; -CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree ('/clickhouse/test/tables/test_mv_c','1') order by tuple() AS SELECT A, count() c FROM test group by A; +CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree () order by tuple() AS SELECT A, count() c FROM test group by A; SET max_partitions_per_insert_block = 1; INSERT INTO test SELECT number%3, 1 FROM numbers(9); -- { serverError 252 } From f22ad04ca1f49c09cf095e3e310d2971ca6a258c Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Thu, 17 Feb 2022 09:31:32 -0400 Subject: [PATCH 006/139] fix zk path --- ...rt_deduplication_token_materialized_views.sql | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql index cc03cc3084a..69a8614afad 100644 --- a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql +++ b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql @@ -7,13 +7,17 @@ drop table if exists test_mv_c sync; set deduplicate_blocks_in_dependent_materialized_views=0; -CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree () ORDER BY tuple() ; +CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree ('/clickhouse/tables/{database}/test_02124/{table}', '1') +ORDER BY tuple() ; -CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree () order by tuple() AS SELECT A, count() c FROM test group by A; +CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree ('/clickhouse/tables/{database}/test_02124/{table}', '1') +order by tuple() AS SELECT A, count() c FROM test group by A; -CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree () partition by A order by tuple() AS SELECT A, count() c FROM test group by A; +CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree ('/clickhouse/tables/{database}/test_02124/{table}', '1') +partition by A order by tuple() AS SELECT A, count() c FROM test group by A; -CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree () order by tuple() AS SELECT A, count() c FROM test group by A; +CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree ('/clickhouse/tables/{database}/test_02124/{table}', '1') +order by tuple() AS SELECT A, count() c FROM test group by A; SET max_partitions_per_insert_block = 1; INSERT INTO test SELECT number%3, 1 FROM numbers(9); -- { serverError 252 } @@ -68,7 +72,7 @@ INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplicatio INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; -select +select (select count() from test), (select sum(c) from test_mv_a), (select sum(c) from test_mv_b), @@ -90,7 +94,7 @@ INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplicatio INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; -select +select (select count() from test), (select sum(c) from test_mv_a), (select sum(c) from test_mv_b), From 22a91d190e836d9e3d0e112eb856b079eb9457e5 Mon Sep 17 00:00:00 2001 From: alexander goryanets Date: Mon, 7 Feb 2022 10:44:05 +0000 Subject: [PATCH 007/139] introduce interserver_listen_host param --- programs/server/config.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/programs/server/config.xml b/programs/server/config.xml index d34340ac995..9e741e50605 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -188,6 +188,10 @@ 127.0.0.1 --> + + + + From ee29e93fc0d08f6daf97fa411230d61d0ccbe37c Mon Sep 17 00:00:00 2001 From: alexander goryanets Date: Mon, 7 Feb 2022 10:58:09 +0000 Subject: [PATCH 008/139] use interserver_listen_host param for configure interserver listen_host --- programs/server/Server.cpp | 109 +++++++++++++++++++++---------------- programs/server/Server.h | 1 + 2 files changed, 63 insertions(+), 47 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index ad835af1159..7e5c22e0975 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -369,6 +369,15 @@ std::vector getListenHosts(const Poco::Util::AbstractConfiguration return listen_hosts; } +std::vector getInterserverListenHosts(const Poco::Util::AbstractConfiguration & config, std::vector listen_hosts) +{ + auto interserver_listen_hosts = DB::getMultipleValuesFromConfig(config, "", "interserver_listen_host"); + if (interserver_listen_hosts.empty()) + return listen_hosts; + else + return interserver_listen_hosts; +} + bool getListenTry(const Poco::Util::AbstractConfiguration & config) { bool listen_try = config.getBool("listen_try", false); @@ -978,6 +987,7 @@ if (ThreadFuzzer::instance().isEffective()) /* already_loaded = */ false); /// Reload it right now (initial loading) const auto listen_hosts = getListenHosts(config()); + const auto interserver_listen_hosts = getInterserverListenHosts(config(), listen_hosts); const auto listen_try = getListenTry(config()); if (config().has("keeper_server")) @@ -1369,7 +1379,7 @@ if (ThreadFuzzer::instance().isEffective()) { std::lock_guard lock(servers_lock); - createServers(config(), listen_hosts, listen_try, server_pool, async_metrics, servers); + createServers(config(), listen_hosts, interserver_listen_hosts, listen_try, server_pool, async_metrics, servers); if (servers.empty()) throw Exception( "No servers started (add valid listen_host and 'tcp_port' or 'http_port' to configuration file.)", @@ -1549,6 +1559,7 @@ if (ThreadFuzzer::instance().isEffective()) void Server::createServers( Poco::Util::AbstractConfiguration & config, const std::vector & listen_hosts, + const std::vector & interserver_listen_hosts, bool listen_try, Poco::ThreadPool & server_pool, AsynchronousMetrics & async_metrics, @@ -1666,51 +1677,6 @@ void Server::createServers( #endif }); - /// Interserver IO HTTP - port_name = "interserver_http_port"; - createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter - { - Poco::Net::ServerSocket socket; - auto address = socketBindListen(socket, listen_host, port); - socket.setReceiveTimeout(settings.http_receive_timeout); - socket.setSendTimeout(settings.http_send_timeout); - return ProtocolServerAdapter( - listen_host, - port_name, - "replica communication (interserver): http://" + address.toString(), - std::make_unique( - context(), - createHandlerFactory(*this, async_metrics, "InterserverIOHTTPHandler-factory"), - server_pool, - socket, - http_params)); - }); - - port_name = "interserver_https_port"; - createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter - { -#if USE_SSL - Poco::Net::SecureServerSocket socket; - auto address = socketBindListen(socket, listen_host, port, /* secure = */ true); - socket.setReceiveTimeout(settings.http_receive_timeout); - socket.setSendTimeout(settings.http_send_timeout); - return ProtocolServerAdapter( - listen_host, - port_name, - "secure replica communication (interserver): https://" + address.toString(), - std::make_unique( - context(), - createHandlerFactory(*this, async_metrics, "InterserverIOHTTPSHandler-factory"), - server_pool, - socket, - http_params)); -#else - UNUSED(port); - throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.", - ErrorCodes::SUPPORT_IS_DISABLED}; -#endif - }); - port_name = "mysql_port"; createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter { @@ -1769,6 +1735,54 @@ void Server::createServers( }); } + for (const auto & interserver_listen_host : interserver_listen_hosts) + { + /// Interserver IO HTTP + const char * port_name = "interserver_http_port"; + createServer(config, interserver_listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter + { + Poco::Net::ServerSocket socket; + auto address = socketBindListen(socket, interserver_listen_host, port); + socket.setReceiveTimeout(settings.http_receive_timeout); + socket.setSendTimeout(settings.http_send_timeout); + return ProtocolServerAdapter( + interserver_listen_host, + port_name, + "replica communication (interserver): http://" + address.toString(), + std::make_unique( + context(), + createHandlerFactory(*this, async_metrics, "InterserverIOHTTPHandler-factory"), + server_pool, + socket, + http_params)); + }); + + port_name = "interserver_https_port"; + createServer(config, interserver_listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter + { +#if USE_SSL + Poco::Net::SecureServerSocket socket; + auto address = socketBindListen(socket, interserver_listen_host, port, /* secure = */ true); + socket.setReceiveTimeout(settings.http_receive_timeout); + socket.setSendTimeout(settings.http_send_timeout); + return ProtocolServerAdapter( + interserver_listen_host, + port_name, + "secure replica communication (interserver): https://" + address.toString(), + std::make_unique( + context(), + createHandlerFactory(*this, async_metrics, "InterserverIOHTTPSHandler-factory"), + server_pool, + socket, + http_params)); +#else + UNUSED(port); + throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.", + ErrorCodes::SUPPORT_IS_DISABLED}; +#endif + }); + } + } void Server::updateServers( @@ -1780,6 +1794,7 @@ void Server::updateServers( Poco::Logger * log = &logger(); /// Gracefully shutdown servers when their port is removed from config const auto listen_hosts = getListenHosts(config); + const auto interserver_listen_hosts = getInterserverListenHosts(config, listen_hosts); const auto listen_try = getListenTry(config); for (auto & server : servers) @@ -1794,7 +1809,7 @@ void Server::updateServers( } } - createServers(config, listen_hosts, listen_try, server_pool, async_metrics, servers, /* start_servers: */ true); + createServers(config, listen_hosts, interserver_listen_hosts, listen_try, server_pool, async_metrics, servers, /* start_servers: */ true); /// Remove servers once all their connections are closed while (std::any_of(servers.begin(), servers.end(), [](const auto & server) { return server.isStopping(); })) diff --git a/programs/server/Server.h b/programs/server/Server.h index b4f2ea3bb79..09fcfba742a 100644 --- a/programs/server/Server.h +++ b/programs/server/Server.h @@ -82,6 +82,7 @@ private: void createServers( Poco::Util::AbstractConfiguration & config, const std::vector & listen_hosts, + const std::vector & interserver_listen_hosts, bool listen_try, Poco::ThreadPool & server_pool, AsynchronousMetrics & async_metrics, From f43fbfadb0fe06e12fcfb95c7b7609c44f8cb0c5 Mon Sep 17 00:00:00 2001 From: alexander goryanets Date: Mon, 7 Feb 2022 11:10:42 +0000 Subject: [PATCH 009/139] add docs for interserver_listen_host setting --- .../server-configuration-parameters/settings.md | 12 ++++++++++++ .../server-configuration-parameters/settings.md | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 266abadb087..5652a294e78 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -389,6 +389,18 @@ For more information, see the section “[Configuration files](../../operations/ /etc/metrica.xml ``` +## interserver_listen_host {#interserver-listen-host} + +Restriction on hosts that can exchange data between ClickHouse servers. +The default value equals to `listen_host` setting. + +Examples: + +``` xml +::ffff:a00:1 +10.0.0.1 +``` + ## interserver_http_port {#interserver-http-port} Port for exchanging data between ClickHouse servers. diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index 5d667ef8238..136a28d7e06 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -390,6 +390,18 @@ ClickHouse проверяет условия для `min_part_size` и `min_part /etc/metrica.xml ``` +## interserver_listen_host {#interserver-listen-host} + +Ограничение по хостам, для обмена между серверами ClickHouse. +Значение по умолчанию совпадает со значением параметра listen_host + +Примеры: + +``` xml +::ffff:a00:1 +10.0.0.1 +``` + ## interserver_http_port {#interserver-http-port} Порт для обмена между серверами ClickHouse. From d9cfba9047bb93dda235ebe73f1f9df94d9943c7 Mon Sep 17 00:00:00 2001 From: alexander goryanets Date: Wed, 9 Feb 2022 15:28:36 +0000 Subject: [PATCH 010/139] add integration test for interserver_listen_host --- .../__init__.py | 0 .../config.d/interserver-listen-host.xml | 3 ++ .../config.d/no-interserver-listen-host.xml | 3 ++ .../test_case.py | 52 +++++++++++++++++++ 4 files changed, 58 insertions(+) create mode 100644 tests/integration/test_tcp_handler_interserver_listen_host/__init__.py create mode 100644 tests/integration/test_tcp_handler_interserver_listen_host/configs/config.d/interserver-listen-host.xml create mode 100644 tests/integration/test_tcp_handler_interserver_listen_host/configs/config.d/no-interserver-listen-host.xml create mode 100644 tests/integration/test_tcp_handler_interserver_listen_host/test_case.py diff --git a/tests/integration/test_tcp_handler_interserver_listen_host/__init__.py b/tests/integration/test_tcp_handler_interserver_listen_host/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_tcp_handler_interserver_listen_host/configs/config.d/interserver-listen-host.xml b/tests/integration/test_tcp_handler_interserver_listen_host/configs/config.d/interserver-listen-host.xml new file mode 100644 index 00000000000..52d45e7c956 --- /dev/null +++ b/tests/integration/test_tcp_handler_interserver_listen_host/configs/config.d/interserver-listen-host.xml @@ -0,0 +1,3 @@ + + 10.0.0.10 + diff --git a/tests/integration/test_tcp_handler_interserver_listen_host/configs/config.d/no-interserver-listen-host.xml b/tests/integration/test_tcp_handler_interserver_listen_host/configs/config.d/no-interserver-listen-host.xml new file mode 100644 index 00000000000..2e7e1a83aa1 --- /dev/null +++ b/tests/integration/test_tcp_handler_interserver_listen_host/configs/config.d/no-interserver-listen-host.xml @@ -0,0 +1,3 @@ + + + diff --git a/tests/integration/test_tcp_handler_interserver_listen_host/test_case.py b/tests/integration/test_tcp_handler_interserver_listen_host/test_case.py new file mode 100644 index 00000000000..7d47a353631 --- /dev/null +++ b/tests/integration/test_tcp_handler_interserver_listen_host/test_case.py @@ -0,0 +1,52 @@ +"""Test Interserver responses on configured IP.""" +from pathlib import Path +import pytest +from helpers.cluster import ClickHouseCluster +import requests +import socket +import time + +cluster = ClickHouseCluster(__file__) + +INTERSERVER_LISTEN_HOST = '10.0.0.10' +INTERSERVER_HTTP_PORT = 9009 + +node_with_interserver_listen_host = cluster.add_instance( + 'node_with_interserver_listen_host', + main_configs=["configs/config.d/interserver-listen-host.xml"], + ipv4_address=INTERSERVER_LISTEN_HOST, # used to configure acc. interface in test container + ipv6_address='2001:3984:3989::1:1000', +) + +node_without_interserver_listen_host = cluster.add_instance( + 'node_without_interserver_listen_host', + main_configs=["configs/config.d/no-interserver-listen-host.xml"], + ipv6_address='2001:3984:3989::2:1000' +) + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + +def test_request_to_node_with_interserver_listen_host(start_cluster): + time.sleep(5) # waiting for interserver listener to start + response_interserver = requests.get( + f'http://{INTERSERVER_LISTEN_HOST}:{INTERSERVER_HTTP_PORT}' + ) + response_client = requests.get( + f'http://{node_without_interserver_listen_host.ip_address}:8123' + ) + assert response_interserver.status_code == 200 + assert "Ok." in response_interserver.text + assert response_client.status_code == 200 + +def test_request_to_node_without_interserver_listen_host(start_cluster): + response = requests.get( + f'http://{node_without_interserver_listen_host.ip_address}:{INTERSERVER_HTTP_PORT}' + ) + assert response.status_code == 200 From d7071e8596f1beefb3f36cf966fa4a8ca2a643ef Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 5 Apr 2022 17:04:53 +0200 Subject: [PATCH 011/139] Style --- programs/server/Server.cpp | 20 ++++++++-------- programs/server/Server.h | 4 ++-- .../test_case.py | 23 +++++++++++-------- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 7e5c22e0975..7b81220a4c1 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -358,7 +358,7 @@ Poco::Net::SocketAddress Server::socketBindListen(Poco::Net::ServerSocket & sock return address; } -std::vector getListenHosts(const Poco::Util::AbstractConfiguration & config) +Strings getListenHosts(const Poco::Util::AbstractConfiguration & config) { auto listen_hosts = DB::getMultipleValuesFromConfig(config, "", "listen_host"); if (listen_hosts.empty()) @@ -369,13 +369,14 @@ std::vector getListenHosts(const Poco::Util::AbstractConfiguration return listen_hosts; } -std::vector getInterserverListenHosts(const Poco::Util::AbstractConfiguration & config, std::vector listen_hosts) +Strings getInterserverListenHosts(const Poco::Util::AbstractConfiguration & config) { auto interserver_listen_hosts = DB::getMultipleValuesFromConfig(config, "", "interserver_listen_host"); - if (interserver_listen_hosts.empty()) - return listen_hosts; - else + if (!interserver_listen_hosts.empty()) return interserver_listen_hosts; + + /// Use more general restriction in case of emptiness + return getListenHosts(config); } bool getListenTry(const Poco::Util::AbstractConfiguration & config) @@ -987,7 +988,7 @@ if (ThreadFuzzer::instance().isEffective()) /* already_loaded = */ false); /// Reload it right now (initial loading) const auto listen_hosts = getListenHosts(config()); - const auto interserver_listen_hosts = getInterserverListenHosts(config(), listen_hosts); + const auto interserver_listen_hosts = getInterserverListenHosts(config()); const auto listen_try = getListenTry(config()); if (config().has("keeper_server")) @@ -1558,8 +1559,8 @@ if (ThreadFuzzer::instance().isEffective()) void Server::createServers( Poco::Util::AbstractConfiguration & config, - const std::vector & listen_hosts, - const std::vector & interserver_listen_hosts, + const Strings & listen_hosts, + const Strings & interserver_listen_hosts, bool listen_try, Poco::ThreadPool & server_pool, AsynchronousMetrics & async_metrics, @@ -1735,6 +1736,7 @@ void Server::createServers( }); } + /// Now iterate over interserver_listen_hosts for (const auto & interserver_listen_host : interserver_listen_hosts) { /// Interserver IO HTTP @@ -1794,7 +1796,7 @@ void Server::updateServers( Poco::Logger * log = &logger(); /// Gracefully shutdown servers when their port is removed from config const auto listen_hosts = getListenHosts(config); - const auto interserver_listen_hosts = getInterserverListenHosts(config, listen_hosts); + const auto interserver_listen_hosts = getInterserverListenHosts(config); const auto listen_try = getListenTry(config); for (auto & server : servers) diff --git a/programs/server/Server.h b/programs/server/Server.h index 09fcfba742a..b973f22890f 100644 --- a/programs/server/Server.h +++ b/programs/server/Server.h @@ -81,8 +81,8 @@ private: void createServers( Poco::Util::AbstractConfiguration & config, - const std::vector & listen_hosts, - const std::vector & interserver_listen_hosts, + const Strings & listen_hosts, + const Strings & interserver_listen_hosts, bool listen_try, Poco::ThreadPool & server_pool, AsynchronousMetrics & async_metrics, diff --git a/tests/integration/test_tcp_handler_interserver_listen_host/test_case.py b/tests/integration/test_tcp_handler_interserver_listen_host/test_case.py index 7d47a353631..44df1c369cf 100644 --- a/tests/integration/test_tcp_handler_interserver_listen_host/test_case.py +++ b/tests/integration/test_tcp_handler_interserver_listen_host/test_case.py @@ -8,22 +8,23 @@ import time cluster = ClickHouseCluster(__file__) -INTERSERVER_LISTEN_HOST = '10.0.0.10' +INTERSERVER_LISTEN_HOST = "10.0.0.10" INTERSERVER_HTTP_PORT = 9009 node_with_interserver_listen_host = cluster.add_instance( - 'node_with_interserver_listen_host', + "node_with_interserver_listen_host", main_configs=["configs/config.d/interserver-listen-host.xml"], - ipv4_address=INTERSERVER_LISTEN_HOST, # used to configure acc. interface in test container - ipv6_address='2001:3984:3989::1:1000', + ipv4_address=INTERSERVER_LISTEN_HOST, # used to configure acc. interface in test container + ipv6_address="2001:3984:3989::1:1000", ) node_without_interserver_listen_host = cluster.add_instance( - 'node_without_interserver_listen_host', + "node_without_interserver_listen_host", main_configs=["configs/config.d/no-interserver-listen-host.xml"], - ipv6_address='2001:3984:3989::2:1000' + ipv6_address="2001:3984:3989::2:1000", ) + @pytest.fixture(scope="module") def start_cluster(): try: @@ -33,20 +34,22 @@ def start_cluster(): finally: cluster.shutdown() + def test_request_to_node_with_interserver_listen_host(start_cluster): - time.sleep(5) # waiting for interserver listener to start + time.sleep(5) # waiting for interserver listener to start response_interserver = requests.get( - f'http://{INTERSERVER_LISTEN_HOST}:{INTERSERVER_HTTP_PORT}' + f"http://{INTERSERVER_LISTEN_HOST}:{INTERSERVER_HTTP_PORT}" ) response_client = requests.get( - f'http://{node_without_interserver_listen_host.ip_address}:8123' + f"http://{node_without_interserver_listen_host.ip_address}:8123" ) assert response_interserver.status_code == 200 assert "Ok." in response_interserver.text assert response_client.status_code == 200 + def test_request_to_node_without_interserver_listen_host(start_cluster): response = requests.get( - f'http://{node_without_interserver_listen_host.ip_address}:{INTERSERVER_HTTP_PORT}' + f"http://{node_without_interserver_listen_host.ip_address}:{INTERSERVER_HTTP_PORT}" ) assert response.status_code == 200 From b91ae916ba051a738b4dc72a9a54221cc85ea3df Mon Sep 17 00:00:00 2001 From: renwujie Date: Wed, 25 May 2022 11:49:37 +0800 Subject: [PATCH 012/139] =?UTF-8?q?partition=20by=20=E8=AF=AD=E5=8F=A5?= =?UTF-8?q?=E8=A7=A3=E9=87=8A=E8=A1=A5=E5=85=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 英文官网不建议手动指定 partition by 字段对表进行分区，中文官网提都没提。 --- docs/zh/engines/table-engines/mergetree-family/mergetree.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/zh/engines/table-engines/mergetree-family/mergetree.md b/docs/zh/engines/table-engines/mergetree-family/mergetree.md index 0ace414b4b8..7b723d38594 100644 --- a/docs/zh/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/zh/engines/table-engines/mergetree-family/mergetree.md @@ -62,6 +62,8 @@ ORDER BY expr - `PARTITION BY` — [分区键](custom-partitioning-key.md) ，可选项。 + 大多数情况下，不需要分使用区键。即使需要使用，也不需要使用比月更细粒度的分区键。分区不会加快查询（这与 ORDER BY 表达式不同）。永远也别使用过细粒度的分区键。不要使用客户端指定分区标识符或分区字段名称来对数据进行分区（而是将分区字段标识或名称作为 ORDER BY 表达式的第一列来指定分区）。 + 要按月分区，可以使用表达式 `toYYYYMM(date_column)` ，这里的 `date_column` 是一个 [Date](../../../engines/table-engines/mergetree-family/mergetree.md) 类型的列。分区名的格式会是 `"YYYYMM"` 。 - `PRIMARY KEY` - 如果要 [选择与排序键不同的主键](#choosing-a-primary-key-that-differs-from-the-sorting-key)，在这里指定，可选项。 From df0ca7fa6f444571027f851fc5f948e49188d825 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Tue, 31 May 2022 16:16:50 -0300 Subject: [PATCH 013/139] try without truncate --- ...deduplication_token_materialized_views.sql | 89 ++++++++----------- 1 file changed, 37 insertions(+), 52 deletions(-) diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql index 69a8614afad..5ab40e3769a 100644 --- a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql +++ b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql @@ -7,98 +7,83 @@ drop table if exists test_mv_c sync; set deduplicate_blocks_in_dependent_materialized_views=0; -CREATE TABLE test (A Int64, B Int64) ENGINE = ReplicatedMergeTree ('/clickhouse/tables/{database}/test_02124/{table}', '1') -ORDER BY tuple() ; +CREATE TABLE test (test String, A Int64, B Int64) ENGINE = ReplicatedMergeTree ('/clickhouse/tables/{database}/test_02124/{table}', '1') +ORDER BY tuple(); CREATE MATERIALIZED VIEW test_mv_a Engine=ReplicatedMergeTree ('/clickhouse/tables/{database}/test_02124/{table}', '1') -order by tuple() AS SELECT A, count() c FROM test group by A; +order by tuple() AS SELECT test, A, count() c FROM test group by test, A; CREATE MATERIALIZED VIEW test_mv_b Engine=ReplicatedMergeTree ('/clickhouse/tables/{database}/test_02124/{table}', '1') -partition by A order by tuple() AS SELECT A, count() c FROM test group by A; +partition by A order by tuple() AS SELECT test, A, count() c FROM test group by test, A; CREATE MATERIALIZED VIEW test_mv_c Engine=ReplicatedMergeTree ('/clickhouse/tables/{database}/test_02124/{table}', '1') -order by tuple() AS SELECT A, count() c FROM test group by A; +order by tuple() AS SELECT test, A, count() c FROM test group by test, A; SET max_partitions_per_insert_block = 1; -INSERT INTO test SELECT number%3, 1 FROM numbers(9); -- { serverError 252 } +INSERT INTO test SELECT 'case1', number%3, 1 FROM numbers(9); -- { serverError 252 } SET max_partitions_per_insert_block = 0; -INSERT INTO test SELECT number%3, 1 FROM numbers(9); -INSERT INTO test SELECT number%3, 2 FROM numbers(9); -INSERT INTO test SELECT number%3, 2 FROM numbers(9); +INSERT INTO test SELECT 'case1', number%3, 1 FROM numbers(9); +INSERT INTO test SELECT 'case1', number%3, 2 FROM numbers(9); +INSERT INTO test SELECT 'case1', number%3, 2 FROM numbers(9); select - (select count() from test), - (select sum(c) from test_mv_a), - (select sum(c) from test_mv_b), - (select sum(c) from test_mv_c); + (select count() from test where test='case1'), + (select sum(c) from test_mv_a where test='case1'), + (select sum(c) from test_mv_b where test='case1'), + (select sum(c) from test_mv_c where test='case1'); select 'deduplicate_blocks_in_dependent_materialized_views=1, insert_deduplication_token = no, results inconsitent'; -truncate test; -truncate test_mv_a; -truncate test_mv_b; -truncate test_mv_c; - set deduplicate_blocks_in_dependent_materialized_views=1; SET max_partitions_per_insert_block = 1; -INSERT INTO test SELECT number%3, 1 FROM numbers(9) ; -- { serverError 252 } +INSERT INTO test SELECT 'case2', number%3, 1 FROM numbers(9) ; -- { serverError 252 } SET max_partitions_per_insert_block = 0; -INSERT INTO test SELECT number%3, 1 FROM numbers(9); -INSERT INTO test SELECT number%3, 2 FROM numbers(9); -INSERT INTO test SELECT number%3, 2 FROM numbers(9); +INSERT INTO test SELECT 'case2', number%3, 1 FROM numbers(9); +INSERT INTO test SELECT 'case2', number%3, 2 FROM numbers(9); +INSERT INTO test SELECT 'case2', number%3, 2 FROM numbers(9); select - (select count() from test), - (select sum(c) from test_mv_a), - (select sum(c) from test_mv_b), - (select sum(c) from test_mv_c); + (select count() from test where test='case2'), + (select sum(c) from test_mv_a where test='case2'), + (select sum(c) from test_mv_b where test='case2'), + (select sum(c) from test_mv_c where test='case2'); select 'deduplicate_blocks_in_dependent_materialized_views=0, insert_deduplication_token = yes, results inconsitent'; -truncate test; -truncate test_mv_a; -truncate test_mv_b; -truncate test_mv_c; - set deduplicate_blocks_in_dependent_materialized_views=0; SET max_partitions_per_insert_block = 1; -INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplication_token = 'test1'; -- { serverError 252 } +INSERT INTO test SELECT 'case3', number%3, 1 FROM numbers(9) SETTINGS insert_deduplication_token = 'case3test1'; -- { serverError 252 } SET max_partitions_per_insert_block = 0; -INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplication_token = 'test1'; -INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; -INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; +INSERT INTO test SELECT 'case3', number%3, 1 FROM numbers(9) SETTINGS insert_deduplication_token = 'case3test1'; +INSERT INTO test SELECT 'case3', number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'case3test2'; +INSERT INTO test SELECT 'case3', number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'case3test2'; select - (select count() from test), - (select sum(c) from test_mv_a), - (select sum(c) from test_mv_b), - (select sum(c) from test_mv_c); + (select count() from test where test='case3'), + (select sum(c) from test_mv_a where test='case3'), + (select sum(c) from test_mv_b where test='case3'), + (select sum(c) from test_mv_c where test='case3'); select 'deduplicate_blocks_in_dependent_materialized_views=1, insert_deduplication_token = yes, results consitent'; -truncate test; -truncate test_mv_a; -truncate test_mv_b; -truncate test_mv_c; - set deduplicate_blocks_in_dependent_materialized_views=1; SET max_partitions_per_insert_block = 1; -INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplication_token = 'test1' ; -- { serverError 252 } +INSERT INTO test SELECT 'case4', number%3, 1 FROM numbers(9) SETTINGS insert_deduplication_token = 'case4test1' ; -- { serverError 252 } SET max_partitions_per_insert_block = 0; -INSERT INTO test SELECT number%3, 1 FROM numbers(9) SETTINGS insert_deduplication_token = 'test1'; -INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; -INSERT INTO test SELECT number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'test2'; +INSERT INTO test SELECT 'case4', number%3, 1 FROM numbers(9) SETTINGS insert_deduplication_token = 'case4test1'; +INSERT INTO test SELECT 'case4', number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'case4test2'; +INSERT INTO test SELECT 'case4', number%3, 2 FROM numbers(9) SETTINGS insert_deduplication_token = 'case4test2'; select - (select count() from test), - (select sum(c) from test_mv_a), - (select sum(c) from test_mv_b), - (select sum(c) from test_mv_c); + (select count() from test where test='case4'), + (select sum(c) from test_mv_a where test='case4'), + (select sum(c) from test_mv_b where test='case4'), + (select sum(c) from test_mv_c where test='case4'); drop table test sync; drop table test_mv_a sync; From 70a46beeefd555e6192b44769fe95551017e9ee6 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Fri, 3 Jun 2022 14:25:15 -0300 Subject: [PATCH 014/139] Update 02124_insert_deduplication_token_materialized_views.sql --- .../02124_insert_deduplication_token_materialized_views.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql index 5ab40e3769a..88d3165d060 100644 --- a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql +++ b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql @@ -1,3 +1,5 @@ +-- Tags: long + select 'deduplicate_blocks_in_dependent_materialized_views=0, insert_deduplication_token = no, results inconsitent'; drop table if exists test sync; From 5475f62363018e4fd8d09cfe9864f935907347d1 Mon Sep 17 00:00:00 2001 From: zvonand Date: Sun, 5 Jun 2022 13:06:48 +0300 Subject: [PATCH 015/139] 32 to 64 --- src/Functions/timeSlots.cpp | 82 +++++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 30 deletions(-) diff --git a/src/Functions/timeSlots.cpp b/src/Functions/timeSlots.cpp index a19ccf62565..e3f0e0fa7a8 100644 --- a/src/Functions/timeSlots.cpp +++ b/src/Functions/timeSlots.cpp @@ -23,9 +23,9 @@ namespace ErrorCodes namespace { -/** timeSlots(StartTime, Duration) +/** timeSlots(StartTime, Duration[, Size=1800]) * - for the time interval beginning at `StartTime` and continuing `Duration` seconds, - * returns an array of time points, consisting of rounding down to half an hour (default; or another value) of points from this interval. + * returns an array of time points, consisting of rounding down to Size (1800 seconds by default) of points from this interval. * For example, timeSlots(toDateTime('2012-01-01 12:20:00'), 600) = [toDateTime('2012-01-01 12:00:00'), toDateTime('2012-01-01 12:30:00')]. * This is necessary to search for hits that are part of the corresponding visit. * @@ -37,8 +37,8 @@ template struct TimeSlotsImpl { static void vectorVector( - const PaddedPODArray & starts, const PaddedPODArray & durations, UInt32 time_slot_size, - PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets) + const PaddedPODArray & starts, const PaddedPODArray & durations, UInt64 time_slot_size, + PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets) { size_t size = starts.size(); @@ -48,7 +48,7 @@ struct TimeSlotsImpl ColumnArray::Offset current_offset = 0; for (size_t i = 0; i < size; ++i) { - for (UInt32 value = starts[i] / time_slot_size, end = (starts[i] + durations[i]) / time_slot_size; value <= end; ++value) + for (Int64 value = starts[i] / time_slot_size, end = (starts[i] + durations[i]) / time_slot_size; value <= end; ++value) { result_values.push_back(value * time_slot_size); ++current_offset; @@ -59,8 +59,8 @@ struct TimeSlotsImpl } static void vectorConstant( - const PaddedPODArray & starts, DurationType duration, UInt32 time_slot_size, - PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets) + const PaddedPODArray & starts, DurationType duration, UInt64 time_slot_size, + PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets) { size_t size = starts.size(); @@ -70,7 +70,7 @@ struct TimeSlotsImpl ColumnArray::Offset current_offset = 0; for (size_t i = 0; i < size; ++i) { - for (UInt32 value = starts[i] / time_slot_size, end = (starts[i] + duration) / time_slot_size; value <= end; ++value) + for (Int64 value = starts[i] / time_slot_size, end = (starts[i] + duration) / time_slot_size; value <= end; ++value) { result_values.push_back(value * time_slot_size); ++current_offset; @@ -81,8 +81,8 @@ struct TimeSlotsImpl } static void constantVector( - UInt32 start, const PaddedPODArray & durations, UInt32 time_slot_size, - PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets) + Int64 start, const PaddedPODArray & durations, UInt64 time_slot_size, + PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets) { size_t size = durations.size(); @@ -92,7 +92,7 @@ struct TimeSlotsImpl ColumnArray::Offset current_offset = 0; for (size_t i = 0; i < size; ++i) { - for (UInt32 value = start / time_slot_size, end = (start + durations[i]) / time_slot_size; value <= end; ++value) + for (Int64 value = start / time_slot_size, end = (start + durations[i]) / time_slot_size; value <= end; ++value) { result_values.push_back(value * time_slot_size); ++current_offset; @@ -108,7 +108,7 @@ class FunctionTimeSlots : public IFunction { public: static constexpr auto name = "timeSlots"; - static constexpr UInt32 TIME_SLOT_SIZE = 1800; + static constexpr UInt64 TIME_SLOT_SIZE = 1800; static FunctionPtr create(ContextPtr) { return std::make_shared(); } String getName() const override @@ -129,33 +129,44 @@ public: + toString(arguments.size()) + ", should be 2 or 3", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - if (!WhichDataType(arguments[0].type).isDateTime()) - throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName() + ". Must be DateTime.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!WhichDataType(arguments[0].type).isDateTime() && !WhichDataType(arguments[0].type).isDateTime64()) + throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName() + + ". Must be DateTime or DateTime64.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (!WhichDataType(arguments[1].type).isUInt32()) - throw Exception("Illegal type " + arguments[1].type->getName() + " of second argument of function " + getName() + ". Must be UInt32.", + if (!WhichDataType(arguments[1].type).isNativeUInt()) + throw Exception("Illegal type " + arguments[1].type->getName() + " of second argument of function " + getName() + ". Must be UInt64.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (arguments.size() == 3 && !WhichDataType(arguments[2].type).isNativeUInt()) - throw Exception("Illegal type " + arguments[2].type->getName() + " of third argument of function " + getName() + ". Must be UInt32.", + throw Exception("Illegal type " + arguments[2].type->getName() + " of third argument of function " + getName() + ". Must be UInt64.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); /// If time zone is specified for source data type, attach it to the resulting type. /// Note that there is no explicit time zone argument for this function (we specify 2 as an argument number with explicit time zone). - return std::make_shared(std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 3, 0))); + if (WhichDataType(arguments[0].type).isDateTime()) + { + return std::make_shared(std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 3, 0))); + + } + else + { + auto dt64_scale = assert_cast(*arguments[0].type).getScale(); + return std::make_shared( + std::make_shared(dt64_scale, extractTimeZoneNameFromFunctionArguments(arguments, 3, 0))); + } + } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override { - const auto * starts = checkAndGetColumn(arguments[0].column.get()); - const auto * const_starts = checkAndGetColumnConst(arguments[0].column.get()); + const auto * starts = checkAndGetColumn(arguments[0].column.get()); + const auto * const_starts = checkAndGetColumnConst(arguments[0].column.get()); - const auto * durations = checkAndGetColumn(arguments[1].column.get()); - const auto * const_durations = checkAndGetColumnConst(arguments[1].column.get()); + const auto * durations = checkAndGetColumn(arguments[1].column.get()); + const auto * const_durations = checkAndGetColumnConst(arguments[1].column.get()); - auto res = ColumnArray::create(ColumnUInt32::create()); - ColumnUInt32::Container & res_values = typeid_cast(res->getData()).getData(); + auto res = ColumnArray::create(ColumnInt64::create()); + ColumnInt64::Container & res_values = typeid_cast(res->getData()).getData(); auto time_slot_size = TIME_SLOT_SIZE; @@ -163,33 +174,44 @@ public: { const auto * time_slot_column = checkAndGetColumn(arguments[2].column.get()); if (!time_slot_column) - throw Exception("Third argument for function " + getName() + " must be constant UInt32", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception("Third argument for function " + getName() + " must be constant UInt64", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (time_slot_size = time_slot_column->getValue(); time_slot_size == 0) + if (time_slot_size = time_slot_column->getValue(); time_slot_size == 0) throw Exception("Third argument for function " + getName() + " must be greater than zero", ErrorCodes::ILLEGAL_COLUMN); } if (starts && durations) { - TimeSlotsImpl::vectorVector(starts->getData(), durations->getData(), time_slot_size, res_values, res->getOffsets()); + TimeSlotsImpl::vectorVector(starts->getData(), durations->getData(), time_slot_size, res_values, res->getOffsets()); return res; } else if (starts && const_durations) { - TimeSlotsImpl::vectorConstant(starts->getData(), const_durations->getValue(), time_slot_size, res_values, res->getOffsets()); + TimeSlotsImpl::vectorConstant(starts->getData(), const_durations->getValue(), time_slot_size, res_values, res->getOffsets()); return res; } else if (const_starts && durations) { - TimeSlotsImpl::constantVector(const_starts->getValue(), durations->getData(), time_slot_size, res_values, res->getOffsets()); + TimeSlotsImpl::constantVector(const_starts->getValue(), durations->getData(), time_slot_size, res_values, res->getOffsets()); return res; } else + if (arguments.size() == 3) + { throw Exception("Illegal columns " + arguments[0].column->getName() + ", " + arguments[1].column->getName() + ", " + arguments[2].column->getName() + " of arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); + } + else + { + throw Exception("Illegal columns " + arguments[0].column->getName() + + ", " + arguments[1].column->getName() + + " of arguments of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + } + } }; From 16087ea400ae7b1a6d28096a8a155f6731debba3 Mon Sep 17 00:00:00 2001 From: zvonand Date: Thu, 9 Jun 2022 15:28:18 +0300 Subject: [PATCH 016/139] enable dt64 for timeslots --- src/Functions/timeSlots.cpp | 227 +++++++++++++++++++++++++++--------- 1 file changed, 173 insertions(+), 54 deletions(-) diff --git a/src/Functions/timeSlots.cpp b/src/Functions/timeSlots.cpp index e3f0e0fa7a8..917537058e2 100644 --- a/src/Functions/timeSlots.cpp +++ b/src/Functions/timeSlots.cpp @@ -33,12 +33,12 @@ namespace * But this function was adopted by wider audience. */ -template struct TimeSlotsImpl { + /// The following three methods process DateTime type static void vectorVector( - const PaddedPODArray & starts, const PaddedPODArray & durations, UInt64 time_slot_size, - PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets) + const PaddedPODArray & starts, const PaddedPODArray & durations, UInt32 time_slot_size, + PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets) { size_t size = starts.size(); @@ -48,7 +48,7 @@ struct TimeSlotsImpl ColumnArray::Offset current_offset = 0; for (size_t i = 0; i < size; ++i) { - for (Int64 value = starts[i] / time_slot_size, end = (starts[i] + durations[i]) / time_slot_size; value <= end; ++value) + for (UInt32 value = starts[i] / time_slot_size, end = (starts[i] + durations[i]) / time_slot_size; value <= end; ++value) { result_values.push_back(value * time_slot_size); ++current_offset; @@ -59,8 +59,8 @@ struct TimeSlotsImpl } static void vectorConstant( - const PaddedPODArray & starts, DurationType duration, UInt64 time_slot_size, - PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets) + const PaddedPODArray & starts, UInt32 duration, UInt32 time_slot_size, + PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets) { size_t size = starts.size(); @@ -70,7 +70,7 @@ struct TimeSlotsImpl ColumnArray::Offset current_offset = 0; for (size_t i = 0; i < size; ++i) { - for (Int64 value = starts[i] / time_slot_size, end = (starts[i] + duration) / time_slot_size; value <= end; ++value) + for (UInt32 value = starts[i] / time_slot_size, end = (starts[i] + duration) / time_slot_size; value <= end; ++value) { result_values.push_back(value * time_slot_size); ++current_offset; @@ -81,8 +81,8 @@ struct TimeSlotsImpl } static void constantVector( - Int64 start, const PaddedPODArray & durations, UInt64 time_slot_size, - PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets) + UInt32 start, const PaddedPODArray & durations, UInt32 time_slot_size, + PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets) { size_t size = durations.size(); @@ -92,7 +92,7 @@ struct TimeSlotsImpl ColumnArray::Offset current_offset = 0; for (size_t i = 0; i < size; ++i) { - for (Int64 value = start / time_slot_size, end = (start + durations[i]) / time_slot_size; value <= end; ++value) + for (UInt32 value = start / time_slot_size, end = (start + durations[i]) / time_slot_size; value <= end; ++value) { result_values.push_back(value * time_slot_size); ++current_offset; @@ -101,6 +101,67 @@ struct TimeSlotsImpl result_offsets[i] = current_offset; } } + + /// The following three methods process DateTime64 type + static void vectorVector( + const PaddedPODArray & starts, const PaddedPODArray & durations, UInt32 time_slot_size, + PaddedPODArray & result_values, UInt16 dt_scale, UInt16 duration_scale) + { + size_t size = starts.size(); + result_values.reserve(size); + + int dt_multiplier = dt_scale < duration_scale ? DecimalUtils::scaleMultiplier(std::abs(duration_scale - dt_scale)) : 1; + int dur_multiplier = dt_scale > duration_scale ? DecimalUtils::scaleMultiplier(std::abs(dt_scale - duration_scale)) : 1; + + + for (size_t i = 0; i < size; ++i) + { + for (DateTime64 value = (starts[i] * dt_multiplier) / time_slot_size, end = (starts[i] + durations[i] * dur_multiplier) / time_slot_size; value <= end; value += 1) + { + result_values.push_back(value * time_slot_size); + } + } + } + + static void vectorConstant( + const PaddedPODArray & starts, Decimal64 duration, UInt32 time_slot_size, + PaddedPODArray & result_values, UInt16 dt_scale, UInt16 duration_scale) + { + size_t size = starts.size(); + result_values.reserve(size); + + int dt_multiplier = dt_scale < duration_scale ? DecimalUtils::scaleMultiplier(std::abs(duration_scale - dt_scale)) : 1; + int dur_multiplier = dt_scale > duration_scale ? DecimalUtils::scaleMultiplier(std::abs(dt_scale - duration_scale)) : 1; + + duration = duration * dur_multiplier; + for (size_t i = 0; i < size; ++i) + { + for (DateTime64 value = (starts[i] * dt_multiplier) / time_slot_size, end = (starts[i] + duration) / time_slot_size; value <= end; value += 1) + { + result_values.push_back(value * time_slot_size); + } + } + } + + static void constantVector( + DateTime64 start, const PaddedPODArray & durations, UInt32 time_slot_size, + PaddedPODArray & result_values, UInt16 dt_scale, UInt16 duration_scale) + { + size_t size = durations.size(); + result_values.reserve(size); + + int dt_multiplier = dt_scale < duration_scale ? DecimalUtils::scaleMultiplier(std::abs(duration_scale - dt_scale)) : 1; + int dur_multiplier = dt_scale > duration_scale ? DecimalUtils::scaleMultiplier(std::abs(dt_scale - duration_scale)) : 1; + + start = dt_multiplier * start; + for (size_t i = 0; i < size; ++i) + { + for (DateTime64 value = start / time_slot_size, end = (start + durations[i] * dur_multiplier) / time_slot_size; value <= end; value += 1) + { + result_values.push_back(value * time_slot_size); + } + } + } }; @@ -108,7 +169,6 @@ class FunctionTimeSlots : public IFunction { public: static constexpr auto name = "timeSlots"; - static constexpr UInt64 TIME_SLOT_SIZE = 1800; static FunctionPtr create(ContextPtr) { return std::make_shared(); } String getName() const override @@ -129,89 +189,148 @@ public: + toString(arguments.size()) + ", should be 2 or 3", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - if (!WhichDataType(arguments[0].type).isDateTime() && !WhichDataType(arguments[0].type).isDateTime64()) + if (!isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName() + ". Must be DateTime or DateTime64.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (!WhichDataType(arguments[1].type).isNativeUInt()) - throw Exception("Illegal type " + arguments[1].type->getName() + " of second argument of function " + getName() + ". Must be UInt64.", + if (!isNumber(arguments[1].type)) + { + throw Exception( + "Illegal type " + arguments[1].type->getName() + " of second argument of function " + getName() + ". Must be numeric type.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + if (isDateTime(arguments[0].type)) + { + if (!isUnsignedInteger(arguments[1].type)) + throw Exception( + "Illegal type " + arguments[1].type->getName() + " of second argument of function " + getName() + ". Must be unsigned integer when first argument is DateTime.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (arguments.size() == 3 && !WhichDataType(arguments[2].type).isNativeUInt()) - throw Exception("Illegal type " + arguments[2].type->getName() + " of third argument of function " + getName() + ". Must be UInt64.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (arguments.size() == 3 && !isUnsignedInteger(arguments[2].type)) + throw Exception( + "Illegal type " + arguments[2].type->getName() + " of third argument of function " + getName() + ". Must be unsigned integer when first argument is DateTime.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + else + { + if (!isNumber(arguments[1].type)) + throw Exception( + "Illegal type " + arguments[1].type->getName() + " of second argument of function " + getName() + ". Must be numeric when first argument is DateTime64.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (arguments.size() == 3 && !isNumber(arguments[2].type)) + throw Exception( + "Illegal type " + arguments[2].type->getName() + " of third argument of function " + getName() + ". Must be numeric when first argument is DateTime64.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + + /// If time zone is specified for source data type, attach it to the resulting type. /// Note that there is no explicit time zone argument for this function (we specify 2 as an argument number with explicit time zone). if (WhichDataType(arguments[0].type).isDateTime()) { return std::make_shared(std::make_shared(extractTimeZoneNameFromFunctionArguments(arguments, 3, 0))); - } else { auto dt64_scale = assert_cast(*arguments[0].type).getScale(); + auto duration_scale = assert_cast &>(*arguments[1].type).getScale(); return std::make_shared( - std::make_shared(dt64_scale, extractTimeZoneNameFromFunctionArguments(arguments, 3, 0))); + std::make_shared(std::max(dt64_scale, duration_scale), extractTimeZoneNameFromFunctionArguments(arguments, 3, 0))); } } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override { - const auto * starts = checkAndGetColumn(arguments[0].column.get()); - const auto * const_starts = checkAndGetColumnConst(arguments[0].column.get()); - - const auto * durations = checkAndGetColumn(arguments[1].column.get()); - const auto * const_durations = checkAndGetColumnConst(arguments[1].column.get()); - - auto res = ColumnArray::create(ColumnInt64::create()); - ColumnInt64::Container & res_values = typeid_cast(res->getData()).getData(); - - auto time_slot_size = TIME_SLOT_SIZE; + UInt32 time_slot_size = 1800; if (arguments.size() == 3) { - const auto * time_slot_column = checkAndGetColumn(arguments[2].column.get()); + const auto * time_slot_column = checkAndGetColumnConst(arguments[2].column.get()); if (!time_slot_column) - throw Exception("Third argument for function " + getName() + " must be constant UInt64", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception("Third argument for function " + getName() + " must be constant integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (time_slot_size = time_slot_column->getValue(); time_slot_size == 0) + if (time_slot_size = time_slot_column->getValue(); time_slot_size == 0) throw Exception("Third argument for function " + getName() + " must be greater than zero", ErrorCodes::ILLEGAL_COLUMN); } - if (starts && durations) + if (WhichDataType(arguments[0].type).isDateTime()) { - TimeSlotsImpl::vectorVector(starts->getData(), durations->getData(), time_slot_size, res_values, res->getOffsets()); - return res; + const auto * dt_starts = checkAndGetColumn(arguments[0].column.get()); + const auto * dt_const_starts = checkAndGetColumnConst(arguments[0].column.get()); + + const auto * durations = checkAndGetColumn(arguments[1].column.get()); + const auto * const_durations = checkAndGetColumnConst(arguments[1].column.get()); + + auto res = ColumnArray::create(ColumnUInt32::create()); + ColumnUInt32::Container & res_values = typeid_cast(res->getData()).getData(); + + if (dt_starts && durations) + { + TimeSlotsImpl::vectorVector(dt_starts->getData(), durations->getData(), time_slot_size, res_values, res->getOffsets()); + return res; + } + else if (dt_starts && const_durations) + { + TimeSlotsImpl::vectorConstant(dt_starts->getData(), const_durations->getValue(), time_slot_size, res_values, res->getOffsets()); + return res; + } + else if (dt_const_starts && durations) + { + TimeSlotsImpl::constantVector(dt_const_starts->getValue(), durations->getData(), time_slot_size, res_values, res->getOffsets()); + return res; + } } - else if (starts && const_durations) + else if (WhichDataType(arguments[0].type).isDateTime64()) { - TimeSlotsImpl::vectorConstant(starts->getData(), const_durations->getValue(), time_slot_size, res_values, res->getOffsets()); - return res; + const auto * dt64_starts = checkAndGetColumn(arguments[0].column.get()); + const auto * dt64_const_starts = checkAndGetColumnConst(arguments[0].column.get()); + + const auto * durations = checkAndGetColumn>(arguments[1].column.get()); + const auto * const_durations = checkAndGetColumnConst>(arguments[1].column.get()); + + const auto dt64_scale = assert_cast(arguments[0].type.get())->getScale(); + const auto duration_scale = assert_cast *>(arguments[1].type.get())->getScale(); + + auto res = ColumnArray::create(DataTypeDateTime64(dt64_scale).createColumn()); + auto & res_values = typeid_cast(res->getData()).getData(); + + if (dt64_starts && durations) + { + TimeSlotsImpl::vectorVector(dt64_starts->getData(), durations->getData(), time_slot_size, res_values, dt64_scale, duration_scale); + return res; + } + else if (dt64_starts && const_durations) + { + TimeSlotsImpl::vectorConstant( + dt64_starts->getData(), const_durations->getValue(), time_slot_size, res_values, dt64_scale, duration_scale); + return res; + } + else if (dt64_const_starts && durations) + { + TimeSlotsImpl::constantVector( + dt64_const_starts->getValue(), durations->getData(), time_slot_size, res_values, dt64_scale, duration_scale); + return res; + } } - else if (const_starts && durations) + + if (arguments.size() == 3) { - TimeSlotsImpl::constantVector(const_starts->getValue(), durations->getData(), time_slot_size, res_values, res->getOffsets()); - return res; - } - else - if (arguments.size() == 3) - { - throw Exception("Illegal columns " + arguments[0].column->getName() - + ", " + arguments[1].column->getName() - + ", " + arguments[2].column->getName() - + " of arguments of function " + getName(), + throw Exception( + "Illegal columns " + arguments[0].column->getName() + ", " + arguments[1].column->getName() + ", " + + arguments[2].column->getName() + " of arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); } else { - throw Exception("Illegal columns " + arguments[0].column->getName() - + ", " + arguments[1].column->getName() - + " of arguments of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); + throw Exception( + "Illegal columns " + arguments[0].column->getName() + ", " + arguments[1].column->getName() + + " of arguments of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); } - } }; From e19653618c1b3821f2177e093d6a4c58e713fd0a Mon Sep 17 00:00:00 2001 From: zvonand Date: Fri, 10 Jun 2022 11:19:38 +0300 Subject: [PATCH 017/139] fix wrongfully added submodule --- contrib/eigen | 1 - src/Functions/timeSlots.cpp | 34 ++++++++++++++++++++++++---------- 2 files changed, 24 insertions(+), 11 deletions(-) delete mode 160000 contrib/eigen diff --git a/contrib/eigen b/contrib/eigen deleted file mode 160000 index 3147391d946..00000000000 --- a/contrib/eigen +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3147391d946bb4b6c68edd901f2add6ac1f31f8c diff --git a/src/Functions/timeSlots.cpp b/src/Functions/timeSlots.cpp index 917537058e2..0163c29e8ef 100644 --- a/src/Functions/timeSlots.cpp +++ b/src/Functions/timeSlots.cpp @@ -105,61 +105,75 @@ struct TimeSlotsImpl /// The following three methods process DateTime64 type static void vectorVector( const PaddedPODArray & starts, const PaddedPODArray & durations, UInt32 time_slot_size, - PaddedPODArray & result_values, UInt16 dt_scale, UInt16 duration_scale) + PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets, UInt16 dt_scale, UInt16 duration_scale) { size_t size = starts.size(); + + result_offsets.resize(size); result_values.reserve(size); int dt_multiplier = dt_scale < duration_scale ? DecimalUtils::scaleMultiplier(std::abs(duration_scale - dt_scale)) : 1; int dur_multiplier = dt_scale > duration_scale ? DecimalUtils::scaleMultiplier(std::abs(dt_scale - duration_scale)) : 1; - + ColumnArray::Offset current_offset = 0; for (size_t i = 0; i < size; ++i) { for (DateTime64 value = (starts[i] * dt_multiplier) / time_slot_size, end = (starts[i] + durations[i] * dur_multiplier) / time_slot_size; value <= end; value += 1) { result_values.push_back(value * time_slot_size); + ++current_offset; } + result_offsets[i] = current_offset; } } static void vectorConstant( const PaddedPODArray & starts, Decimal64 duration, UInt32 time_slot_size, - PaddedPODArray & result_values, UInt16 dt_scale, UInt16 duration_scale) + PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets, UInt16 dt_scale, UInt16 duration_scale) { size_t size = starts.size(); + + result_offsets.resize(size); result_values.reserve(size); int dt_multiplier = dt_scale < duration_scale ? DecimalUtils::scaleMultiplier(std::abs(duration_scale - dt_scale)) : 1; int dur_multiplier = dt_scale > duration_scale ? DecimalUtils::scaleMultiplier(std::abs(dt_scale - duration_scale)) : 1; + ColumnArray::Offset current_offset = 0; duration = duration * dur_multiplier; for (size_t i = 0; i < size; ++i) { for (DateTime64 value = (starts[i] * dt_multiplier) / time_slot_size, end = (starts[i] + duration) / time_slot_size; value <= end; value += 1) { result_values.push_back(value * time_slot_size); + ++current_offset; } + result_offsets[i] = current_offset; } } static void constantVector( DateTime64 start, const PaddedPODArray & durations, UInt32 time_slot_size, - PaddedPODArray & result_values, UInt16 dt_scale, UInt16 duration_scale) + PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets, UInt16 dt_scale, UInt16 duration_scale) { size_t size = durations.size(); + + result_offsets.resize(size); result_values.reserve(size); int dt_multiplier = dt_scale < duration_scale ? DecimalUtils::scaleMultiplier(std::abs(duration_scale - dt_scale)) : 1; int dur_multiplier = dt_scale > duration_scale ? DecimalUtils::scaleMultiplier(std::abs(dt_scale - duration_scale)) : 1; + ColumnArray::Offset current_offset = 0; start = dt_multiplier * start; for (size_t i = 0; i < size; ++i) { for (DateTime64 value = start / time_slot_size, end = (start + durations[i] * dur_multiplier) / time_slot_size; value <= end; value += 1) { result_values.push_back(value * time_slot_size); + ++current_offset; } + result_offsets[i] = current_offset; } } }; @@ -257,7 +271,7 @@ public: throw Exception("Third argument for function " + getName() + " must be greater than zero", ErrorCodes::ILLEGAL_COLUMN); } - if (WhichDataType(arguments[0].type).isDateTime()) + if (isDateTime(arguments[0].type)) { const auto * dt_starts = checkAndGetColumn(arguments[0].column.get()); const auto * dt_const_starts = checkAndGetColumnConst(arguments[0].column.get()); @@ -284,7 +298,7 @@ public: return res; } } - else if (WhichDataType(arguments[0].type).isDateTime64()) + else if (isDateTime64(arguments[0].type)) { const auto * dt64_starts = checkAndGetColumn(arguments[0].column.get()); const auto * dt64_const_starts = checkAndGetColumnConst(arguments[0].column.get()); @@ -296,23 +310,23 @@ public: const auto duration_scale = assert_cast *>(arguments[1].type.get())->getScale(); auto res = ColumnArray::create(DataTypeDateTime64(dt64_scale).createColumn()); - auto & res_values = typeid_cast(res->getData()).getData(); + DataTypeDateTime64::ColumnType::Container & res_values = typeid_cast(res->getData()).getData(); if (dt64_starts && durations) { - TimeSlotsImpl::vectorVector(dt64_starts->getData(), durations->getData(), time_slot_size, res_values, dt64_scale, duration_scale); + TimeSlotsImpl::vectorVector(dt64_starts->getData(), durations->getData(), time_slot_size, res_values, res->getOffsets(), dt64_scale, duration_scale); return res; } else if (dt64_starts && const_durations) { TimeSlotsImpl::vectorConstant( - dt64_starts->getData(), const_durations->getValue(), time_slot_size, res_values, dt64_scale, duration_scale); + dt64_starts->getData(), const_durations->getValue(), time_slot_size, res_values, res->getOffsets(), dt64_scale, duration_scale); return res; } else if (dt64_const_starts && durations) { TimeSlotsImpl::constantVector( - dt64_const_starts->getValue(), durations->getData(), time_slot_size, res_values, dt64_scale, duration_scale); + dt64_const_starts->getValue(), durations->getData(), time_slot_size, res_values, res->getOffsets(), dt64_scale, duration_scale); return res; } } From 551d1ea8756611528779464d9e33d74e67a50e61 Mon Sep 17 00:00:00 2001 From: zvonand Date: Fri, 10 Jun 2022 13:21:31 +0300 Subject: [PATCH 018/139] fix wrong interval --- src/Functions/timeSlots.cpp | 133 ++++++++++++++++++++---------------- 1 file changed, 75 insertions(+), 58 deletions(-) diff --git a/src/Functions/timeSlots.cpp b/src/Functions/timeSlots.cpp index 0163c29e8ef..142a36db943 100644 --- a/src/Functions/timeSlots.cpp +++ b/src/Functions/timeSlots.cpp @@ -104,18 +104,23 @@ struct TimeSlotsImpl /// The following three methods process DateTime64 type static void vectorVector( - const PaddedPODArray & starts, const PaddedPODArray & durations, UInt32 time_slot_size, - PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets, UInt16 dt_scale, UInt16 duration_scale) + const PaddedPODArray & starts, const PaddedPODArray & durations, Decimal64 time_slot_size, + PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets, UInt16 dt_scale, UInt16 duration_scale, UInt16 time_slot_scale) { size_t size = starts.size(); result_offsets.resize(size); result_values.reserve(size); - int dt_multiplier = dt_scale < duration_scale ? DecimalUtils::scaleMultiplier(std::abs(duration_scale - dt_scale)) : 1; - int dur_multiplier = dt_scale > duration_scale ? DecimalUtils::scaleMultiplier(std::abs(dt_scale - duration_scale)) : 1; + UInt16 max_scale = dt_scale > duration_scale ? dt_scale : duration_scale; + max_scale = time_slot_scale > max_scale ? time_slot_scale : max_scale; + + Int64 dt_multiplier = DecimalUtils::scaleMultiplier(max_scale - dt_scale); + Int64 dur_multiplier = DecimalUtils::scaleMultiplier(max_scale - duration_scale); + Int64 ts_multiplier = DecimalUtils::scaleMultiplier(max_scale - time_slot_scale); ColumnArray::Offset current_offset = 0; + time_slot_size = time_slot_size.value * ts_multiplier; for (size_t i = 0; i < size; ++i) { for (DateTime64 value = (starts[i] * dt_multiplier) / time_slot_size, end = (starts[i] + durations[i] * dur_multiplier) / time_slot_size; value <= end; value += 1) @@ -128,19 +133,24 @@ struct TimeSlotsImpl } static void vectorConstant( - const PaddedPODArray & starts, Decimal64 duration, UInt32 time_slot_size, - PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets, UInt16 dt_scale, UInt16 duration_scale) + const PaddedPODArray & starts, Decimal64 duration, Decimal64 time_slot_size, + PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets, UInt16 dt_scale, UInt16 duration_scale, UInt16 time_slot_scale) { size_t size = starts.size(); result_offsets.resize(size); result_values.reserve(size); - int dt_multiplier = dt_scale < duration_scale ? DecimalUtils::scaleMultiplier(std::abs(duration_scale - dt_scale)) : 1; - int dur_multiplier = dt_scale > duration_scale ? DecimalUtils::scaleMultiplier(std::abs(dt_scale - duration_scale)) : 1; + UInt16 max_scale = dt_scale > duration_scale ? dt_scale : duration_scale; + max_scale = time_slot_scale > max_scale ? time_slot_scale : max_scale; + + Int64 dt_multiplier = DecimalUtils::scaleMultiplier(max_scale - dt_scale); + Int64 dur_multiplier = DecimalUtils::scaleMultiplier(max_scale - duration_scale); + Int64 ts_multiplier = DecimalUtils::scaleMultiplier(max_scale - time_slot_scale); ColumnArray::Offset current_offset = 0; duration = duration * dur_multiplier; + time_slot_size = time_slot_size.value * ts_multiplier; for (size_t i = 0; i < size; ++i) { for (DateTime64 value = (starts[i] * dt_multiplier) / time_slot_size, end = (starts[i] + duration) / time_slot_size; value <= end; value += 1) @@ -153,19 +163,24 @@ struct TimeSlotsImpl } static void constantVector( - DateTime64 start, const PaddedPODArray & durations, UInt32 time_slot_size, - PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets, UInt16 dt_scale, UInt16 duration_scale) + DateTime64 start, const PaddedPODArray & durations, Decimal64 time_slot_size, + PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets, UInt16 dt_scale, UInt16 duration_scale, UInt16 time_slot_scale) { size_t size = durations.size(); result_offsets.resize(size); result_values.reserve(size); - int dt_multiplier = dt_scale < duration_scale ? DecimalUtils::scaleMultiplier(std::abs(duration_scale - dt_scale)) : 1; - int dur_multiplier = dt_scale > duration_scale ? DecimalUtils::scaleMultiplier(std::abs(dt_scale - duration_scale)) : 1; + UInt16 max_scale = dt_scale > duration_scale ? dt_scale : duration_scale; + max_scale = time_slot_scale > max_scale ? time_slot_scale : max_scale; + + Int64 dt_multiplier = DecimalUtils::scaleMultiplier(max_scale - dt_scale); + Int64 dur_multiplier = DecimalUtils::scaleMultiplier(max_scale - duration_scale); + Int64 ts_multiplier = DecimalUtils::scaleMultiplier(max_scale - time_slot_scale); ColumnArray::Offset current_offset = 0; start = dt_multiplier * start; + time_slot_size = time_slot_size.value * ts_multiplier; for (size_t i = 0; i < size; ++i) { for (DateTime64 value = start / time_slot_size, end = (start + durations[i] * dur_multiplier) / time_slot_size; value <= end; value += 1) @@ -203,43 +218,33 @@ public: + toString(arguments.size()) + ", should be 2 or 3", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - if (!isDateTime(arguments[0].type) && !isDateTime64(arguments[0].type)) - throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName() - + ". Must be DateTime or DateTime64.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (!isNumber(arguments[1].type)) - { - throw Exception( - "Illegal type " + arguments[1].type->getName() + " of second argument of function " + getName() + ". Must be numeric type.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - if (isDateTime(arguments[0].type)) { - if (!isUnsignedInteger(arguments[1].type)) - throw Exception( - "Illegal type " + arguments[1].type->getName() + " of second argument of function " + getName() + ". Must be unsigned integer when first argument is DateTime.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (arguments.size() == 3 && !isUnsignedInteger(arguments[2].type)) + if (!WhichDataType(arguments[1].type).isUInt32()) throw Exception( - "Illegal type " + arguments[2].type->getName() + " of third argument of function " + getName() + ". Must be unsigned integer when first argument is DateTime.", + "Illegal type " + arguments[1].type->getName() + " of second argument of function " + getName() + ". Must be UInt32 when first argument is DateTime.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (arguments.size() == 3 && !WhichDataType(arguments[2].type).isUInt32()) + throw Exception( + "Illegal type " + arguments[2].type->getName() + " of third argument of function " + getName() + ". Must be UInt32 when first argument is DateTime.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + else if (isDateTime64(arguments[0].type)) + { + if (!WhichDataType(arguments[1].type).isDecimal64()) + throw Exception( + "Illegal type " + arguments[1].type->getName() + " of second argument of function " + getName() + ". Must be Decimal64 when first argument is DateTime64.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (arguments.size() == 3 && !WhichDataType(arguments[2].type).isDecimal64()) + throw Exception( + "Illegal type " + arguments[2].type->getName() + " of third argument of function " + getName() + ". Must be Decimal64 when first argument is DateTime64.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } else - { - if (!isNumber(arguments[1].type)) - throw Exception( - "Illegal type " + arguments[1].type->getName() + " of second argument of function " + getName() + ". Must be numeric when first argument is DateTime64.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (arguments.size() == 3 && !isNumber(arguments[2].type)) - throw Exception( - "Illegal type " + arguments[2].type->getName() + " of third argument of function " + getName() + ". Must be numeric when first argument is DateTime64.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } - - + throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName() + + ". Must be DateTime or DateTime64.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); /// If time zone is specified for source data type, attach it to the resulting type. /// Note that there is no explicit time zone argument for this function (we specify 2 as an argument number with explicit time zone). @@ -259,20 +264,19 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override { - UInt32 time_slot_size = 1800; - - if (arguments.size() == 3) - { - const auto * time_slot_column = checkAndGetColumnConst(arguments[2].column.get()); - if (!time_slot_column) - throw Exception("Third argument for function " + getName() + " must be constant integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (time_slot_size = time_slot_column->getValue(); time_slot_size == 0) - throw Exception("Third argument for function " + getName() + " must be greater than zero", ErrorCodes::ILLEGAL_COLUMN); - } - if (isDateTime(arguments[0].type)) { + UInt32 time_slot_size = 1800; + if (arguments.size() == 3) + { + const auto * time_slot_column = checkAndGetColumnConst(arguments[2].column.get()); + if (!time_slot_column) + throw Exception("Third argument for function " + getName() + " must be constant integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (time_slot_size = time_slot_column->getValue(); time_slot_size == 0) + throw Exception("Third argument for function " + getName() + " must be greater than zero", ErrorCodes::ILLEGAL_COLUMN); + } + const auto * dt_starts = checkAndGetColumn(arguments[0].column.get()); const auto * dt_const_starts = checkAndGetColumnConst(arguments[0].column.get()); @@ -300,6 +304,19 @@ public: } else if (isDateTime64(arguments[0].type)) { + Decimal64 time_slot_size = Decimal64(1800); + UInt16 time_slot_scale = 0; + if (arguments.size() == 3) + { + const auto * time_slot_column = checkAndGetColumnConst>(arguments[2].column.get()); + if (!time_slot_column) + throw Exception("Third argument for function " + getName() + " must be constant integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (time_slot_size = time_slot_column->getValue(); time_slot_size == 0) + throw Exception("Third argument for function " + getName() + " must be greater than zero", ErrorCodes::ILLEGAL_COLUMN); + time_slot_scale = assert_cast *>(arguments[2].type.get())->getScale(); + } + const auto * dt64_starts = checkAndGetColumn(arguments[0].column.get()); const auto * dt64_const_starts = checkAndGetColumnConst(arguments[0].column.get()); @@ -314,19 +331,19 @@ public: if (dt64_starts && durations) { - TimeSlotsImpl::vectorVector(dt64_starts->getData(), durations->getData(), time_slot_size, res_values, res->getOffsets(), dt64_scale, duration_scale); + TimeSlotsImpl::vectorVector(dt64_starts->getData(), durations->getData(), time_slot_size, res_values, res->getOffsets(), dt64_scale, duration_scale, time_slot_scale); return res; } else if (dt64_starts && const_durations) { TimeSlotsImpl::vectorConstant( - dt64_starts->getData(), const_durations->getValue(), time_slot_size, res_values, res->getOffsets(), dt64_scale, duration_scale); + dt64_starts->getData(), const_durations->getValue(), time_slot_size, res_values, res->getOffsets(), dt64_scale, duration_scale, time_slot_scale); return res; } else if (dt64_const_starts && durations) { TimeSlotsImpl::constantVector( - dt64_const_starts->getValue(), durations->getData(), time_slot_size, res_values, res->getOffsets(), dt64_scale, duration_scale); + dt64_const_starts->getValue(), durations->getData(), time_slot_size, res_values, res->getOffsets(), dt64_scale, duration_scale, time_slot_scale); return res; } } From b1c143fbce49c33773ac504c8ab14b3b8f152918 Mon Sep 17 00:00:00 2001 From: zvonand Date: Fri, 10 Jun 2022 13:40:23 +0300 Subject: [PATCH 019/139] added tests --- tests/queries/0_stateless/02319_timeslots_dt64.reference | 3 +++ tests/queries/0_stateless/02319_timeslots_dt64.sql | 9 +++++++++ 2 files changed, 12 insertions(+) create mode 100644 tests/queries/0_stateless/02319_timeslots_dt64.reference create mode 100644 tests/queries/0_stateless/02319_timeslots_dt64.sql diff --git a/tests/queries/0_stateless/02319_timeslots_dt64.reference b/tests/queries/0_stateless/02319_timeslots_dt64.reference new file mode 100644 index 00000000000..83041581fe6 --- /dev/null +++ b/tests/queries/0_stateless/02319_timeslots_dt64.reference @@ -0,0 +1,3 @@ +['2000-01-02 03:00:00.00','2000-01-02 03:30:00.00','2000-01-02 04:00:00.00','2000-01-02 04:30:00.00','2000-01-02 05:00:00.00','2000-01-02 05:30:00.00'] +['2000-01-02 02:54:45.100','2000-01-02 03:05:19.200','2000-01-02 03:15:53.300','2000-01-02 03:26:27.400','2000-01-02 03:37:01.500','2000-01-02 03:47:35.600','2000-01-02 03:58:09.700','2000-01-02 04:08:43.800','2000-01-02 04:19:17.900','2000-01-02 04:29:52.000','2000-01-02 04:40:26.100','2000-01-02 04:51:00.200','2000-01-02 05:01:34.300','2000-01-02 05:12:08.400','2000-01-02 05:22:42.500','2000-01-02 05:33:16.600','2000-01-02 05:43:50.700'] +['2000-01-02 03:04:00.0000','2000-01-02 03:04:30.0000','2000-01-02 03:05:00.0000','2000-01-02 03:05:30.0000','2000-01-02 03:06:00.0000','2000-01-02 03:06:30.0000','2000-01-02 03:07:00.0000','2000-01-02 03:07:30.0000','2000-01-02 03:08:00.0000','2000-01-02 03:08:30.0000','2000-01-02 03:09:00.0000','2000-01-02 03:09:30.0000','2000-01-02 03:10:00.0000','2000-01-02 03:10:30.0000','2000-01-02 03:11:00.0000','2000-01-02 03:11:30.0000','2000-01-02 03:12:00.0000','2000-01-02 03:12:30.0000','2000-01-02 03:13:00.0000','2000-01-02 03:13:30.0000','2000-01-02 03:14:00.0000'] diff --git a/tests/queries/0_stateless/02319_timeslots_dt64.sql b/tests/queries/0_stateless/02319_timeslots_dt64.sql new file mode 100644 index 00000000000..3d8f8a22e5a --- /dev/null +++ b/tests/queries/0_stateless/02319_timeslots_dt64.sql @@ -0,0 +1,9 @@ +SELECT timeSlots(toDateTime64('2000-01-02 03:04:05.12', 2, 'UTC'), toDecimal64(10000, 0)); +SELECT timeSlots(toDateTime64('2000-01-02 03:04:05.233', 3, 'UTC'), toDecimal64(10000.12, 2), toDecimal64(634.1, 1)); +SELECT timeSlots(toDateTime64('2000-01-02 03:04:05.3456', 4, 'UTC'), toDecimal64(600, 0), toDecimal64(30, 0)); + +SELECT timeSlots(toDateTime64('2000-01-02 03:04:05.23', 2, 'UTC')); -- { serverError 42 } +SELECT timeSlots(toDateTime64('2000-01-02 03:04:05.345', 3, 'UTC'), toDecimal64(62.3, 1), toDecimal64(12.34, 2), 'one more'); -- { serverError 42 } +SELECT timeSlots(toDateTime64('2000-01-02 03:04:05.456', 3, 'UTC'), 'wrong argument'); -- { serverError 43 } +SELECT timeSlots(toDateTime64('2000-01-02 03:04:05.123', 3, 'UTC'), toDecimal64(600, 0), 'wrong argument'); -- { serverError 43 } +SELECT timeSlots(toDateTime64('2000-01-02 03:04:05.1232', 4, 'UTC'), toDecimal64(600, 0), toDecimal64(0, 0)); -- { serverError 44 } \ No newline at end of file From fb67b080b9ea122d880c1fa6460f8c911e1cbf62 Mon Sep 17 00:00:00 2001 From: zvonand Date: Fri, 10 Jun 2022 14:30:17 +0300 Subject: [PATCH 020/139] added docs --- .../functions/date-time-functions.md | 23 +++++++++++++--- .../functions/date-time-functions.md | 27 ++++++++++++++----- src/Functions/timeSlots.cpp | 10 +++---- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 621429fb02c..f2500e6eedc 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -950,9 +950,26 @@ SELECT ## timeSlots(StartTime, Duration,\[, Size\]) -For a time interval starting at ‘StartTime’ and continuing for ‘Duration’ seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the ‘Size’ in seconds. ‘Size’ is an optional parameter: a constant UInt32, set to 1800 by default. -For example, `timeSlots(toDateTime('2012-01-01 12:20:00'), 600) = [toDateTime('2012-01-01 12:00:00'), toDateTime('2012-01-01 12:30:00')]`. -This is necessary for searching for pageviews in the corresponding session. +For a time interval starting at ‘StartTime’ and continuing for ‘Duration’ seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the ‘Size’ in seconds. ‘Size’ is an optional parameter set to 1800 (30 minutes) by default. +This is necessary for searching for pageviews in the corresponding session. +Accepts DateTime and DateTime64. For DateTime, `Duration` and `Size` arguments must be `UInt32`. For DateTime64 they must be `Decimal64`. +Example: +```sql +SELECT timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0)); +SELECT timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299); +SELECT timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600)); +``` +``` text +┌─timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0))─┐ +│ ['1980-12-12 20:56:13.0000','1980-12-12 21:01:12.0000','1980-12-12 21:06:11.0000'] │ +└───────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +┌─timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299)─┐ +│ ['1980-12-12 20:56:13','1980-12-12 21:01:12','1980-12-12 21:06:11'] │ +└─────────────────────────────────────────────────────────────────────────┘ +┌─timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600))─┐ +│ ['2012-01-01 12:00:00','2012-01-01 12:30:00'] │ +└─────────────────────────────────────────────────────────────┘ +``` ## formatDateTime diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index da48cd940a7..b38e1ddf918 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -944,14 +944,29 @@ SELECT now('Europe/Moscow'); ## timeSlot {#timeslot} Округляет время до получаса. -Эта функция является специфичной для Яндекс.Метрики, так как пол часа - минимальное время, для которого, если соседние по времени хиты одного посетителя на одном счётчике отстоят друг от друга строго более, чем на это время, визит может быть разбит на два визита. То есть, кортежи (номер счётчика, идентификатор посетителя, тайм-слот) могут использоваться для поиска хитов, входящий в соответствующий визит. +Эта функция является специфичной для Яндекс.Метрики, так как полчаса - минимальное время, для которого, если соседние по времени хиты одного посетителя на одном счётчике отстоят друг от друга строго более, чем на это время, визит может быть разбит на два визита. То есть, кортежи (номер счётчика, идентификатор посетителя, тайм-слот) могут использоваться для поиска хитов, входящий в соответствующий визит. ## timeSlots(StartTime, Duration,\[, Size\]) {#timeslotsstarttime-duration-size} - -Для интервала времени, начинающегося в ‘StartTime’ и продолжающегося ‘Duration’ секунд, возвращает массив моментов времени, состоящий из округлений вниз до ‘Size’ точек в секундах из этого интервала. ‘Size’ - необязательный параметр, константный UInt32, по умолчанию равен 1800. - -Например, `timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600)) = [toDateTime('2012-01-01 12:00:00'), toDateTime('2012-01-01 12:30:00')]`. -Это нужно для поиска хитов, входящих в соответствующий визит. +Для интервала, начинающегося в `StartTime` и длящегося `Duration` секунд, возвращает массив моментов времени, кратных `Size`. Параметр `Size` указывать необязательно, по умолчанию он равен 1800 секундам (30 минутам) - необязательный параметр. +Данная функция может использоваться, например, для анализа количества просмотров страницы за соответствующую сессию. +Аргумент `StartTime` может иметь тип DateTime или DateTime64. В случае, если используется DateTime, аргументы `Duration` и `Size` должны иметь тип `UInt32`; Для DateTime64 они должны быть типа `Decimal64`. +Пример использования: +```sql +SELECT timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0)); +SELECT timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299); +SELECT timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600)); +``` +``` text +┌─timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0))─┐ +│ ['1980-12-12 20:56:13.0000','1980-12-12 21:01:12.0000','1980-12-12 21:06:11.0000'] │ +└───────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +┌─timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299)─┐ +│ ['1980-12-12 20:56:13','1980-12-12 21:01:12','1980-12-12 21:06:11'] │ +└─────────────────────────────────────────────────────────────────────────┘ +┌─timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600))─┐ +│ ['2012-01-01 12:00:00','2012-01-01 12:30:00'] │ +└─────────────────────────────────────────────────────────────┘ +``` ## toYYYYMM diff --git a/src/Functions/timeSlots.cpp b/src/Functions/timeSlots.cpp index 142a36db943..9ec98ff9fd3 100644 --- a/src/Functions/timeSlots.cpp +++ b/src/Functions/timeSlots.cpp @@ -225,7 +225,7 @@ public: "Illegal type " + arguments[1].type->getName() + " of second argument of function " + getName() + ". Must be UInt32 when first argument is DateTime.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (arguments.size() == 3 && !WhichDataType(arguments[2].type).isUInt32()) + if (arguments.size() == 3 && !WhichDataType(arguments[2].type).isNativeUInt()) throw Exception( "Illegal type " + arguments[2].type->getName() + " of third argument of function " + getName() + ". Must be UInt32 when first argument is DateTime.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); @@ -269,9 +269,9 @@ public: UInt32 time_slot_size = 1800; if (arguments.size() == 3) { - const auto * time_slot_column = checkAndGetColumnConst(arguments[2].column.get()); + const auto * time_slot_column = checkAndGetColumn(arguments[2].column.get()); if (!time_slot_column) - throw Exception("Third argument for function " + getName() + " must be constant integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception("Third argument for function " + getName() + " must be constant UInt32", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (time_slot_size = time_slot_column->getValue(); time_slot_size == 0) throw Exception("Third argument for function " + getName() + " must be greater than zero", ErrorCodes::ILLEGAL_COLUMN); @@ -308,9 +308,9 @@ public: UInt16 time_slot_scale = 0; if (arguments.size() == 3) { - const auto * time_slot_column = checkAndGetColumnConst>(arguments[2].column.get()); + const auto * time_slot_column = checkAndGetColumn(arguments[2].column.get()); if (!time_slot_column) - throw Exception("Third argument for function " + getName() + " must be constant integer", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception("Third argument for function " + getName() + " must be constant Decimal64", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); if (time_slot_size = time_slot_column->getValue(); time_slot_size == 0) throw Exception("Third argument for function " + getName() + " must be greater than zero", ErrorCodes::ILLEGAL_COLUMN); From 54b8709cb10a0d7f2b3e40b4a41e2b869f84aa5e Mon Sep 17 00:00:00 2001 From: zvonand Date: Mon, 13 Jun 2022 19:21:07 +0500 Subject: [PATCH 021/139] minor fix --- src/Functions/timeSlots.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Functions/timeSlots.cpp b/src/Functions/timeSlots.cpp index 9ec98ff9fd3..523379ef7b8 100644 --- a/src/Functions/timeSlots.cpp +++ b/src/Functions/timeSlots.cpp @@ -112,8 +112,9 @@ struct TimeSlotsImpl result_offsets.resize(size); result_values.reserve(size); - UInt16 max_scale = dt_scale > duration_scale ? dt_scale : duration_scale; - max_scale = time_slot_scale > max_scale ? time_slot_scale : max_scale; + /// Modify all units to have same scale + UInt16 max_scale = (dt_scale > duration_scale) ? dt_scale : duration_scale; + max_scale = (time_slot_scale > max_scale) ? time_slot_scale : max_scale; Int64 dt_multiplier = DecimalUtils::scaleMultiplier(max_scale - dt_scale); Int64 dur_multiplier = DecimalUtils::scaleMultiplier(max_scale - duration_scale); @@ -123,7 +124,7 @@ struct TimeSlotsImpl time_slot_size = time_slot_size.value * ts_multiplier; for (size_t i = 0; i < size; ++i) { - for (DateTime64 value = (starts[i] * dt_multiplier) / time_slot_size, end = (starts[i] + durations[i] * dur_multiplier) / time_slot_size; value <= end; value += 1) + for (DateTime64 value = (starts[i] * dt_multiplier) / time_slot_size, end = (starts[i] * dt_multiplier + durations[i] * dur_multiplier) / time_slot_size; value <= end; value += 1) { result_values.push_back(value * time_slot_size); ++current_offset; @@ -141,8 +142,9 @@ struct TimeSlotsImpl result_offsets.resize(size); result_values.reserve(size); - UInt16 max_scale = dt_scale > duration_scale ? dt_scale : duration_scale; - max_scale = time_slot_scale > max_scale ? time_slot_scale : max_scale; + /// Modify all units to have same scale + UInt16 max_scale = (dt_scale > duration_scale) ? dt_scale : duration_scale; + max_scale = (time_slot_scale > max_scale) ? time_slot_scale : max_scale; Int64 dt_multiplier = DecimalUtils::scaleMultiplier(max_scale - dt_scale); Int64 dur_multiplier = DecimalUtils::scaleMultiplier(max_scale - duration_scale); @@ -153,7 +155,7 @@ struct TimeSlotsImpl time_slot_size = time_slot_size.value * ts_multiplier; for (size_t i = 0; i < size; ++i) { - for (DateTime64 value = (starts[i] * dt_multiplier) / time_slot_size, end = (starts[i] + duration) / time_slot_size; value <= end; value += 1) + for (DateTime64 value = (starts[i] * dt_multiplier) / time_slot_size, end = (starts[i] * dt_multiplier + duration) / time_slot_size; value <= end; value += 1) { result_values.push_back(value * time_slot_size); ++current_offset; @@ -171,8 +173,9 @@ struct TimeSlotsImpl result_offsets.resize(size); result_values.reserve(size); - UInt16 max_scale = dt_scale > duration_scale ? dt_scale : duration_scale; - max_scale = time_slot_scale > max_scale ? time_slot_scale : max_scale; + /// Modify all units to have same scale + UInt16 max_scale = (dt_scale > duration_scale) ? dt_scale : duration_scale; + max_scale = (time_slot_scale > max_scale) ? time_slot_scale : max_scale; Int64 dt_multiplier = DecimalUtils::scaleMultiplier(max_scale - dt_scale); Int64 dur_multiplier = DecimalUtils::scaleMultiplier(max_scale - duration_scale); From a5a980b69d82d5a8a03b15c6a80664f6a07df162 Mon Sep 17 00:00:00 2001 From: zvonand Date: Mon, 13 Jun 2022 19:45:54 +0500 Subject: [PATCH 022/139] Added no_sanitize --- src/Functions/timeSlots.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Functions/timeSlots.cpp b/src/Functions/timeSlots.cpp index 523379ef7b8..663cc52eb6b 100644 --- a/src/Functions/timeSlots.cpp +++ b/src/Functions/timeSlots.cpp @@ -101,9 +101,12 @@ struct TimeSlotsImpl result_offsets[i] = current_offset; } } - - /// The following three methods process DateTime64 type - static void vectorVector( + /* + The following three methods process DateTime64 type + NO_SANITIZE_UNDEFINED is put here because user shall be careful when working with Decimal + Adjusting different scales can cause overflow -- it is OK for us. Don't use scales that differ a lot :) + */ + static NO_SANITIZE_UNDEFINED void vectorVector( const PaddedPODArray & starts, const PaddedPODArray & durations, Decimal64 time_slot_size, PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets, UInt16 dt_scale, UInt16 duration_scale, UInt16 time_slot_scale) { @@ -133,7 +136,7 @@ struct TimeSlotsImpl } } - static void vectorConstant( + static NO_SANITIZE_UNDEFINED void vectorConstant( const PaddedPODArray & starts, Decimal64 duration, Decimal64 time_slot_size, PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets, UInt16 dt_scale, UInt16 duration_scale, UInt16 time_slot_scale) { @@ -164,7 +167,7 @@ struct TimeSlotsImpl } } - static void constantVector( + static NO_SANITIZE_UNDEFINED void constantVector( DateTime64 start, const PaddedPODArray & durations, Decimal64 time_slot_size, PaddedPODArray & result_values, ColumnArray::Offsets & result_offsets, UInt16 dt_scale, UInt16 duration_scale, UInt16 time_slot_scale) { From fdf35b20a836e5dccf0b0452e43132921defbdfd Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 17 Jun 2022 17:32:26 +0000 Subject: [PATCH 023/139] Limit number of analyze for one query --- src/Interpreters/Context.h | 9 +++++++++ src/Interpreters/InterpreterSelectQuery.cpp | 10 ++++++++++ .../0_stateless/02337_join_analyze_stuck.reference | 0 .../0_stateless/02337_join_analyze_stuck.sql | 13 +++++++++++++ 4 files changed, 32 insertions(+) create mode 100644 tests/queries/0_stateless/02337_join_analyze_stuck.reference create mode 100644 tests/queries/0_stateless/02337_join_analyze_stuck.sql diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 83193dd589b..6e3e6de76c1 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -364,6 +364,15 @@ public: // Top-level OpenTelemetry trace context for the query. Makes sense only for a query context. OpenTelemetryTraceContext query_trace_context; + /// Some counters for current query execution. + /// Most of them is workarounds and should be removed in the future. + struct KitchenSink + { + size_t analyze_counter = 0; + }; + + KitchenSink kitchen_sink; + private: using SampleBlockCache = std::unordered_map; mutable SampleBlockCache sample_block_cache; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 4f1caf74a45..4cdd0428a0d 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -98,6 +98,7 @@ namespace ErrorCodes extern const int SAMPLING_NOT_SUPPORTED; extern const int ILLEGAL_FINAL; extern const int ILLEGAL_PREWHERE; + extern const int TOO_DEEP_PIPELINE; extern const int TOO_MANY_COLUMNS; extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; @@ -450,6 +451,15 @@ InterpreterSelectQuery::InterpreterSelectQuery( auto analyze = [&] (bool try_move_to_prewhere) { + if (context->hasQueryContext()) + { + size_t & current_query_analyze_count = context->getQueryContext()->kitchen_sink.analyze_counter; + current_query_analyze_count++; + if (current_query_analyze_count >= settings.max_pipeline_depth) + throw DB::Exception(ErrorCodes::TOO_DEEP_PIPELINE, "Query analyze overflow. Try to increase `max_pipeline_depth` or simplify the query" ); + } + + /// Allow push down and other optimizations for VIEW: replace with subquery and rewrite it. ASTPtr view_table; if (view) diff --git a/tests/queries/0_stateless/02337_join_analyze_stuck.reference b/tests/queries/0_stateless/02337_join_analyze_stuck.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02337_join_analyze_stuck.sql b/tests/queries/0_stateless/02337_join_analyze_stuck.sql new file mode 100644 index 00000000000..8d8f604906f --- /dev/null +++ b/tests/queries/0_stateless/02337_join_analyze_stuck.sql @@ -0,0 +1,13 @@ +-- Tags: long + +-- https://github.com/ClickHouse/ClickHouse/issues/21557 +-- Not stuck at first, but proper fix should make able to process it +EXPLAIN SYNTAX +WITH + x AS ( SELECT number FROM numbers(10) ), + cross_sales AS ( + SELECT 1 AS xx + FROM x, x AS d1, x AS d2, x AS d3, x AS d4, x AS d5, x AS d6, x AS d7, x AS d8, x AS d9 + WHERE x.number = d9.number + ) +SELECT xx FROM cross_sales WHERE xx = 2000; -- { serverError TOO_DEEP_PIPELINE } From 4a8454fae0c355abe66a9a55df4b990a2e5b626c Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 17 Jun 2022 17:59:15 +0000 Subject: [PATCH 024/139] increase max_pipeline_depth --- src/Core/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 5597d9076a4..436e57993a9 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -335,7 +335,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(UInt64, max_temporary_non_const_columns, 0, "", 0) \ \ M(UInt64, max_subquery_depth, 100, "", 0) \ - M(UInt64, max_pipeline_depth, 1000, "", 0) \ + M(UInt64, max_pipeline_depth, 10000, "", 0) \ M(UInt64, max_ast_depth, 1000, "Maximum depth of query syntax tree. Checked after parsing.", 0) \ M(UInt64, max_ast_elements, 50000, "Maximum size of query syntax tree in number of nodes. Checked after parsing.", 0) \ M(UInt64, max_expanded_ast_elements, 500000, "Maximum size of query syntax tree in number of nodes after expansion of aliases and the asterisk.", 0) \ From eae82d753044fdb586e6c6e2308b3fc9b37b79aa Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Jul 2022 01:48:00 +0300 Subject: [PATCH 025/139] Update src/Interpreters/InterpreterSelectQuery.cpp --- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 4cdd0428a0d..04551b9f098 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -454,7 +454,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (context->hasQueryContext()) { size_t & current_query_analyze_count = context->getQueryContext()->kitchen_sink.analyze_counter; - current_query_analyze_count++; + ++current_query_analyze_count; if (current_query_analyze_count >= settings.max_pipeline_depth) throw DB::Exception(ErrorCodes::TOO_DEEP_PIPELINE, "Query analyze overflow. Try to increase `max_pipeline_depth` or simplify the query" ); } From a0488dc901cad4d40fe66cae8e0ce7a64274b269 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 5 Jul 2022 01:48:05 +0300 Subject: [PATCH 026/139] Update src/Interpreters/Context.h --- src/Interpreters/Context.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 6e3e6de76c1..7a4c68f1007 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -365,7 +365,7 @@ public: OpenTelemetryTraceContext query_trace_context; /// Some counters for current query execution. - /// Most of them is workarounds and should be removed in the future. + /// Most of them are workarounds and should be removed in the future. struct KitchenSink { size_t analyze_counter = 0; From 9c348380f6aeb1bac342086db200a92e88eabbd1 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 5 Jul 2022 11:50:20 +0000 Subject: [PATCH 027/139] second analyze and max_pipeline_depth --- src/Interpreters/InterpreterSelectQuery.cpp | 22 ++++++++++++---- .../02337_join_analyze_stuck.reference | 26 +++++++++++++++++++ .../0_stateless/02337_join_analyze_stuck.sql | 17 +++++++++++- 3 files changed, 59 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 04551b9f098..019fc103f42 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -455,11 +455,10 @@ InterpreterSelectQuery::InterpreterSelectQuery( { size_t & current_query_analyze_count = context->getQueryContext()->kitchen_sink.analyze_counter; ++current_query_analyze_count; - if (current_query_analyze_count >= settings.max_pipeline_depth) - throw DB::Exception(ErrorCodes::TOO_DEEP_PIPELINE, "Query analyze overflow. Try to increase `max_pipeline_depth` or simplify the query" ); + if (settings.max_pipeline_depth && current_query_analyze_count >= settings.max_pipeline_depth) + throw DB::Exception(ErrorCodes::TOO_DEEP_PIPELINE, "Query analyze overflow. Try to increase `max_pipeline_depth` or simplify the query"); } - /// Allow push down and other optimizations for VIEW: replace with subquery and rewrite it. ASTPtr view_table; if (view) @@ -597,8 +596,19 @@ InterpreterSelectQuery::InterpreterSelectQuery( analyze(shouldMoveToPrewhere()); + bool analyze_overflow = false; + if (context->hasQueryContext()) + { + /// if we analyze to many times, give up analyzing a second time not to get stuck in the loop + /// the threshold is `max_pipeline_depth / 10` to disable the second analyze and `max_pipeline_depth` to cancel the query at all + const size_t & current_query_analyze_count = context->getQueryContext()->kitchen_sink.analyze_counter; + analyze_overflow = settings.max_pipeline_depth && current_query_analyze_count >= settings.max_pipeline_depth / 10; + } + bool need_analyze_again = false; - if (analysis_result.prewhere_constant_filter_description.always_false || analysis_result.prewhere_constant_filter_description.always_true) + + bool prewhere_always_const = analysis_result.prewhere_constant_filter_description.always_false || analysis_result.prewhere_constant_filter_description.always_true; + if (!analyze_overflow && prewhere_always_const) { if (analysis_result.prewhere_constant_filter_description.always_true) query.setExpression(ASTSelectQuery::Expression::PREWHERE, {}); @@ -606,7 +616,9 @@ InterpreterSelectQuery::InterpreterSelectQuery( query.setExpression(ASTSelectQuery::Expression::PREWHERE, std::make_shared(0u)); need_analyze_again = true; } - if (analysis_result.where_constant_filter_description.always_false || analysis_result.where_constant_filter_description.always_true) + + bool where_always_const = analysis_result.where_constant_filter_description.always_false || analysis_result.where_constant_filter_description.always_true; + if (!analyze_overflow && where_always_const) { if (analysis_result.where_constant_filter_description.always_true) query.setExpression(ASTSelectQuery::Expression::WHERE, {}); diff --git a/tests/queries/0_stateless/02337_join_analyze_stuck.reference b/tests/queries/0_stateless/02337_join_analyze_stuck.reference index e69de29bb2d..4c8ac5b69c5 100644 --- a/tests/queries/0_stateless/02337_join_analyze_stuck.reference +++ b/tests/queries/0_stateless/02337_join_analyze_stuck.reference @@ -0,0 +1,26 @@ +WITH + x AS + ( + SELECT number + FROM numbers(10) + ), + cross_sales AS + ( + SELECT 1 AS xx + FROM + x, + x AS d1, + x AS d2, + x AS d3, + x AS d4, + x AS d5, + x AS d6, + x AS d7, + x AS d8, + x AS d9 + WHERE x.number = d9.number + ) +SELECT xx +FROM +cross_sales +WHERE xx = 2000 diff --git a/tests/queries/0_stateless/02337_join_analyze_stuck.sql b/tests/queries/0_stateless/02337_join_analyze_stuck.sql index 8d8f604906f..aabb9e5c078 100644 --- a/tests/queries/0_stateless/02337_join_analyze_stuck.sql +++ b/tests/queries/0_stateless/02337_join_analyze_stuck.sql @@ -1,7 +1,10 @@ -- Tags: long + -- https://github.com/ClickHouse/ClickHouse/issues/21557 --- Not stuck at first, but proper fix should make able to process it + +SET max_pipeline_depth = 100; + EXPLAIN SYNTAX WITH x AS ( SELECT number FROM numbers(10) ), @@ -11,3 +14,15 @@ WITH WHERE x.number = d9.number ) SELECT xx FROM cross_sales WHERE xx = 2000; -- { serverError TOO_DEEP_PIPELINE } + +SET max_pipeline_depth = 10000; + +EXPLAIN SYNTAX +WITH + x AS ( SELECT number FROM numbers(10) ), + cross_sales AS ( + SELECT 1 AS xx + FROM x, x AS d1, x AS d2, x AS d3, x AS d4, x AS d5, x AS d6, x AS d7, x AS d8, x AS d9 + WHERE x.number = d9.number + ) +SELECT xx FROM cross_sales WHERE xx = 2000; From d681469275250a6c347c0437c13441b55149b6d2 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 6 Jul 2022 13:13:20 +0200 Subject: [PATCH 028/139] max_pipeline_depth only throws an error --- src/Interpreters/InterpreterSelectQuery.cpp | 15 ++--------- .../02337_join_analyze_stuck.reference | 26 ------------------- .../0_stateless/02337_join_analyze_stuck.sql | 15 +---------- 3 files changed, 3 insertions(+), 53 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 019fc103f42..516342476ae 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -596,19 +596,9 @@ InterpreterSelectQuery::InterpreterSelectQuery( analyze(shouldMoveToPrewhere()); - bool analyze_overflow = false; - if (context->hasQueryContext()) - { - /// if we analyze to many times, give up analyzing a second time not to get stuck in the loop - /// the threshold is `max_pipeline_depth / 10` to disable the second analyze and `max_pipeline_depth` to cancel the query at all - const size_t & current_query_analyze_count = context->getQueryContext()->kitchen_sink.analyze_counter; - analyze_overflow = settings.max_pipeline_depth && current_query_analyze_count >= settings.max_pipeline_depth / 10; - } - bool need_analyze_again = false; - bool prewhere_always_const = analysis_result.prewhere_constant_filter_description.always_false || analysis_result.prewhere_constant_filter_description.always_true; - if (!analyze_overflow && prewhere_always_const) + if (analysis_result.prewhere_constant_filter_description.always_false || analysis_result.prewhere_constant_filter_description.always_true) { if (analysis_result.prewhere_constant_filter_description.always_true) query.setExpression(ASTSelectQuery::Expression::PREWHERE, {}); @@ -617,8 +607,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( need_analyze_again = true; } - bool where_always_const = analysis_result.where_constant_filter_description.always_false || analysis_result.where_constant_filter_description.always_true; - if (!analyze_overflow && where_always_const) + if (analysis_result.where_constant_filter_description.always_false || analysis_result.where_constant_filter_description.always_true) { if (analysis_result.where_constant_filter_description.always_true) query.setExpression(ASTSelectQuery::Expression::WHERE, {}); diff --git a/tests/queries/0_stateless/02337_join_analyze_stuck.reference b/tests/queries/0_stateless/02337_join_analyze_stuck.reference index 4c8ac5b69c5..e69de29bb2d 100644 --- a/tests/queries/0_stateless/02337_join_analyze_stuck.reference +++ b/tests/queries/0_stateless/02337_join_analyze_stuck.reference @@ -1,26 +0,0 @@ -WITH - x AS - ( - SELECT number - FROM numbers(10) - ), - cross_sales AS - ( - SELECT 1 AS xx - FROM - x, - x AS d1, - x AS d2, - x AS d3, - x AS d4, - x AS d5, - x AS d6, - x AS d7, - x AS d8, - x AS d9 - WHERE x.number = d9.number - ) -SELECT xx -FROM -cross_sales -WHERE xx = 2000 diff --git a/tests/queries/0_stateless/02337_join_analyze_stuck.sql b/tests/queries/0_stateless/02337_join_analyze_stuck.sql index aabb9e5c078..9bdc418f028 100644 --- a/tests/queries/0_stateless/02337_join_analyze_stuck.sql +++ b/tests/queries/0_stateless/02337_join_analyze_stuck.sql @@ -1,9 +1,8 @@ -- Tags: long - -- https://github.com/ClickHouse/ClickHouse/issues/21557 -SET max_pipeline_depth = 100; +SET max_pipeline_depth = 1000; EXPLAIN SYNTAX WITH @@ -14,15 +13,3 @@ WITH WHERE x.number = d9.number ) SELECT xx FROM cross_sales WHERE xx = 2000; -- { serverError TOO_DEEP_PIPELINE } - -SET max_pipeline_depth = 10000; - -EXPLAIN SYNTAX -WITH - x AS ( SELECT number FROM numbers(10) ), - cross_sales AS ( - SELECT 1 AS xx - FROM x, x AS d1, x AS d2, x AS d3, x AS d4, x AS d5, x AS d6, x AS d7, x AS d8, x AS d9 - WHERE x.number = d9.number - ) -SELECT xx FROM cross_sales WHERE xx = 2000; From 79c185d22bf99869cbc284c5446556786bbed906 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 7 Jul 2022 17:09:32 +0200 Subject: [PATCH 029/139] make current_query_analyze_count atomic --- src/Interpreters/Context.h | 14 +++++++++++++- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 7a4c68f1007..560571f74e9 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -368,7 +368,19 @@ public: /// Most of them are workarounds and should be removed in the future. struct KitchenSink { - size_t analyze_counter = 0; + std::atomic analyze_counter = 0; + + KitchenSink() = default; + + KitchenSink(const KitchenSink & rhs) + : analyze_counter(rhs.analyze_counter.load()) + {} + + KitchenSink & operator=(const KitchenSink & rhs) + { + analyze_counter = rhs.analyze_counter.load(); + return *this; + } }; KitchenSink kitchen_sink; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 516342476ae..cab9208f66b 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -453,7 +453,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( { if (context->hasQueryContext()) { - size_t & current_query_analyze_count = context->getQueryContext()->kitchen_sink.analyze_counter; + std::atomic & current_query_analyze_count = context->getQueryContext()->kitchen_sink.analyze_counter; ++current_query_analyze_count; if (settings.max_pipeline_depth && current_query_analyze_count >= settings.max_pipeline_depth) throw DB::Exception(ErrorCodes::TOO_DEEP_PIPELINE, "Query analyze overflow. Try to increase `max_pipeline_depth` or simplify the query"); From 7e578a64e803403dbb0ccbb93536a6ebe21037ff Mon Sep 17 00:00:00 2001 From: Dan Roscigno Date: Wed, 13 Jul 2022 08:02:09 -0400 Subject: [PATCH 030/139] Update performance-test.md The manual process does not work as the prepared partitions are not usable by the current version of ClickHouse. --- docs/en/operations/performance-test.md | 59 +------------------------- 1 file changed, 2 insertions(+), 57 deletions(-) diff --git a/docs/en/operations/performance-test.md b/docs/en/operations/performance-test.md index 5b2a772a009..09c5f5b023e 100644 --- a/docs/en/operations/performance-test.md +++ b/docs/en/operations/performance-test.md @@ -5,12 +5,12 @@ sidebar_label: Testing Hardware # How to Test Your Hardware with ClickHouse -You can run basic ClickHouse performance test on any server without installation of ClickHouse packages. +You can run a basic ClickHouse performance test on any server without installation of ClickHouse packages. ## Automated Run -You can run benchmark with a single script. +You can run the benchmark with a single script. 1. Download the script. ``` @@ -26,58 +26,3 @@ chmod a+x ./hardware.sh 3. Copy the output and send it to feedback@clickhouse.com All the results are published here: https://clickhouse.com/benchmark/hardware/ - - -## Manual Run - -Alternatively you can perform benchmark in the following steps. - -1. ssh to the server and download the binary with wget: -```bash -# For amd64: -wget https://builds.clickhouse.com/master/amd64/clickhouse -# For aarch64: -wget https://builds.clickhouse.com/master/aarch64/clickhouse -# For powerpc64le: -wget https://builds.clickhouse.com/master/powerpc64le/clickhouse -# For freebsd: -wget https://builds.clickhouse.com/master/freebsd/clickhouse -# For freebsd-aarch64: -wget https://builds.clickhouse.com/master/freebsd-aarch64/clickhouse -# For freebsd-powerpc64le: -wget https://builds.clickhouse.com/master/freebsd-powerpc64le/clickhouse -# For macos: -wget https://builds.clickhouse.com/master/macos/clickhouse -# For macos-aarch64: -wget https://builds.clickhouse.com/master/macos-aarch64/clickhouse -# Then do: -chmod a+x clickhouse -``` -2. Download benchmark files: -```bash -wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/hardware/benchmark-new.sh -chmod a+x benchmark-new.sh -wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/hardware/queries.sql -``` -3. Download the [web analytics dataset](../getting-started/example-datasets/metrica.md) (“hits” table containing 100 million rows). -```bash -wget https://datasets.clickhouse.com/hits/partitions/hits_100m_obfuscated_v1.tar.xz -tar xvf hits_100m_obfuscated_v1.tar.xz -C . -mv hits_100m_obfuscated_v1/* . -``` -4. Run the server: -```bash -./clickhouse server -``` -5. Check the data: ssh to the server in another terminal -```bash -./clickhouse client --query "SELECT count() FROM hits_100m_obfuscated" -100000000 -``` -6. Run the benchmark: -```bash -./benchmark-new.sh hits_100m_obfuscated -``` -7. Send the numbers and the info about your hardware configuration to feedback@clickhouse.com - -All the results are published here: https://clickhouse.com/benchmark/hardware/ From 4f25a08b7c7f69d62d8b577d92036cea6d66657c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 13 Jul 2022 15:58:02 +0300 Subject: [PATCH 031/139] Add connection info for Distributed sends log message Signed-off-by: Azat Khuzhin --- src/Storages/Distributed/DirectoryMonitor.cpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 5e9da48fc68..6d9e52486e2 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -619,11 +619,14 @@ void StorageDistributedDirectoryMonitor::processFile(const std::string & file_pa ReadBufferFromFile in(file_path); const auto & distributed_header = readDistributedHeader(in, log); - LOG_DEBUG(log, "Started processing `{}` ({} rows, {} bytes)", file_path, + auto connection = pool->get(timeouts, &distributed_header.insert_settings); + + LOG_DEBUG(log, "Sending `{}` to {} ({} rows, {} bytes)", + file_path, + connection->getDescription(), formatReadableQuantity(distributed_header.rows), formatReadableSizeWithBinarySuffix(distributed_header.bytes)); - auto connection = pool->get(timeouts, &distributed_header.insert_settings); RemoteInserter remote{*connection, timeouts, distributed_header.insert_query, distributed_header.insert_settings, @@ -718,10 +721,6 @@ struct StorageDistributedDirectoryMonitor::Batch Stopwatch watch; - LOG_DEBUG(parent.log, "Sending a batch of {} files ({} rows, {} bytes).", file_indices.size(), - formatReadableQuantity(total_rows), - formatReadableSizeWithBinarySuffix(total_bytes)); - if (!recovered) { /// For deduplication in Replicated tables to work, in case of error @@ -750,6 +749,12 @@ struct StorageDistributedDirectoryMonitor::Batch auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(parent.storage.getContext()->getSettingsRef()); auto connection = parent.pool->get(timeouts); + LOG_DEBUG(parent.log, "Sending a batch of {} files to {} ({} rows, {} bytes).", + file_indices.size(), + connection->getDescription(), + formatReadableQuantity(total_rows), + formatReadableSizeWithBinarySuffix(total_bytes)); + bool batch_broken = false; bool batch_marked_as_broken = false; try From 9ca2935d728ff25d852d7cf74140277e8cfe0356 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Thu, 14 Jul 2022 09:49:30 +0000 Subject: [PATCH 032/139] Add default argument to function file --- docs/en/sql-reference/functions/files.md | 3 +- docs/ru/sql-reference/functions/files.md | 3 +- src/Functions/FunctionFile.cpp | 79 +++++++++++++++++-- .../02357_file_default_value.reference | 2 + .../0_stateless/02357_file_default_value.sql | 3 + 5 files changed, 80 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/02357_file_default_value.reference create mode 100644 tests/queries/0_stateless/02357_file_default_value.sql diff --git a/docs/en/sql-reference/functions/files.md b/docs/en/sql-reference/functions/files.md index fc2f8a0e050..b17981ed0e0 100644 --- a/docs/en/sql-reference/functions/files.md +++ b/docs/en/sql-reference/functions/files.md @@ -12,12 +12,13 @@ Reads file as a String. The file content is not parsed, so any information is re **Syntax** ``` sql -file(path) +file(path[, default]) ``` **Arguments** - `path` — The relative path to the file from [user_files_path](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Path to file support following wildcards: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc', 'def'` — strings. +- `default` — The value that will be returned in the case when a file does not exist or cannot be accessed. Data types supported: [String](../../sql-reference/data-types/string.md) and [NULL](../../sql-reference/syntax.md#null-literal). **Example** diff --git a/docs/ru/sql-reference/functions/files.md b/docs/ru/sql-reference/functions/files.md index 1dae9478425..ef4369870ed 100644 --- a/docs/ru/sql-reference/functions/files.md +++ b/docs/ru/sql-reference/functions/files.md @@ -12,12 +12,13 @@ sidebar_label: "Функции для работы с файлами" **Синтаксис** ``` sql -file(path) +file(path[, default]) ``` **Аргументы** - `path` — относительный путь до файла от [user_files_path](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Путь к файлу может включать следующие символы подстановки и шаблоны: `*`, `?`, `{abc,def}` и `{N..M}`, где `N`, `M` — числа, `'abc', 'def'` — строки. +- `default` — Значение возвращаемое в случае, если указанный файл не существует. Поддерживаемые типы данных: [String](../../sql-reference/data-types/string.md) и [NULL](../../sql-reference/syntax.md#null-literal). **Примеры** diff --git a/src/Functions/FunctionFile.cpp b/src/Functions/FunctionFile.cpp index 01314b52119..b0225c72c26 100644 --- a/src/Functions/FunctionFile.cpp +++ b/src/Functions/FunctionFile.cpp @@ -1,7 +1,10 @@ +#include #include +#include #include #include #include +#include #include #include #include @@ -19,6 +22,7 @@ namespace ErrorCodes { extern const int ILLEGAL_COLUMN; extern const int NOT_IMPLEMENTED; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int DATABASE_ACCESS_DENIED; } @@ -30,21 +34,39 @@ public: static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } explicit FunctionFile(ContextPtr context_) : WithContext(context_) {} + bool isVariadic() const override { return true; } String getName() const override { return name; } - size_t getNumberOfArguments() const override { return 1; } + size_t getNumberOfArguments() const override { return 0; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { + if (arguments.size() < 1 || arguments.size() > 2) + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Number of arguments for function {} doesn't match: passed {}, should be 1 or 2", + getName(), toString(arguments.size())); + if (!isString(arguments[0].type)) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "{} is only implemented for type String", getName()); + if (arguments.size() == 2) + { + if (arguments[1].type->onlyNull()) + return makeNullable(std::make_shared()); + + if (!isString(arguments[1].type)) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "{} only accepts String or Null as second argument", getName()); + } + return std::make_shared(); } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } + bool useDefaultImplementationForConstants() const override { return true; } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { const ColumnPtr column = arguments[0].column; const ColumnString * column_src = checkAndGetColumn(column.get()); @@ -53,6 +75,31 @@ public: fmt::format("Illegal column {} of argument of function {}", arguments[0].column->getName(), getName()), ErrorCodes::ILLEGAL_COLUMN); + String default_result; + + ColumnUInt8::MutablePtr col_null_map_to; + ColumnUInt8::Container * vec_null_map_to [[maybe_unused]] = nullptr; + + if (arguments.size() == 2) + { + if (result_type->isNullable()) + { + col_null_map_to = ColumnUInt8::create(input_rows_count, false); + vec_null_map_to = &col_null_map_to->getData(); + } + else + { + const auto & default_column = arguments[1].column; + const ColumnConst * default_col = checkAndGetColumn(default_column.get()); + + if (!default_col) + throw Exception( + "Illegal column " + arguments[1].column->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); + + default_result = default_col->getValue(); + } + } + auto result = ColumnString::create(); auto & res_chars = result->getChars(); auto & res_offsets = result->getOffsets(); @@ -77,18 +124,34 @@ public: /// Otherwise it will not allow to work with symlinks in `user_files_path` directory. file_path = fs::absolute(file_path).lexically_normal(); - if (need_check && file_path.string().find(user_files_absolute_path_string) != 0) - throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "File is not inside {}", user_files_absolute_path.string()); + try + { + if (need_check && file_path.string().find(user_files_absolute_path_string) != 0) + throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "File is not inside {}", user_files_absolute_path.string()); - ReadBufferFromFile in(file_path); - WriteBufferFromVector out(res_chars, AppendModeTag{}); - copyData(in, out); - out.finalize(); + ReadBufferFromFile in(file_path); + WriteBufferFromVector out(res_chars, AppendModeTag{}); + copyData(in, out); + out.finalize(); + } + catch (...) + { + if (arguments.size() == 1) + throw; + + if (vec_null_map_to) + (*vec_null_map_to)[row] = true; + else + res_chars.insert(default_result.data(), default_result.data() + default_result.size()); + } res_chars.push_back(0); res_offsets[row] = res_chars.size(); } + if (vec_null_map_to) + return ColumnNullable::create(std::move(result), std::move(col_null_map_to)); + return result; } }; diff --git a/tests/queries/0_stateless/02357_file_default_value.reference b/tests/queries/0_stateless/02357_file_default_value.reference new file mode 100644 index 00000000000..7c0c4634e42 --- /dev/null +++ b/tests/queries/0_stateless/02357_file_default_value.reference @@ -0,0 +1,2 @@ +default +\N diff --git a/tests/queries/0_stateless/02357_file_default_value.sql b/tests/queries/0_stateless/02357_file_default_value.sql new file mode 100644 index 00000000000..008fc4edb1f --- /dev/null +++ b/tests/queries/0_stateless/02357_file_default_value.sql @@ -0,0 +1,3 @@ +SELECT file('nonexistent.txt'); -- { serverError 107 } +SELECT file('nonexistent.txt', 'default'); +SELECT file('nonexistent.txt', NULL); From a068c397dfd7943359a8b554566c3c70b78baf8d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 14 Jul 2022 15:10:14 +0300 Subject: [PATCH 033/139] Do not report "Failed communicating with" on and on for parts exchange For parts exchange right now an error message in stored in the HTTPSession, via HTTPSession::attachSessionData(), and for each request this HTTPSession::sessionData() is checked and if not empty printed into the log. However it should be reported only once, otherwise you will get this message on and on, even for different tables. Here is an example of such messages: 2022.07.13 07:56:09.342997 [ 683 ] {} test_juq8qk.rmt2 (ede90518-4710-48bb-ab43-caf0b1b157f4): auto DB::StorageReplicatedMergeTree::processQueueEntry(ReplicatedMergeTreeQueue::SelectedEntryPtr)::(anonymous class)::operator()(DB::StorageReplicatedMergeTree::LogEntryPtr &) const: Code: 86. DB::Exception: Received error from remote server /?endpoint=DataPartsExchange%3A%2Ftest%2F01165%2Ftest_juq8qk%2Frmt%2Freplicas%2F1&part=206--1406300905-20220713-1657673769_0_0_0&client_protocol_version=7&compress=false&remote_fs_metadata=s3. HTTP status code: 500 Internal Server Error, body: Code: 236. DB::Exception: Transferring part to replica was cancelled. (ABORTED) (version 22.7.1.1781 (official build)). (RECEIVED_ERROR_FROM_REMOTE_IO_SERVER), Stack trace (when copying this message, always include the lines below): # this is the time when this message is written ^ 2022.07.13 07:56:10.528554 [ 814 ] {} DatabaseCatalog: Removing metadata /var/lib/clickhouse/metadata_dropped/test_juq8qk.rmt2.ede90518-4710-48bb-ab43-caf0b1b157f4.sql of dropped table test_juq8qk.rmt2 (ede90518-4710-48bb-ab43-caf0b1b157f4) # now this table had been removed ^ 2022.07.13 07:56:27.442003 [ 683 ] {} test_1orbeb.mutations_and_quorum2 (bc811afd-1f57-4f9c-a04d-9e1e1b60e891): Fetching part 201901_0_0_0 from /clickhouse/tables/test_1orbeb/test_01090/mutations_and_quorum/replicas/1 # here fetch part is scheduled for another table ^ 2022.07.13 07:56:27.442213 [ 683 ] {} HTTPCommon: Failed communicating with 250c6af4615d with error 'Received error from remote server /?endpoint=DataPartsExchange%3A%2Ftest%2F01165%2Ftest_juq8qk%2Frmt%2Freplicas%2F1&part=255--1372061156-20220713-1657673769_0_0_0&client_protocol_version=7&compress=false&remote_fs_metadata=s3. HTTP status code: 500 Internal Server Error, body: Code: 236. DB::Exception: Transferring part to replica was cancelled. (ABORTED) (version 22.7.1.1781 (official build))' will try to reconnect session # however it still reports an error for the already removed table test_juq8qk.rmt 2022.07.13 07:56:27.442246 [ 683 ] {} ReadWriteBufferFromHTTP: Sending request to http://250c6af4615d:9009/?endpoint=DataPartsExchange%3A%2Fclickhouse%2Ftables%2Ftest_1orbeb% # but this is just a noisy message and it still doing the job correctly ^ Signed-off-by: Azat Khuzhin --- src/IO/HTTPCommon.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/IO/HTTPCommon.cpp b/src/IO/HTTPCommon.cpp index b31ee08eaf5..8d2f7b4c39b 100644 --- a/src/IO/HTTPCommon.cpp +++ b/src/IO/HTTPCommon.cpp @@ -255,10 +255,13 @@ namespace { session->reset(); session->setHost(ip); - session->attachSessionData({}); } } } + /// Reset the message, once it has been printed, + /// otherwise you will get report for failed parts on and on, + /// even for different tables (since they uses the same session). + session->attachSessionData({}); } setTimeouts(*session, timeouts); From 6f5275d9bbe3b1dc062801e45bcb8bc597d0aa9a Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Fri, 15 Jul 2022 10:08:14 +0000 Subject: [PATCH 034/139] Fix build --- src/Functions/FunctionFile.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/FunctionFile.cpp b/src/Functions/FunctionFile.cpp index b0225c72c26..ba0a713a12c 100644 --- a/src/Functions/FunctionFile.cpp +++ b/src/Functions/FunctionFile.cpp @@ -41,7 +41,7 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - if (arguments.size() < 1 || arguments.size() > 2) + if (arguments.empty() || arguments.size() > 2) throw Exception( ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Number of arguments for function {} doesn't match: passed {}, should be 1 or 2", From 728219e640375c9e6f7179af0715d107fc398f17 Mon Sep 17 00:00:00 2001 From: zvonand Date: Sat, 16 Jul 2022 22:16:19 +0200 Subject: [PATCH 035/139] updated due to review --- .../functions/date-time-functions.md | 24 +++++----- .../functions/date-time-functions.md | 20 +++++---- src/Functions/timeSlots.cpp | 45 ++++++++++--------- 3 files changed, 47 insertions(+), 42 deletions(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index f2500e6eedc..8a595b63bdf 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -951,24 +951,26 @@ SELECT ## timeSlots(StartTime, Duration,\[, Size\]) For a time interval starting at ‘StartTime’ and continuing for ‘Duration’ seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the ‘Size’ in seconds. ‘Size’ is an optional parameter set to 1800 (30 minutes) by default. -This is necessary for searching for pageviews in the corresponding session. -Accepts DateTime and DateTime64. For DateTime, `Duration` and `Size` arguments must be `UInt32`. For DateTime64 they must be `Decimal64`. +This is necessary, for example, when searching for pageviews in the corresponding session. +Accepts DateTime and DateTime64 as ’StartTime’ argument. For DateTime, ’Duration’ and ’Size’ arguments must be `UInt32`. For ’DateTime64’ they must be `Decimal64`. +Returns an array of DateTime/DateTime64 (return type matches the type of ’StartTime’). For DateTime64, the return value's scale can differ from the scale of ’StartTime’ --- the highest scale among all given arguments is taken. + Example: ```sql -SELECT timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0)); -SELECT timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299); SELECT timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600)); -``` +SELECT timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299); +SELECT timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0)); +``` ``` text -┌─timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0))─┐ -│ ['1980-12-12 20:56:13.0000','1980-12-12 21:01:12.0000','1980-12-12 21:06:11.0000'] │ -└───────────────────────────────────────────────────────────────────────────────────────────────────────────┘ -┌─timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299)─┐ -│ ['1980-12-12 20:56:13','1980-12-12 21:01:12','1980-12-12 21:06:11'] │ -└─────────────────────────────────────────────────────────────────────────┘ ┌─timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600))─┐ │ ['2012-01-01 12:00:00','2012-01-01 12:30:00'] │ └─────────────────────────────────────────────────────────────┘ +┌─timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299)─┐ +│ ['1980-12-12 20:56:13','1980-12-12 21:01:12','1980-12-12 21:06:11'] │ +└─────────────────────────────────────────────────────────────────────────┘ +┌─timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0))─┐ +│ ['1980-12-12 20:56:13.0000','1980-12-12 21:01:12.0000','1980-12-12 21:06:11.0000'] │ +└───────────────────────────────────────────────────────────────────────────────────────────────────────────┘ ``` ## formatDateTime diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index b38e1ddf918..5a214fe2664 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -949,23 +949,25 @@ SELECT now('Europe/Moscow'); ## timeSlots(StartTime, Duration,\[, Size\]) {#timeslotsstarttime-duration-size} Для интервала, начинающегося в `StartTime` и длящегося `Duration` секунд, возвращает массив моментов времени, кратных `Size`. Параметр `Size` указывать необязательно, по умолчанию он равен 1800 секундам (30 минутам) - необязательный параметр. Данная функция может использоваться, например, для анализа количества просмотров страницы за соответствующую сессию. -Аргумент `StartTime` может иметь тип DateTime или DateTime64. В случае, если используется DateTime, аргументы `Duration` и `Size` должны иметь тип `UInt32`; Для DateTime64 они должны быть типа `Decimal64`. +Аргумент `StartTime` может иметь тип `DateTime` или `DateTime64`. В случае, если используется `DateTime`, аргументы `Duration` и `Size` должны иметь тип `UInt32`; Для DateTime64 они должны быть типа `Decimal64`. +Возвращает массив DateTime/DateTime64 (тип будет совпадать с типом параметра ’StartTime’). Для DateTime64 масштаб(scale) возвращаемой величины может отличаться от масштаба фргумента ’StartTime’ --- результат будет иметь наибольший масштаб среди всех данных аргументов. + Пример использования: ```sql -SELECT timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0)); -SELECT timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299); SELECT timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600)); +SELECT timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299); +SELECT timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0)); ``` ``` text -┌─timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0))─┐ -│ ['1980-12-12 20:56:13.0000','1980-12-12 21:01:12.0000','1980-12-12 21:06:11.0000'] │ -└───────────────────────────────────────────────────────────────────────────────────────────────────────────┘ -┌─timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299)─┐ -│ ['1980-12-12 20:56:13','1980-12-12 21:01:12','1980-12-12 21:06:11'] │ -└─────────────────────────────────────────────────────────────────────────┘ ┌─timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600))─┐ │ ['2012-01-01 12:00:00','2012-01-01 12:30:00'] │ └─────────────────────────────────────────────────────────────┘ +┌─timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299)─┐ +│ ['1980-12-12 20:56:13','1980-12-12 21:01:12','1980-12-12 21:06:11'] │ +└─────────────────────────────────────────────────────────────────────────┘ +┌─timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0))─┐ +│ ['1980-12-12 20:56:13.0000','1980-12-12 21:01:12.0000','1980-12-12 21:06:11.0000'] │ +└───────────────────────────────────────────────────────────────────────────────────────────────────────────┘ ``` ## toYYYYMM diff --git a/src/Functions/timeSlots.cpp b/src/Functions/timeSlots.cpp index 663cc52eb6b..39a784a70ba 100644 --- a/src/Functions/timeSlots.cpp +++ b/src/Functions/timeSlots.cpp @@ -116,8 +116,7 @@ struct TimeSlotsImpl result_values.reserve(size); /// Modify all units to have same scale - UInt16 max_scale = (dt_scale > duration_scale) ? dt_scale : duration_scale; - max_scale = (time_slot_scale > max_scale) ? time_slot_scale : max_scale; + UInt16 max_scale = std::max({dt_scale, duration_scale, time_slot_scale}); Int64 dt_multiplier = DecimalUtils::scaleMultiplier(max_scale - dt_scale); Int64 dur_multiplier = DecimalUtils::scaleMultiplier(max_scale - duration_scale); @@ -146,8 +145,7 @@ struct TimeSlotsImpl result_values.reserve(size); /// Modify all units to have same scale - UInt16 max_scale = (dt_scale > duration_scale) ? dt_scale : duration_scale; - max_scale = (time_slot_scale > max_scale) ? time_slot_scale : max_scale; + UInt16 max_scale = std::max({dt_scale, duration_scale, time_slot_scale}); Int64 dt_multiplier = DecimalUtils::scaleMultiplier(max_scale - dt_scale); Int64 dur_multiplier = DecimalUtils::scaleMultiplier(max_scale - duration_scale); @@ -177,8 +175,7 @@ struct TimeSlotsImpl result_values.reserve(size); /// Modify all units to have same scale - UInt16 max_scale = (dt_scale > duration_scale) ? dt_scale : duration_scale; - max_scale = (time_slot_scale > max_scale) ? time_slot_scale : max_scale; + UInt16 max_scale = std::max({dt_scale, duration_scale, time_slot_scale}); Int64 dt_multiplier = DecimalUtils::scaleMultiplier(max_scale - dt_scale); Int64 dur_multiplier = DecimalUtils::scaleMultiplier(max_scale - duration_scale); @@ -224,7 +221,7 @@ public: + toString(arguments.size()) + ", should be 2 or 3", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - if (isDateTime(arguments[0].type)) + if (WhichDataType(arguments[0].type).isDateTime()) { if (!WhichDataType(arguments[1].type).isUInt32()) throw Exception( @@ -236,7 +233,7 @@ public: "Illegal type " + arguments[2].type->getName() + " of third argument of function " + getName() + ". Must be UInt32 when first argument is DateTime.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } - else if (isDateTime64(arguments[0].type)) + else if (WhichDataType(arguments[0].type).isDateTime64()) { if (!WhichDataType(arguments[1].type).isDecimal64()) throw Exception( @@ -260,17 +257,17 @@ public: } else { - auto dt64_scale = assert_cast(*arguments[0].type).getScale(); + auto start_time_scale = assert_cast(*arguments[0].type).getScale(); auto duration_scale = assert_cast &>(*arguments[1].type).getScale(); return std::make_shared( - std::make_shared(std::max(dt64_scale, duration_scale), extractTimeZoneNameFromFunctionArguments(arguments, 3, 0))); + std::make_shared(std::max(start_time_scale, duration_scale), extractTimeZoneNameFromFunctionArguments(arguments, 3, 0))); } } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override { - if (isDateTime(arguments[0].type)) + if (WhichDataType(arguments[0].type).isDateTime()) { UInt32 time_slot_size = 1800; if (arguments.size() == 3) @@ -308,8 +305,9 @@ public: return res; } } - else if (isDateTime64(arguments[0].type)) + else { + assert(WhichDataType(arguments[0].type).isDateTime64()); Decimal64 time_slot_size = Decimal64(1800); UInt16 time_slot_scale = 0; if (arguments.size() == 3) @@ -323,33 +321,36 @@ public: time_slot_scale = assert_cast *>(arguments[2].type.get())->getScale(); } - const auto * dt64_starts = checkAndGetColumn(arguments[0].column.get()); - const auto * dt64_const_starts = checkAndGetColumnConst(arguments[0].column.get()); + const auto * starts = checkAndGetColumn(arguments[0].column.get()); + const auto * const_starts = checkAndGetColumnConst(arguments[0].column.get()); const auto * durations = checkAndGetColumn>(arguments[1].column.get()); const auto * const_durations = checkAndGetColumnConst>(arguments[1].column.get()); - const auto dt64_scale = assert_cast(arguments[0].type.get())->getScale(); + const auto start_time_scale = assert_cast(arguments[0].type.get())->getScale(); const auto duration_scale = assert_cast *>(arguments[1].type.get())->getScale(); - auto res = ColumnArray::create(DataTypeDateTime64(dt64_scale).createColumn()); + auto res = ColumnArray::create(DataTypeDateTime64(start_time_scale).createColumn()); DataTypeDateTime64::ColumnType::Container & res_values = typeid_cast(res->getData()).getData(); - if (dt64_starts && durations) + if (starts && durations) { - TimeSlotsImpl::vectorVector(dt64_starts->getData(), durations->getData(), time_slot_size, res_values, res->getOffsets(), dt64_scale, duration_scale, time_slot_scale); + TimeSlotsImpl::vectorVector(starts->getData(), durations->getData(), time_slot_size, res_values, res->getOffsets(), + start_time_scale, duration_scale, time_slot_scale); return res; } - else if (dt64_starts && const_durations) + else if (starts && const_durations) { TimeSlotsImpl::vectorConstant( - dt64_starts->getData(), const_durations->getValue(), time_slot_size, res_values, res->getOffsets(), dt64_scale, duration_scale, time_slot_scale); + starts->getData(), const_durations->getValue(), time_slot_size, res_values, res->getOffsets(), + start_time_scale, duration_scale, time_slot_scale); return res; } - else if (dt64_const_starts && durations) + else if (const_starts && durations) { TimeSlotsImpl::constantVector( - dt64_const_starts->getValue(), durations->getData(), time_slot_size, res_values, res->getOffsets(), dt64_scale, duration_scale, time_slot_scale); + const_starts->getValue(), durations->getData(), time_slot_size, res_values, res->getOffsets(), + start_time_scale, duration_scale, time_slot_scale); return res; } } From e6437f46b1f908336fb79bd5fe05b026f09f1ca2 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 16 Jul 2022 22:23:49 +0200 Subject: [PATCH 036/139] Fix --- src/Interpreters/ExpressionAnalyzer.cpp | 76 +----------------- src/Interpreters/TreeRewriter.cpp | 20 +++++ .../replaceForPositionalArguments.cpp | 78 +++++++++++++++++++ .../replaceForPositionalArguments.h | 8 ++ .../02006_test_positional_arguments.reference | 4 + .../02006_test_positional_arguments.sql | 4 + 6 files changed, 115 insertions(+), 75 deletions(-) create mode 100644 src/Interpreters/replaceForPositionalArguments.cpp create mode 100644 src/Interpreters/replaceForPositionalArguments.h diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 23258c60099..95735745c14 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include @@ -110,79 +111,6 @@ bool allowEarlyConstantFolding(const ActionsDAG & actions, const Settings & sett return true; } -bool checkPositionalArguments(ASTPtr & argument, const ASTSelectQuery * select_query, ASTSelectQuery::Expression expression) -{ - auto columns = select_query->select()->children; - - const auto * expr_with_alias = dynamic_cast(argument.get()); - if (expr_with_alias && !expr_with_alias->alias.empty()) - return false; - - const auto * ast_literal = typeid_cast(argument.get()); - if (!ast_literal) - return false; - - auto which = ast_literal->value.getType(); - if (which != Field::Types::UInt64) - return false; - - auto pos = ast_literal->value.get(); - if (!pos || pos > columns.size()) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Positional argument out of bounds: {} (exprected in range [1, {}]", - pos, columns.size()); - - const auto & column = columns[--pos]; - if (typeid_cast(column.get()) || typeid_cast(column.get())) - { - argument = column->clone(); - } - else if (typeid_cast(column.get())) - { - std::function throw_if_aggregate_function = [&](ASTPtr node) - { - if (const auto * function = typeid_cast(node.get())) - { - auto is_aggregate_function = AggregateUtils::isAggregateFunction(*function); - if (is_aggregate_function) - { - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal value (aggregate function) for positional argument in {}", - ASTSelectQuery::expressionToString(expression)); - } - else - { - if (function->arguments) - { - for (const auto & arg : function->arguments->children) - throw_if_aggregate_function(arg); - } - } - } - }; - - if (expression == ASTSelectQuery::Expression::GROUP_BY) - throw_if_aggregate_function(column); - - argument = column->clone(); - } - else - { - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal value for positional argument in {}", - ASTSelectQuery::expressionToString(expression)); - } - - return true; -} - -void replaceForPositionalArguments(ASTPtr & argument, const ASTSelectQuery * select_query, ASTSelectQuery::Expression expression) -{ - auto argument_with_replacement = argument->clone(); - if (checkPositionalArguments(argument_with_replacement, select_query, expression)) - argument = argument_with_replacement; -} - } bool sanitizeBlock(Block & block, bool throw_if_cannot_create_column) @@ -1556,9 +1484,7 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChai throw Exception("Bad ORDER BY expression AST", ErrorCodes::UNKNOWN_TYPE_OF_AST_NODE); if (getContext()->getSettingsRef().enable_positional_arguments) - { replaceForPositionalArguments(ast->children.at(0), select_query, ASTSelectQuery::Expression::ORDER_BY); - } } getRootActions(select_query->orderBy(), only_types, step.actions()); diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index b389c3eb705..ca4849717fb 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -1237,6 +1238,25 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( source_columns_set, right_table.table.getQualifiedNamePrefix()); } + if (getContext()->getSettingsRef().enable_positional_arguments) + { + if (select_query->groupBy()) + { + for (auto & expr : select_query->groupBy()->children) + replaceForPositionalArguments(expr, select_query, ASTSelectQuery::Expression::GROUP_BY); + } + if (select_query->orderBy()) + { + for (auto & expr : select_query->orderBy()->children) + replaceForPositionalArguments(expr, select_query, ASTSelectQuery::Expression::ORDER_BY); + } + if (select_query->limitBy()) + { + for (auto & expr : select_query->limitBy()->children) + replaceForPositionalArguments(expr, select_query, ASTSelectQuery::Expression::LIMIT_BY); + } + } + translateQualifiedNames(query, *select_query, source_columns_set, tables_with_columns); /// Optimizes logical expressions. diff --git a/src/Interpreters/replaceForPositionalArguments.cpp b/src/Interpreters/replaceForPositionalArguments.cpp new file mode 100644 index 00000000000..deac5801d7a --- /dev/null +++ b/src/Interpreters/replaceForPositionalArguments.cpp @@ -0,0 +1,78 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ + +bool replaceForPositionalArguments(ASTPtr & argument, const ASTSelectQuery * select_query, ASTSelectQuery::Expression expression) +{ + auto columns = select_query->select()->children; + + const auto * expr_with_alias = dynamic_cast(argument.get()); + + if (expr_with_alias && !expr_with_alias->alias.empty()) + return false; + + const auto * ast_literal = typeid_cast(argument.get()); + if (!ast_literal) + return false; + + auto which = ast_literal->value.getType(); + if (which != Field::Types::UInt64) + return false; + + auto pos = ast_literal->value.get(); + if (!pos || pos > columns.size()) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Positional argument out of bounds: {} (exprected in range [1, {}]", + pos, columns.size()); + + const auto & column = columns[--pos]; + if (typeid_cast(column.get()) || typeid_cast(column.get())) + { + argument = column->clone(); + } + else if (typeid_cast(column.get())) + { + std::function throw_if_aggregate_function = [&](ASTPtr node) + { + if (const auto * function = typeid_cast(node.get())) + { + auto is_aggregate_function = AggregateUtils::isAggregateFunction(*function); + if (is_aggregate_function) + { + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal value (aggregate function) for positional argument in {}", + ASTSelectQuery::expressionToString(expression)); + } + else + { + if (function->arguments) + { + for (const auto & arg : function->arguments->children) + throw_if_aggregate_function(arg); + } + } + } + }; + + if (expression == ASTSelectQuery::Expression::GROUP_BY) + throw_if_aggregate_function(column); + + argument = column->clone(); + } + else + { + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal value for positional argument in {}", + ASTSelectQuery::expressionToString(expression)); + } + + return true; +} + +} diff --git a/src/Interpreters/replaceForPositionalArguments.h b/src/Interpreters/replaceForPositionalArguments.h new file mode 100644 index 00000000000..9906978c561 --- /dev/null +++ b/src/Interpreters/replaceForPositionalArguments.h @@ -0,0 +1,8 @@ +#include + +namespace DB +{ + +bool replaceForPositionalArguments(ASTPtr & argument, const ASTSelectQuery * select_query, ASTSelectQuery::Expression expression); + +} diff --git a/tests/queries/0_stateless/02006_test_positional_arguments.reference b/tests/queries/0_stateless/02006_test_positional_arguments.reference index f86a1ab6c47..b62604c3a9e 100644 --- a/tests/queries/0_stateless/02006_test_positional_arguments.reference +++ b/tests/queries/0_stateless/02006_test_positional_arguments.reference @@ -117,3 +117,7 @@ select b from (select 5 as a, 'Hello' as b group by a); Hello select b from (select 5 as a, 'Hello' as b order by 1); Hello +create table tp2(first_col String, second_col Int32) engine = MergeTree() order by tuple(); +select count(*) from (select first_col, count(second_col) from tp2 group by 1); +0 +select total from (select first_col, count(second_col) as total from tp2 group by 1); diff --git a/tests/queries/0_stateless/02006_test_positional_arguments.sql b/tests/queries/0_stateless/02006_test_positional_arguments.sql index 2a02cd03c93..eb2b9a2d480 100644 --- a/tests/queries/0_stateless/02006_test_positional_arguments.sql +++ b/tests/queries/0_stateless/02006_test_positional_arguments.sql @@ -48,3 +48,7 @@ select substr('aaaaaaaaaaaaaa', 8) as a group by substr('aaaaaaaaaaaaaa', 8); select b from (select 5 as a, 'Hello' as b order by a); select b from (select 5 as a, 'Hello' as b group by a); select b from (select 5 as a, 'Hello' as b order by 1); + +create table tp2(first_col String, second_col Int32) engine = MergeTree() order by tuple(); +select count(*) from (select first_col, count(second_col) from tp2 group by 1); +select total from (select first_col, count(second_col) as total from tp2 group by 1); From cf1a5baa23b8e81a8e2a11ffe88d0c5c69beb72f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 18 Jul 2022 16:20:57 +0300 Subject: [PATCH 037/139] tests/performance: improve parallel_mv test Right now it is possible for parallel_mv to fail [1] due to exceeding 15 seconds limit. [1]: https://s3.amazonaws.com/clickhouse-test-reports/39183/ad6b50b087086fef8aa6f0f72b3a42f014266763/performance_comparison_aarch64_[4/4]/report.html Let's try to really disable MERGES and see. Signed-off-by: Azat Khuzhin --- tests/performance/parallel_mv.xml | 40 ++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/tests/performance/parallel_mv.xml b/tests/performance/parallel_mv.xml index 0664602b848..5b856740a19 100644 --- a/tests/performance/parallel_mv.xml +++ b/tests/performance/parallel_mv.xml @@ -3,15 +3,21 @@ 1 - create table main_table (number UInt64) engine = MergeTree order by tuple(); - create materialized view mv_1 engine = MergeTree order by tuple() as - select number, toString(number) from main_table where number % 13 != 0; - create materialized view mv_2 engine = MergeTree order by tuple() as - select number, toString(number) from main_table where number % 13 != 1; - create materialized view mv_3 engine = MergeTree order by tuple() as - select number, toString(number) from main_table where number % 13 != 3; - create materialized view mv_4 engine = MergeTree order by tuple() as - select number, toString(number) from main_table where number % 13 != 4; + create table main_table (number UInt64) engine = MergeTree order by tuple() + + create table mt_1 (n UInt64, s String) engine = MergeTree order by tuple() + create table mt_2 (n UInt64, s String) engine = MergeTree order by tuple() + create table mt_3 (n UInt64, s String) engine = MergeTree order by tuple() + create table mt_4 (n UInt64, s String) engine = MergeTree order by tuple() + + create materialized view mv_1 to mt_1 as + select number, toString(number) from main_table where number % 13 != 0 + create materialized view mv_2 to mt_2 as + select number, toString(number) from main_table where number % 13 != 1 + create materialized view mv_3 to mt_3 as + select number, toString(number) from main_table where number % 13 != 3 + create materialized view mv_4 to mt_4 as + select number, toString(number) from main_table where number % 13 != 4 SYSTEM STOP MERGES main_table SYSTEM STOP MERGES mt_1 @@ -21,9 +27,15 @@ insert into main_table select number from numbers(10000000) - drop table if exists main_table; - drop table if exists mv_1; - drop table if exists mv_2; - drop table if exists mv_3; - drop table if exists mv_4; + drop table if exists mv_1 + drop table if exists mv_2 + drop table if exists mv_3 + drop table if exists mv_4 + + drop table if exists main_table + + drop table if exists mt_1 + drop table if exists mt_2 + drop table if exists mt_3 + drop table if exists mt_4 From 39edbeba35f3671f7e1a7760d24cded23c5935d2 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotaryov <108669454+quickhouse@users.noreply.github.com> Date: Mon, 11 Jul 2022 07:39:03 +0300 Subject: [PATCH 038/139] Fixed regexp in `test_match_process_uid_against_data_owner`. --- .../test_match_process_uid_against_data_owner/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_match_process_uid_against_data_owner/test.py b/tests/integration/test_match_process_uid_against_data_owner/test.py index bbcee941833..7ce40e3e61b 100644 --- a/tests/integration/test_match_process_uid_against_data_owner/test.py +++ b/tests/integration/test_match_process_uid_against_data_owner/test.py @@ -33,7 +33,7 @@ def test_different_user(started_cluster): node.start_clickhouse(start_wait_sec=3) log = node.grep_in_log("Effective") - expected_message = "Effective user of the process \(.*\) does not match the owner of the data \(.*\)\. Run under 'sudo -u .*'\." + expected_message = "Effective user of the process (.*) does not match the owner of the data (.*)\\. Run under 'sudo -u .*'\\." if re.search(expected_message, log) is None: pytest.fail( 'Expected the server to fail with a message "{}", but the last message is "{}"'.format( From 4ae356f218df31efd55465bf8ce3854d572ce4bc Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Tue, 19 Jul 2022 09:15:42 +0000 Subject: [PATCH 039/139] Fix NULL, add test --- src/Functions/FunctionFile.cpp | 2 ++ .../02358_file_default_value.reference | 20 ++++++++++++++++++ .../0_stateless/02358_file_default_value.sh | 21 +++++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 tests/queries/0_stateless/02358_file_default_value.reference create mode 100755 tests/queries/0_stateless/02358_file_default_value.sh diff --git a/src/Functions/FunctionFile.cpp b/src/Functions/FunctionFile.cpp index ba0a713a12c..5b56a04be41 100644 --- a/src/Functions/FunctionFile.cpp +++ b/src/Functions/FunctionFile.cpp @@ -64,6 +64,8 @@ public: ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } + bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForConstants() const override { return true; } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override diff --git a/tests/queries/0_stateless/02358_file_default_value.reference b/tests/queries/0_stateless/02358_file_default_value.reference new file mode 100644 index 00000000000..5002c1e4391 --- /dev/null +++ b/tests/queries/0_stateless/02358_file_default_value.reference @@ -0,0 +1,20 @@ +text_0\n text_0\n +default \N +text_2\n text_2\n +default \N +default \N +text_5\n text_5\n +text_6\n text_6\n +text_7\n text_7\n +default \N +text_9\n text_9\n +default \N +default \N +text_5\n text_5\n +text_6\n text_6\n +text_7\n text_7\n +default \N +text_9\n text_9\n +default \N +default \N +default \N diff --git a/tests/queries/0_stateless/02358_file_default_value.sh b/tests/queries/0_stateless/02358_file_default_value.sh new file mode 100755 index 00000000000..7d2cb75c4e4 --- /dev/null +++ b/tests/queries/0_stateless/02358_file_default_value.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +USER_FILES_PATH=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') +WORKING_FOLDER_02357="${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}" + +rm -rf "${WORKING_FOLDER_02357}" +mkdir "${WORKING_FOLDER_02357}" + +for i in 0 2 5 6 7 9 +do + echo "text_${i}" > "${WORKING_FOLDER_02357}/file_${i}" +done + +${CLICKHOUSE_CLIENT} --query "WITH '${WORKING_FOLDER_02357}/file_' || toString(number) AS path SELECT file(path, 'default'), file(path, NULL) from numbers(10);" +${CLICKHOUSE_CLIENT} --query "WITH '${WORKING_FOLDER_02357}/file_' || toString(number) AS path SELECT file(path, 'default'), file(path, NULL) from numbers(3, 10);" + +rm -rf "${WORKING_FOLDER_02357}" From d245d6349a7b79013a4e87c5988a132fa1aff9f6 Mon Sep 17 00:00:00 2001 From: zvonand Date: Tue, 19 Jul 2022 12:08:08 +0200 Subject: [PATCH 040/139] fixed zero check --- src/Functions/timeSlots.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/timeSlots.cpp b/src/Functions/timeSlots.cpp index 39a784a70ba..2dc28c46083 100644 --- a/src/Functions/timeSlots.cpp +++ b/src/Functions/timeSlots.cpp @@ -316,7 +316,7 @@ public: if (!time_slot_column) throw Exception("Third argument for function " + getName() + " must be constant Decimal64", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (time_slot_size = time_slot_column->getValue(); time_slot_size == 0) + if (time_slot_size = time_slot_column->getValue().value; time_slot_size == 0) throw Exception("Third argument for function " + getName() + " must be greater than zero", ErrorCodes::ILLEGAL_COLUMN); time_slot_scale = assert_cast *>(arguments[2].type.get())->getScale(); } From e9e405e330cd5272b7eb200573582adc23b6b99d Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 19 Jul 2022 14:07:41 +0200 Subject: [PATCH 041/139] Fix --- src/Parsers/ASTCreateQuery.cpp | 2 ++ .../queries/0_stateless/02364_window_view_segfault.reference | 0 tests/queries/0_stateless/02364_window_view_segfault.sql | 4 ++++ 3 files changed, 6 insertions(+) create mode 100644 tests/queries/0_stateless/02364_window_view_segfault.reference create mode 100644 tests/queries/0_stateless/02364_window_view_segfault.sql diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index e676be63921..a277960643b 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -196,6 +196,8 @@ ASTPtr ASTCreateQuery::clone() const res->set(res->columns_list, columns_list->clone()); if (storage) res->set(res->storage, storage->clone()); + if (inner_storage) + res->set(res->inner_storage, inner_storage->clone()); if (select) res->set(res->select, select->clone()); if (table_overrides) diff --git a/tests/queries/0_stateless/02364_window_view_segfault.reference b/tests/queries/0_stateless/02364_window_view_segfault.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02364_window_view_segfault.sql b/tests/queries/0_stateless/02364_window_view_segfault.sql new file mode 100644 index 00000000000..3b454d3006e --- /dev/null +++ b/tests/queries/0_stateless/02364_window_view_segfault.sql @@ -0,0 +1,4 @@ +DROP TABLE IF EXISTS mt; +DROP TABLE IF EXISTS wv; +CREATE TABLE mt(a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple(); +CREATE WINDOW VIEW wv ON CLUSTER test_shard_localhost TO input_deduplicated INNER ENGINE Memory WATERMARK=INTERVAL '1' SECOND AS SELECT count(a), hopStart(wid) AS w_start, hopEnd(wid) AS w_end FROM mt GROUP BY hop(timestamp, INTERVAL '3' SECOND, INTERVAL '5' SECOND) AS wid; -- { serverError 344 } From 592499e290d89edb1260d72c1599e052711d5feb Mon Sep 17 00:00:00 2001 From: zvonand Date: Tue, 19 Jul 2022 23:11:03 +0200 Subject: [PATCH 042/139] fix server crash on negative size --- src/Functions/timeSlots.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Functions/timeSlots.cpp b/src/Functions/timeSlots.cpp index 2dc28c46083..12aeddcf621 100644 --- a/src/Functions/timeSlots.cpp +++ b/src/Functions/timeSlots.cpp @@ -276,7 +276,7 @@ public: if (!time_slot_column) throw Exception("Third argument for function " + getName() + " must be constant UInt32", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (time_slot_size = time_slot_column->getValue(); time_slot_size == 0) + if (time_slot_size = time_slot_column->getValue(); time_slot_size <= 0) throw Exception("Third argument for function " + getName() + " must be greater than zero", ErrorCodes::ILLEGAL_COLUMN); } @@ -316,7 +316,7 @@ public: if (!time_slot_column) throw Exception("Third argument for function " + getName() + " must be constant Decimal64", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (time_slot_size = time_slot_column->getValue().value; time_slot_size == 0) + if (time_slot_size = time_slot_column->getValue(); time_slot_size <= 0) throw Exception("Third argument for function " + getName() + " must be greater than zero", ErrorCodes::ILLEGAL_COLUMN); time_slot_scale = assert_cast *>(arguments[2].type.get())->getScale(); } From 128e3ec97f60c4d8349e728c4a66e3c44c477325 Mon Sep 17 00:00:00 2001 From: Peng Liu Date: Wed, 20 Jul 2022 17:33:55 +0800 Subject: [PATCH 043/139] compile secure rpc need the latest libgsasl --- contrib/libgsasl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libgsasl b/contrib/libgsasl index 383ee28e82f..0324680f13f 160000 --- a/contrib/libgsasl +++ b/contrib/libgsasl @@ -1 +1 @@ -Subproject commit 383ee28e82f69fa16ed43b48bd9c8ee5b313ab84 +Subproject commit 0324680f13f22bb43df5353a08e26453d7d640ac From 6443116e807034b9073a886ebf7bebd2888c14a0 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 21 Jul 2022 12:12:04 +0200 Subject: [PATCH 044/139] DirectDictionary improve performance of dictHas with duplicate keys --- src/Dictionaries/DirectDictionary.cpp | 15 +++-- ...02366_direct_dictionary_dict_has.reference | 62 +++++++++++++++++++ .../02366_direct_dictionary_dict_has.sql | 56 +++++++++++++++++ 3 files changed, 127 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/02366_direct_dictionary_dict_has.reference create mode 100644 tests/queries/0_stateless/02366_direct_dictionary_dict_has.sql diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index 6ecc216e370..189ea2a7bca 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -171,13 +171,13 @@ ColumnUInt8::Ptr DirectDictionary::hasKeys( auto requested_keys = requested_keys_extractor.extractAllKeys(); size_t requested_keys_size = requested_keys.size(); - HashMap requested_key_to_index; + HashMap> requested_key_to_index; requested_key_to_index.reserve(requested_keys_size); for (size_t i = 0; i < requested_keys.size(); ++i) { auto requested_key = requested_keys[i]; - requested_key_to_index[requested_key] = i; + requested_key_to_index[requested_key].push_back(i); } auto result = ColumnUInt8::create(requested_keys_size, false); @@ -208,10 +208,13 @@ ColumnUInt8::Ptr DirectDictionary::hasKeys( const auto * it = requested_key_to_index.find(block_key); assert(it); - size_t result_data_found_index = it->getMapped(); - /// block_keys_size cannot be used, due to duplicates. - keys_found += !result_data[result_data_found_index]; - result_data[result_data_found_index] = true; + auto & result_data_found_indexes = it->getMapped(); + for (size_t result_data_found_index : result_data_found_indexes) + { + /// block_keys_size cannot be used, due to duplicates. + keys_found += !result_data[result_data_found_index]; + result_data[result_data_found_index] = true; + } block_keys_extractor.rollbackCurrentKey(); } diff --git a/tests/queries/0_stateless/02366_direct_dictionary_dict_has.reference b/tests/queries/0_stateless/02366_direct_dictionary_dict_has.reference new file mode 100644 index 00000000000..49b34f828cd --- /dev/null +++ b/tests/queries/0_stateless/02366_direct_dictionary_dict_has.reference @@ -0,0 +1,62 @@ +0 +0 +0 +1 +0 +1 +0 +2 +1 +0 +0 0 1 +1 0 1 +2 0 1 +3 1 0 +4 0 1 +5 1 0 +6 0 1 +7 2 0 +8 1 0 +9 0 1 +1 +1 +1 +0 +1 +0 +1 +0 +0 +1 +1 +1 +1 +0 +1 +0 +1 +0 +0 +1 +1 +1 +1 +0 +1 +0 +1 +0 +0 +1 +value_0 +value_0 +value_0 +UNKNOWN +value_0 +UNKNOWN +value_0 +UNKNOWN +UNKNOWN +value_0 +4 0 +6 1 diff --git a/tests/queries/0_stateless/02366_direct_dictionary_dict_has.sql b/tests/queries/0_stateless/02366_direct_dictionary_dict_has.sql new file mode 100644 index 00000000000..b111415b56c --- /dev/null +++ b/tests/queries/0_stateless/02366_direct_dictionary_dict_has.sql @@ -0,0 +1,56 @@ +-- Tags: no-backward-compatibility-check +DROP DATABASE IF EXISTS 02366_dictionary_db; +CREATE DATABASE 02366_dictionary_db; + +CREATE TABLE 02366_dictionary_db.dict_data +( + id UInt64, + val String +) +ENGINE = Memory; + +CREATE TABLE 02366_dictionary_db.lookup_data +( + id UInt64, + lookup_key UInt64, +) +ENGINE = Memory; + +INSERT INTO 02366_dictionary_db.dict_data VALUES(0, 'value_0'); + +INSERT INTO 02366_dictionary_db.lookup_data VALUES(0, 0); +INSERT INTO 02366_dictionary_db.lookup_data VALUES(1, 0); +INSERT INTO 02366_dictionary_db.lookup_data VALUES(2, 0); +INSERT INTO 02366_dictionary_db.lookup_data VALUES(3, 1); +INSERT INTO 02366_dictionary_db.lookup_data VALUES(4, 0); +INSERT INTO 02366_dictionary_db.lookup_data VALUES(5, 1); +INSERT INTO 02366_dictionary_db.lookup_data VALUES(6, 0); +INSERT INTO 02366_dictionary_db.lookup_data VALUES(7, 2); +INSERT INTO 02366_dictionary_db.lookup_data VALUES(8, 1); +INSERT INTO 02366_dictionary_db.lookup_data VALUES(9, 0); + +CREATE DICTIONARY 02366_dictionary_db.dict0 +( + id UInt64, + val String +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'dict_data')) +LAYOUT(DIRECT()); + +SELECT lookup_key FROM 02366_dictionary_db.lookup_data ORDER BY id ASC; +SELECT id, lookup_key, dictHas(02366_dictionary_db.dict0, lookup_key) FROM 02366_dictionary_db.lookup_data ORDER BY id ASC; + +-- Nesting this way seems to help it make all the lookups as a single block, although even then it isn't guaranteed +SELECT dictHas(02366_dictionary_db.dict0, lk) FROM (SELECT any(lookup_key) as lk FROM 02366_dictionary_db.lookup_data group by id ORDER BY id ASC); +-- Same with this group by +SELECT dictHas(02366_dictionary_db.dict0, any(lookup_key)) FROM 02366_dictionary_db.lookup_data GROUP BY id ORDER BY id ASC; + + +SELECT dictHas(02366_dictionary_db.dict0, lookup_key) FROM 02366_dictionary_db.lookup_data ORDER BY id ASC; +SELECT dictGetOrDefault(02366_dictionary_db.dict0, 'val', lookup_key, 'UNKNOWN') FROM 02366_dictionary_db.lookup_data ORDER BY id ASC; +SELECT count(), has FROM 02366_dictionary_db.lookup_data group by dictHas(02366_dictionary_db.dict0, lookup_key) as has; + +DROP DICTIONARY 02366_dictionary_db.dict0; +DROP TABLE 02366_dictionary_db.lookup_data; +DROP TABLE 02366_dictionary_db.dict_data; From ffdc79d78c59ed16c1b184f2044cad08fb72cae2 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 21 Jul 2022 16:28:02 +0200 Subject: [PATCH 045/139] Updated tests --- ...02366_direct_dictionary_dict_has.reference | 52 ----------------- .../02366_direct_dictionary_dict_has.sql | 58 ++++++++----------- 2 files changed, 24 insertions(+), 86 deletions(-) diff --git a/tests/queries/0_stateless/02366_direct_dictionary_dict_has.reference b/tests/queries/0_stateless/02366_direct_dictionary_dict_has.reference index 49b34f828cd..2a74183e40e 100644 --- a/tests/queries/0_stateless/02366_direct_dictionary_dict_has.reference +++ b/tests/queries/0_stateless/02366_direct_dictionary_dict_has.reference @@ -1,13 +1,3 @@ -0 -0 -0 -1 -0 -1 -0 -2 -1 -0 0 0 1 1 0 1 2 0 1 @@ -18,45 +8,3 @@ 7 2 0 8 1 0 9 0 1 -1 -1 -1 -0 -1 -0 -1 -0 -0 -1 -1 -1 -1 -0 -1 -0 -1 -0 -0 -1 -1 -1 -1 -0 -1 -0 -1 -0 -0 -1 -value_0 -value_0 -value_0 -UNKNOWN -value_0 -UNKNOWN -value_0 -UNKNOWN -UNKNOWN -value_0 -4 0 -6 1 diff --git a/tests/queries/0_stateless/02366_direct_dictionary_dict_has.sql b/tests/queries/0_stateless/02366_direct_dictionary_dict_has.sql index b111415b56c..9d6950051f0 100644 --- a/tests/queries/0_stateless/02366_direct_dictionary_dict_has.sql +++ b/tests/queries/0_stateless/02366_direct_dictionary_dict_has.sql @@ -1,56 +1,46 @@ -- Tags: no-backward-compatibility-check -DROP DATABASE IF EXISTS 02366_dictionary_db; -CREATE DATABASE 02366_dictionary_db; -CREATE TABLE 02366_dictionary_db.dict_data +DROP TABLE IF EXISTS test_table; +CREATE TABLE test_table ( id UInt64, - val String + value String ) ENGINE = Memory; -CREATE TABLE 02366_dictionary_db.lookup_data +DROP TABLE IF EXISTS test_lookup_table; +CREATE TABLE test_lookup_table ( id UInt64, lookup_key UInt64, ) ENGINE = Memory; -INSERT INTO 02366_dictionary_db.dict_data VALUES(0, 'value_0'); +INSERT INTO test_table VALUES(0, 'value_0'); -INSERT INTO 02366_dictionary_db.lookup_data VALUES(0, 0); -INSERT INTO 02366_dictionary_db.lookup_data VALUES(1, 0); -INSERT INTO 02366_dictionary_db.lookup_data VALUES(2, 0); -INSERT INTO 02366_dictionary_db.lookup_data VALUES(3, 1); -INSERT INTO 02366_dictionary_db.lookup_data VALUES(4, 0); -INSERT INTO 02366_dictionary_db.lookup_data VALUES(5, 1); -INSERT INTO 02366_dictionary_db.lookup_data VALUES(6, 0); -INSERT INTO 02366_dictionary_db.lookup_data VALUES(7, 2); -INSERT INTO 02366_dictionary_db.lookup_data VALUES(8, 1); -INSERT INTO 02366_dictionary_db.lookup_data VALUES(9, 0); +INSERT INTO test_lookup_table VALUES(0, 0); +INSERT INTO test_lookup_table VALUES(1, 0); +INSERT INTO test_lookup_table VALUES(2, 0); +INSERT INTO test_lookup_table VALUES(3, 1); +INSERT INTO test_lookup_table VALUES(4, 0); +INSERT INTO test_lookup_table VALUES(5, 1); +INSERT INTO test_lookup_table VALUES(6, 0); +INSERT INTO test_lookup_table VALUES(7, 2); +INSERT INTO test_lookup_table VALUES(8, 1); +INSERT INTO test_lookup_table VALUES(9, 0); -CREATE DICTIONARY 02366_dictionary_db.dict0 +DROP DICTIONARY IF EXISTS test_dictionary; +CREATE DICTIONARY test_dictionary ( id UInt64, - val String + value String ) PRIMARY KEY id -SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'dict_data')) +SOURCE(CLICKHOUSE(TABLE 'test_table')) LAYOUT(DIRECT()); -SELECT lookup_key FROM 02366_dictionary_db.lookup_data ORDER BY id ASC; -SELECT id, lookup_key, dictHas(02366_dictionary_db.dict0, lookup_key) FROM 02366_dictionary_db.lookup_data ORDER BY id ASC; +SELECT id, lookup_key, dictHas('test_dictionary', lookup_key) FROM test_lookup_table ORDER BY id ASC; --- Nesting this way seems to help it make all the lookups as a single block, although even then it isn't guaranteed -SELECT dictHas(02366_dictionary_db.dict0, lk) FROM (SELECT any(lookup_key) as lk FROM 02366_dictionary_db.lookup_data group by id ORDER BY id ASC); --- Same with this group by -SELECT dictHas(02366_dictionary_db.dict0, any(lookup_key)) FROM 02366_dictionary_db.lookup_data GROUP BY id ORDER BY id ASC; - - -SELECT dictHas(02366_dictionary_db.dict0, lookup_key) FROM 02366_dictionary_db.lookup_data ORDER BY id ASC; -SELECT dictGetOrDefault(02366_dictionary_db.dict0, 'val', lookup_key, 'UNKNOWN') FROM 02366_dictionary_db.lookup_data ORDER BY id ASC; -SELECT count(), has FROM 02366_dictionary_db.lookup_data group by dictHas(02366_dictionary_db.dict0, lookup_key) as has; - -DROP DICTIONARY 02366_dictionary_db.dict0; -DROP TABLE 02366_dictionary_db.lookup_data; -DROP TABLE 02366_dictionary_db.dict_data; +DROP DICTIONARY test_dictionary; +DROP TABLE test_table; +DROP TABLE test_lookup_table; From 64b41e8676da7036050e7b08e8fadc3bd3f91e6f Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 27 Jan 2022 20:36:30 +0000 Subject: [PATCH 046/139] Performance check build AVX --- cmake/cpu_features.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index 1fc3c2db804..f1f6dfb9a9c 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -16,7 +16,7 @@ option (ENABLE_SSE41 "Use SSE4.1 instructions on x86_64" 1) option (ENABLE_SSE42 "Use SSE4.2 instructions on x86_64" 1) option (ENABLE_PCLMULQDQ "Use pclmulqdq instructions on x86_64" 1) option (ENABLE_POPCNT "Use popcnt instructions on x86_64" 1) -option (ENABLE_AVX "Use AVX instructions on x86_64" 0) +option (ENABLE_AVX "Use AVX instructions on x86_64" 1) option (ENABLE_AVX2 "Use AVX2 instructions on x86_64" 0) option (ENABLE_AVX512 "Use AVX512 instructions on x86_64" 0) option (ENABLE_AVX512_VBMI "Use AVX512_VBMI instruction on x86_64 (depends on ENABLE_AVX512)" 0) From 65d58aa7528330dcbe074eade87cc01df0386bc7 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 8 Jul 2022 09:04:58 +0000 Subject: [PATCH 047/139] Disable 01103_check_cpu_instructions_at_startup.sh More than a decade after AVX was released, AVX is still not supported by QEMU, even if "-cpu help" pretends to. As a result, we cannot use QEMU to verify that a ClickHouse binary compiled for a SIMD level up to AVX runs on a system with a SIMD level up to AVX. The alternative is to disassemble the binary and grep for unwanted instructions (e.g. AVX512) which is just too fragile ... https://gitlab.com/qemu-project/qemu/-/issues/164 https://www.mail-archive.com/qemu-devel@nongnu.org/msg713932.html https://lore.kernel.org/all/CAObpvQmejWBh+RNz2vhk16-kcY_QveM_pSmM5ZeWqWv1d8AJzQ@mail.gmail.com/T/ --- ...01103_check_cpu_instructions_at_startup.reference | 7 ------- .../01103_check_cpu_instructions_at_startup.sh | 12 ++++++++++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.reference b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.reference index 8984d35930a..e69de29bb2d 100644 --- a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.reference +++ b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.reference @@ -1,7 +0,0 @@ -Instruction check fail. The CPU does not support SSSE3 instruction set. -Instruction check fail. The CPU does not support SSE4.1 instruction set. -Instruction check fail. The CPU does not support SSE4.2 instruction set. -Instruction check fail. The CPU does not support POPCNT instruction set. -: MADV_DONTNEED does not work (memset will be used instead) -: (This is the expected behaviour if you are running under QEMU) -1 diff --git a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh index 9b6e1e05f2d..9fb239e87b2 100755 --- a/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh +++ b/tests/queries/0_stateless/01103_check_cpu_instructions_at_startup.sh @@ -2,6 +2,18 @@ # Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-fasttest, no-cpu-aarch64 # Tag no-fasttest: avoid dependency on qemu -- invonvenient when running locally +# More than a decade after AVX was released, AVX is still not supported by QEMU, even if "-cpu help" pretends to. As a result, we cannot use +# QEMU to verify that a ClickHouse binary compiled for a SIMD level up to AVX runs on a system with a SIMD level up to AVX. The alternative +# is to disassemble the binary and grep for unwanted instructions (e.g. AVX512) which is just too fragile ... +# +# https://gitlab.com/qemu-project/qemu/-/issues/164 +# https://www.mail-archive.com/qemu-devel@nongnu.org/msg713932.html +# https://lore.kernel.org/all/CAObpvQmejWBh+RNz2vhk16-kcY_QveM_pSmM5ZeWqWv1d8AJzQ@mail.gmail.com/T/ + +exit 0 + +# keeping the original test because it is instructive and maybe QEMU will be fixed at some point ... + CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh From cad0e7a62c8e6b44cdd77c3455a58ddf4a76b139 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 20 Jul 2022 19:38:06 +0000 Subject: [PATCH 048/139] Move hot loop inside nested if/else statements - required to control the vectorization of a specific cast which annotates the loop using pragmas --- src/Functions/FunctionsConversion.h | 86 ++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 21 deletions(-) diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index b666602e366..f0ce4c94731 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -191,27 +191,29 @@ struct ConvertImpl vec_null_map_to = &col_null_map_to->getData(); } - bool result_is_bool = isBool(result_type); - for (size_t i = 0; i < input_rows_count; ++i) + if constexpr (std::is_same_v) { - if constexpr (std::is_same_v) + if (isBool(result_type)) { - if (result_is_bool) + for (size_t i = 0; i < input_rows_count; ++i) { vec_to[i] = vec_from[i] != FromFieldType(0); - continue; } + goto done; } + } - if constexpr (std::is_same_v != std::is_same_v) + if constexpr (std::is_same_v != std::is_same_v) + { + throw Exception("Conversion between numeric types and UUID is not supported. Probably the passed UUID is unquoted", ErrorCodes::NOT_IMPLEMENTED); + } + else + { + if constexpr (IsDataTypeDecimal || IsDataTypeDecimal) { - throw Exception("Conversion between numeric types and UUID is not supported. Probably the passed UUID is unquoted", ErrorCodes::NOT_IMPLEMENTED); - } - else - { - if constexpr (IsDataTypeDecimal || IsDataTypeDecimal) + if constexpr (std::is_same_v) { - if constexpr (std::is_same_v) + for (size_t i = 0; i < input_rows_count; ++i) { ToFieldType result; bool convert_result = false; @@ -231,7 +233,10 @@ struct ConvertImpl (*vec_null_map_to)[i] = true; } } - else + } + else + { + for (size_t i = 0; i < input_rows_count; ++i) { if constexpr (IsDataTypeDecimal && IsDataTypeDecimal) vec_to[i] = convertDecimals(vec_from[i], col_from->getScale(), col_to->getScale()); @@ -243,10 +248,13 @@ struct ConvertImpl throw Exception("Unsupported data type in conversion function", ErrorCodes::CANNOT_CONVERT_TYPE); } } - else + } + else + { + /// If From Data is Nan or Inf and we convert to integer type, throw exception + if constexpr (std::is_floating_point_v && !std::is_floating_point_v) { - /// If From Data is Nan or Inf and we convert to integer type, throw exception - if constexpr (std::is_floating_point_v && !std::is_floating_point_v) + for (size_t i = 0; i < input_rows_count; ++i) { if (!isFinite(vec_from[i])) { @@ -254,15 +262,46 @@ struct ConvertImpl { vec_to[i] = 0; (*vec_null_map_to)[i] = true; - continue; } else throw Exception("Unexpected inf or nan to integer conversion", ErrorCodes::CANNOT_CONVERT_TYPE); } - } + else + { + if constexpr (std::is_same_v + || std::is_same_v) + { + bool convert_result = accurate::convertNumeric(vec_from[i], vec_to[i]); - if constexpr (std::is_same_v - || std::is_same_v) + if (!convert_result) + { + if (std::is_same_v) + { + vec_to[i] = 0; + (*vec_null_map_to)[i] = true; + } + else + { + throw Exception( + "Value in column " + named_from.column->getName() + " cannot be safely converted into type " + + result_type->getName(), + ErrorCodes::CANNOT_CONVERT_TYPE); + } + } + } + else + { + vec_to[i] = static_cast(vec_from[i]); + } + } + } + goto done; + } + + if constexpr (std::is_same_v + || std::is_same_v) + { + for (size_t i = 0; i < input_rows_count; ++i) { bool convert_result = accurate::convertNumeric(vec_from[i], vec_to[i]); @@ -282,7 +321,10 @@ struct ConvertImpl } } } - else + } + else + { + for (size_t i = 0; i < input_rows_count; ++i) { vec_to[i] = static_cast(vec_from[i]); } @@ -290,6 +332,8 @@ struct ConvertImpl } } +done: + if constexpr (std::is_same_v) return ColumnNullable::create(std::move(col_to), std::move(col_null_map_to)); else From 6a631426b7e06f493ad3e784461046d8991132e0 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 22 Jul 2022 10:59:56 +0000 Subject: [PATCH 049/139] Disable vectorization for uint64 --> float32 cast --- src/Functions/FunctionsConversion.h | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index f0ce4c94731..014ce98a795 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -324,9 +324,28 @@ struct ConvertImpl } else { - for (size_t i = 0; i < input_rows_count; ++i) + if constexpr (std::is_same_v && std::is_same_v) { - vec_to[i] = static_cast(vec_from[i]); + /// Turns out that when ClickHouse is compiled with AVX1 or AVX2 instructions, Clang's autovectorizer produces + /// code for UInt64-to-Float23 conversion which is only ~50% as fast as scalar code. Interestingly, scalar code + /// is equally fast than code compiled for SSE4.2, so we might as well disable vectorization. This situation + /// may change with AVX512 which has a dediated instruction for that usecase (_mm512_cvtepi64_ps). +#if defined(__x86_64__) +# ifdef __clang__ +# pragma clang loop vectorize(disable) interleave(disable) +# endif +#endif + for (size_t i = 0; i < input_rows_count; ++i) + { + vec_to[i] = static_cast(vec_from[i]); + } + } + else + { + for (size_t i = 0; i < input_rows_count; ++i) + { + vec_to[i] = static_cast(vec_from[i]); + } } } } From d41292bf426ac11475224f6c2e307ec176963f8c Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 22 Jul 2022 18:51:26 +0000 Subject: [PATCH 050/139] Update arrow to fix possible data race --- contrib/arrow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/arrow b/contrib/arrow index efdcd015cfd..87fbdebf43c 160000 --- a/contrib/arrow +++ b/contrib/arrow @@ -1 +1 @@ -Subproject commit efdcd015cfdee1b6aa349c9ca227ca12c3d697f5 +Subproject commit 87fbdebf43ccc44ce1c0204ba40b314a1358766a From 071374b1525b9b7c6ae51a41e0cf3bd62b49e700 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 24 Jul 2022 01:15:54 +0200 Subject: [PATCH 051/139] Remove SPLIT_BINARY --- CMakeLists.txt | 5 +- docker/packager/packager | 3 +- docs/en/development/developer-instruction.md | 2 +- docs/ru/development/developer-instruction.md | 2 +- programs/CMakeLists.txt | 359 ++++++++---------- src/Common/config.h.in | 1 - ...StorageSystemBuildOptions.generated.cpp.in | 2 - tests/integration/CMakeLists.txt | 6 +- 8 files changed, 157 insertions(+), 223 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 05f88f3530e..61af62811cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,10 +77,9 @@ option(USE_STATIC_LIBRARIES "Disable to use shared libraries" ON) # DEVELOPER ONLY. # Faster linking if turned on. option(SPLIT_SHARED_LIBRARIES "Keep all internal libraries as separate .so files" OFF) -option(CLICKHOUSE_SPLIT_BINARY "Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled" OFF) -if (USE_STATIC_LIBRARIES AND (SPLIT_SHARED_LIBRARIES OR CLICKHOUSE_SPLIT_BINARY)) - message(FATAL_ERROR "SPLIT_SHARED_LIBRARIES=1 or CLICKHOUSE_SPLIT_BINARY=1 must not be used together with USE_STATIC_LIBRARIES=1") +if (USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES) + message(FATAL_ERROR "SPLIT_SHARED_LIBRARIES=1 must not be used together with USE_STATIC_LIBRARIES=1") endif() if (NOT USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES) diff --git a/docker/packager/packager b/docker/packager/packager index 7c0f046b76c..9d66b2d4be5 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -261,8 +261,7 @@ def parse_env_variables( if split_binary: cmake_flags.append( - "-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 " - "-DCLICKHOUSE_SPLIT_BINARY=1" + "-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1" ) # We can't always build utils because it requires too much space, but # we have to build them at least in some way in CI. The split build is diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md index 77ddae6a756..57c11451011 100644 --- a/docs/en/development/developer-instruction.md +++ b/docs/en/development/developer-instruction.md @@ -281,7 +281,7 @@ Most probably some of the builds will fail at first times. This is due to the fa ClickHouse is normally statically linked into a single static `clickhouse` binary with minimal dependencies. This is convenient for distribution, but it means that for every change the entire binary needs to be re-linked, which is slow and inconvenient for development. As an alternative, you can instead build dynamically linked shared libraries and separate binaries `clickhouse-server`, `clickhouse-client` etc., allowing for faster incremental builds. To use it, add the following flags to your `cmake` invocation: ``` --DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 +-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 ``` Note that the split build has several drawbacks: diff --git a/docs/ru/development/developer-instruction.md b/docs/ru/development/developer-instruction.md index 4c1981198a0..f2fac4ccdf8 100644 --- a/docs/ru/development/developer-instruction.md +++ b/docs/ru/development/developer-instruction.md @@ -170,7 +170,7 @@ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" В случае использования на разработческой машине старого HDD или SSD, а также при желании использовать меньше места для артефактов сборки можно использовать следующую команду: ```bash -cmake -DUSE_DEBUG_HELPERS=1 -DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 .. +cmake -DUSE_DEBUG_HELPERS=1 -DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 .. ``` При этом надо учесть, что получаемые в результате сборки исполнимые файлы будут динамически слинкованы с библиотеками, и поэтому фактически станут непереносимыми на другие компьютеры (либо для этого нужно будет предпринять значительно больше усилий по сравнению со статической сборкой). Плюсом же в данном случае является значительно меньшее время сборки (это проявляется не на первой сборке, а на последующих, после внесения изменений в исходный код - тратится меньшее время на линковку по сравнению со статической сборкой) и значительно меньшее использование места на жёстком диске (экономия более, чем в 3 раза по сравнению со статической сборкой). Для целей разработки, когда планируются только отладочные запуски на том же компьютере, где осуществлялась сборка, это может быть наиболее удобным вариантом. diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index c6b8c8b4e0d..294b2597af5 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -18,7 +18,7 @@ option (ENABLE_CLICKHOUSE_SERVER "Server mode (main mode)" ${ENABLE_CLICKHOUSE_A option (ENABLE_CLICKHOUSE_CLIENT "Client mode (interactive tui/shell that connects to the server)" ${ENABLE_CLICKHOUSE_ALL}) -if (CLICKHOUSE_SPLIT_BINARY OR NOT ENABLE_UTILS) +if (NOT ENABLE_UTILS) option (ENABLE_CLICKHOUSE_SELF_EXTRACTING "Self-extracting executable" OFF) else () option (ENABLE_CLICKHOUSE_SELF_EXTRACTING "Self-extracting executable" ON) @@ -80,12 +80,7 @@ if (NOT ENABLE_NURAFT) set(ENABLE_CLICKHOUSE_KEEPER_CONVERTER OFF) endif() -if (CLICKHOUSE_SPLIT_BINARY) - option(ENABLE_CLICKHOUSE_INSTALL "Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only)" OFF) -else () - option(ENABLE_CLICKHOUSE_INSTALL "Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only)" - ${ENABLE_CLICKHOUSE_ALL}) -endif () +option(ENABLE_CLICKHOUSE_INSTALL "Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only)" ${ENABLE_CLICKHOUSE_ALL}) message(STATUS "ClickHouse modes:") @@ -205,10 +200,6 @@ macro(clickhouse_target_link_split_lib target name) endif() endmacro() -macro(clickhouse_program_link_split_binary name) - clickhouse_target_link_split_lib(clickhouse-${name} ${name}) -endmacro() - macro(clickhouse_program_add_library name) string(TOUPPER ${name} name_uc) string(REPLACE "-" "_" name_uc ${name_uc}) @@ -233,17 +224,8 @@ macro(clickhouse_program_add_library name) endif() endmacro() -macro(clickhouse_program_add_executable name) - if(CLICKHOUSE_SPLIT_BINARY) - clickhouse_add_executable(clickhouse-${name} clickhouse-${name}.cpp) - clickhouse_program_link_split_binary(${name}) - install(TARGETS clickhouse-${name} ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - endif() -endmacro() - macro(clickhouse_program_add name) clickhouse_program_add_library(${name}) - clickhouse_program_add_executable(${name}) endmacro() add_subdirectory (server) @@ -336,203 +318,164 @@ if (CLICKHOUSE_ONE_SHARED) install (TARGETS clickhouse-lib LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse) endif() -if (CLICKHOUSE_SPLIT_BINARY) - set (CLICKHOUSE_ALL_TARGETS - clickhouse-server - clickhouse-client - clickhouse-local - clickhouse-benchmark - clickhouse-extract-from-config - clickhouse-compressor - clickhouse-format - clickhouse-obfuscator - clickhouse-git-import - clickhouse-copier - clickhouse-static-files-disk-uploader - clickhouse-disks) +clickhouse_add_executable (clickhouse main.cpp) - if (ENABLE_CLICKHOUSE_ODBC_BRIDGE) - list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-odbc-bridge) - endif () +# A library that prevent usage of several functions from libc. +if (ARCH_AMD64 AND OS_LINUX AND NOT OS_ANDROID) + set (HARMFUL_LIB harmful) +endif () - if (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE) - list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-library-bridge) - endif () +target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils ${HARMFUL_LIB}) +target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) - if (ENABLE_CLICKHOUSE_KEEPER) - list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-keeper) - endif () +if (ENABLE_CLICKHOUSE_SERVER) + clickhouse_target_link_split_lib(clickhouse server) +endif () +if (ENABLE_CLICKHOUSE_CLIENT) + clickhouse_target_link_split_lib(clickhouse client) +endif () +if (ENABLE_CLICKHOUSE_LOCAL) + clickhouse_target_link_split_lib(clickhouse local) +endif () +if (ENABLE_CLICKHOUSE_BENCHMARK) + clickhouse_target_link_split_lib(clickhouse benchmark) +endif () +if (ENABLE_CLICKHOUSE_COPIER) + clickhouse_target_link_split_lib(clickhouse copier) +endif () +if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG) + clickhouse_target_link_split_lib(clickhouse extract-from-config) +endif () +if (ENABLE_CLICKHOUSE_COMPRESSOR) + clickhouse_target_link_split_lib(clickhouse compressor) +endif () +if (ENABLE_CLICKHOUSE_FORMAT) + clickhouse_target_link_split_lib(clickhouse format) +endif () +if (ENABLE_CLICKHOUSE_OBFUSCATOR) + clickhouse_target_link_split_lib(clickhouse obfuscator) +endif () +if (ENABLE_CLICKHOUSE_GIT_IMPORT) + clickhouse_target_link_split_lib(clickhouse git-import) +endif () +if (ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER) + clickhouse_target_link_split_lib(clickhouse static-files-disk-uploader) +endif () +if (ENABLE_CLICKHOUSE_SU) + clickhouse_target_link_split_lib(clickhouse su) +endif () +if (ENABLE_CLICKHOUSE_KEEPER) + clickhouse_target_link_split_lib(clickhouse keeper) +endif() +if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER) + clickhouse_target_link_split_lib(clickhouse keeper-converter) +endif() +if (ENABLE_CLICKHOUSE_INSTALL) + clickhouse_target_link_split_lib(clickhouse install) +endif () +if (ENABLE_CLICKHOUSE_DISKS) + clickhouse_target_link_split_lib(clickhouse disks) +endif () - if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER) - list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-keeper-converter) - endif () +set (CLICKHOUSE_BUNDLE) +if (ENABLE_CLICKHOUSE_SERVER) + add_custom_target (clickhouse-server ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-server DEPENDS clickhouse) + install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-server" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-server) +endif () +if (ENABLE_CLICKHOUSE_CLIENT) + add_custom_target (clickhouse-client ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-client DEPENDS clickhouse) + install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-client" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-client) +endif () +if (ENABLE_CLICKHOUSE_LOCAL) + add_custom_target (clickhouse-local ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-local DEPENDS clickhouse) + install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-local" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-local) +endif () +if (ENABLE_CLICKHOUSE_BENCHMARK) + add_custom_target (clickhouse-benchmark ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-benchmark DEPENDS clickhouse) + install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-benchmark" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-benchmark) +endif () +if (ENABLE_CLICKHOUSE_COPIER) + add_custom_target (clickhouse-copier ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-copier DEPENDS clickhouse) + install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-copier" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-copier) +endif () +if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG) + add_custom_target (clickhouse-extract-from-config ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-extract-from-config DEPENDS clickhouse) + install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-extract-from-config" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-extract-from-config) +endif () +if (ENABLE_CLICKHOUSE_COMPRESSOR) + add_custom_target (clickhouse-compressor ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-compressor DEPENDS clickhouse) + install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-compressor" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-compressor) +endif () +if (ENABLE_CLICKHOUSE_FORMAT) + add_custom_target (clickhouse-format ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-format DEPENDS clickhouse) + install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-format" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-format) +endif () +if (ENABLE_CLICKHOUSE_OBFUSCATOR) + add_custom_target (clickhouse-obfuscator ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-obfuscator DEPENDS clickhouse) + install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-obfuscator" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-obfuscator) +endif () +if (ENABLE_CLICKHOUSE_GIT_IMPORT) + add_custom_target (clickhouse-git-import ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-git-import DEPENDS clickhouse) + install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-git-import" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-git-import) +endif () +if (ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER) + add_custom_target (clickhouse-static-files-disk-uploader ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-static-files-disk-uploader DEPENDS clickhouse) + install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-static-files-disk-uploader" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-static-files-disk-uploader) +endif () +if (ENABLE_CLICKHOUSE_SU) + add_custom_target (clickhouse-su ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-su DEPENDS clickhouse) + install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-su" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-su) +endif () - set_target_properties(${CLICKHOUSE_ALL_TARGETS} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) - - add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_ALL_TARGETS}) - add_custom_target (clickhouse ALL DEPENDS clickhouse-bundle) - - install(PROGRAMS clickhouse-split-helper DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME clickhouse COMPONENT clickhouse) -else () - clickhouse_add_executable (clickhouse main.cpp) - - # A library that prevent usage of several functions from libc. - if (ARCH_AMD64 AND OS_LINUX AND NOT OS_ANDROID) - set (HARMFUL_LIB harmful) - endif () - - target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils ${HARMFUL_LIB}) - target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) - - if (ENABLE_CLICKHOUSE_SERVER) - clickhouse_target_link_split_lib(clickhouse server) - endif () - if (ENABLE_CLICKHOUSE_CLIENT) - clickhouse_target_link_split_lib(clickhouse client) - endif () - if (ENABLE_CLICKHOUSE_LOCAL) - clickhouse_target_link_split_lib(clickhouse local) - endif () - if (ENABLE_CLICKHOUSE_BENCHMARK) - clickhouse_target_link_split_lib(clickhouse benchmark) - endif () - if (ENABLE_CLICKHOUSE_COPIER) - clickhouse_target_link_split_lib(clickhouse copier) - endif () - if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG) - clickhouse_target_link_split_lib(clickhouse extract-from-config) - endif () - if (ENABLE_CLICKHOUSE_COMPRESSOR) - clickhouse_target_link_split_lib(clickhouse compressor) - endif () - if (ENABLE_CLICKHOUSE_FORMAT) - clickhouse_target_link_split_lib(clickhouse format) - endif () - if (ENABLE_CLICKHOUSE_OBFUSCATOR) - clickhouse_target_link_split_lib(clickhouse obfuscator) - endif () - if (ENABLE_CLICKHOUSE_GIT_IMPORT) - clickhouse_target_link_split_lib(clickhouse git-import) - endif () - if (ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER) - clickhouse_target_link_split_lib(clickhouse static-files-disk-uploader) - endif () - if (ENABLE_CLICKHOUSE_SU) - clickhouse_target_link_split_lib(clickhouse su) - endif () - if (ENABLE_CLICKHOUSE_KEEPER) - clickhouse_target_link_split_lib(clickhouse keeper) - endif() - if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER) - clickhouse_target_link_split_lib(clickhouse keeper-converter) - endif() - if (ENABLE_CLICKHOUSE_INSTALL) - clickhouse_target_link_split_lib(clickhouse install) - endif () - if (ENABLE_CLICKHOUSE_DISKS) - clickhouse_target_link_split_lib(clickhouse disks) - endif () - - set (CLICKHOUSE_BUNDLE) - if (ENABLE_CLICKHOUSE_SERVER) - add_custom_target (clickhouse-server ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-server DEPENDS clickhouse) - install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-server" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - list(APPEND CLICKHOUSE_BUNDLE clickhouse-server) - endif () - if (ENABLE_CLICKHOUSE_CLIENT) - add_custom_target (clickhouse-client ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-client DEPENDS clickhouse) - install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-client" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - list(APPEND CLICKHOUSE_BUNDLE clickhouse-client) - endif () - if (ENABLE_CLICKHOUSE_LOCAL) - add_custom_target (clickhouse-local ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-local DEPENDS clickhouse) - install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-local" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - list(APPEND CLICKHOUSE_BUNDLE clickhouse-local) - endif () - if (ENABLE_CLICKHOUSE_BENCHMARK) - add_custom_target (clickhouse-benchmark ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-benchmark DEPENDS clickhouse) - install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-benchmark" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - list(APPEND CLICKHOUSE_BUNDLE clickhouse-benchmark) - endif () - if (ENABLE_CLICKHOUSE_COPIER) - add_custom_target (clickhouse-copier ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-copier DEPENDS clickhouse) - install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-copier" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - list(APPEND CLICKHOUSE_BUNDLE clickhouse-copier) - endif () - if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG) - add_custom_target (clickhouse-extract-from-config ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-extract-from-config DEPENDS clickhouse) - install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-extract-from-config" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - list(APPEND CLICKHOUSE_BUNDLE clickhouse-extract-from-config) - endif () - if (ENABLE_CLICKHOUSE_COMPRESSOR) - add_custom_target (clickhouse-compressor ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-compressor DEPENDS clickhouse) - install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-compressor" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - list(APPEND CLICKHOUSE_BUNDLE clickhouse-compressor) - endif () - if (ENABLE_CLICKHOUSE_FORMAT) - add_custom_target (clickhouse-format ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-format DEPENDS clickhouse) - install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-format" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - list(APPEND CLICKHOUSE_BUNDLE clickhouse-format) - endif () - if (ENABLE_CLICKHOUSE_OBFUSCATOR) - add_custom_target (clickhouse-obfuscator ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-obfuscator DEPENDS clickhouse) - install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-obfuscator" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - list(APPEND CLICKHOUSE_BUNDLE clickhouse-obfuscator) - endif () - if (ENABLE_CLICKHOUSE_GIT_IMPORT) - add_custom_target (clickhouse-git-import ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-git-import DEPENDS clickhouse) - install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-git-import" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - list(APPEND CLICKHOUSE_BUNDLE clickhouse-git-import) - endif () - if (ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER) - add_custom_target (clickhouse-static-files-disk-uploader ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-static-files-disk-uploader DEPENDS clickhouse) - install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-static-files-disk-uploader" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - list(APPEND CLICKHOUSE_BUNDLE clickhouse-static-files-disk-uploader) - endif () - if (ENABLE_CLICKHOUSE_SU) - add_custom_target (clickhouse-su ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-su DEPENDS clickhouse) - install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-su" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - list(APPEND CLICKHOUSE_BUNDLE clickhouse-su) - endif () - - if (ENABLE_CLICKHOUSE_KEEPER) - if (NOT BUILD_STANDALONE_KEEPER AND CREATE_KEEPER_SYMLINK) - add_custom_target (clickhouse-keeper ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper DEPENDS clickhouse) - install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - endif() - - # otherwise we don't build keeper - if (BUILD_STANDALONE_KEEPER OR CREATE_KEEPER_SYMLINK) - list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper) - endif() - endif () - if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER) - add_custom_target (clickhouse-keeper-converter ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper-converter DEPENDS clickhouse) - install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper-converter" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper-converter) - endif () - if (ENABLE_CLICKHOUSE_DISKS) - add_custom_target (clickhouse-disks ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-disks DEPENDS clickhouse) - install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-disks" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - list(APPEND CLICKHOUSE_BUNDLE clickhouse-disks) - endif () - - add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_BUNDLE}) - - if (USE_GDB_ADD_INDEX) - add_custom_command(TARGET clickhouse POST_BUILD COMMAND ${GDB_ADD_INDEX_EXE} clickhouse COMMENT "Adding .gdb-index to clickhouse" VERBATIM) +if (ENABLE_CLICKHOUSE_KEEPER) + if (NOT BUILD_STANDALONE_KEEPER AND CREATE_KEEPER_SYMLINK) + add_custom_target (clickhouse-keeper ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper DEPENDS clickhouse) + install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) endif() - if (USE_BINARY_HASH) - add_custom_command(TARGET clickhouse POST_BUILD COMMAND ./clickhouse hash-binary > hash && ${OBJCOPY_PATH} --add-section .clickhouse.hash=hash clickhouse COMMENT "Adding section '.clickhouse.hash' to clickhouse binary" VERBATIM) + # otherwise we don't build keeper + if (BUILD_STANDALONE_KEEPER OR CREATE_KEEPER_SYMLINK) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper) endif() +endif () +if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER) + add_custom_target (clickhouse-keeper-converter ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper-converter DEPENDS clickhouse) + install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper-converter" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper-converter) +endif () +if (ENABLE_CLICKHOUSE_DISKS) + add_custom_target (clickhouse-disks ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-disks DEPENDS clickhouse) + install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-disks" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-disks) +endif () - if (SPLIT_DEBUG_SYMBOLS) - clickhouse_split_debug_symbols(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH clickhouse) - else() - clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR}) - install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) - endif() +add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_BUNDLE}) + +if (USE_GDB_ADD_INDEX) + add_custom_command(TARGET clickhouse POST_BUILD COMMAND ${GDB_ADD_INDEX_EXE} clickhouse COMMENT "Adding .gdb-index to clickhouse" VERBATIM) +endif() + +if (USE_BINARY_HASH) + add_custom_command(TARGET clickhouse POST_BUILD COMMAND ./clickhouse hash-binary > hash && ${OBJCOPY_PATH} --add-section .clickhouse.hash=hash clickhouse COMMENT "Adding section '.clickhouse.hash' to clickhouse binary" VERBATIM) +endif() + +if (SPLIT_DEBUG_SYMBOLS) + clickhouse_split_debug_symbols(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH clickhouse) +else() + clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR}) + install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) endif() if (ENABLE_TESTS) diff --git a/src/Common/config.h.in b/src/Common/config.h.in index d8d308c59bd..27db791b954 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -18,7 +18,6 @@ #cmakedefine01 USE_DATASKETCHES #cmakedefine01 USE_YAML_CPP -#cmakedefine01 CLICKHOUSE_SPLIT_BINARY #cmakedefine01 USE_BZIP2 #cmakedefine01 USE_MINIZIP #cmakedefine01 USE_SNAPPY diff --git a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index 56c96a0857c..dde90ce459a 100644 --- a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -21,7 +21,6 @@ const char * auto_config_build[] "LINK_FLAGS", "@FULL_EXE_LINKER_FLAGS_NORMALIZED@", "BUILD_COMPILE_DEFINITIONS", "@BUILD_COMPILE_DEFINITIONS@", "STATIC", "@USE_STATIC_LIBRARIES@", - "SPLIT_BINARY", "@CLICKHOUSE_SPLIT_BINARY@", "USE_EMBEDDED_COMPILER", "@USE_EMBEDDED_COMPILER@", "USE_GLIBC_COMPATIBILITY", "@GLIBC_COMPATIBILITY@", "USE_JEMALLOC", "@ENABLE_JEMALLOC@", @@ -59,7 +58,6 @@ const char * auto_config_build[] "USE_AWS_S3", "@USE_AWS_S3@", "USE_CASSANDRA", "@USE_CASSANDRA@", "USE_YAML_CPP", "@USE_YAML_CPP@", - "CLICKHOUSE_SPLIT_BINARY", "@CLICKHOUSE_SPLIT_BINARY@", "USE_SENTRY", "@USE_SENTRY@", "USE_DATASKETCHES", "@USE_DATASKETCHES@", "USE_AVRO", "@USE_AVRO@", diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 68c695f57a0..1482c068be5 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -1,8 +1,4 @@ -if(CLICKHOUSE_SPLIT_BINARY) - set (TEST_USE_BINARIES CLICKHOUSE_TESTS_SERVER_BIN_PATH=${ClickHouse_BINARY_DIR}/programs/clickhouse-server CLICKHOUSE_TESTS_CLIENT_BIN_PATH=${ClickHouse_BINARY_DIR}/programs/clickhouse-client) -else() - set (TEST_USE_BINARIES CLICKHOUSE_TESTS_SERVER_BIN_PATH=${ClickHouse_BINARY_DIR}/programs/clickhouse CLICKHOUSE_TESTS_CLIENT_BIN_PATH=${ClickHouse_BINARY_DIR}/programs/clickhouse) -endif() +set (TEST_USE_BINARIES CLICKHOUSE_TESTS_SERVER_BIN_PATH=${ClickHouse_BINARY_DIR}/programs/clickhouse CLICKHOUSE_TESTS_CLIENT_BIN_PATH=${ClickHouse_BINARY_DIR}/programs/clickhouse) find_program(DOCKER_CMD docker) find_program(DOCKER_COMPOSE_CMD docker-compose) From 22d8e532ed003c4d6a6a25c02bb90762c94d3cd4 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 1 Jul 2022 16:36:20 +0300 Subject: [PATCH 052/139] Fix ORDER BY that matches projections ORDER BY In case of projection is seleted and it is comlete (all parts has such projection) and ORDER BY matches projection ORDER BY, and optimize_read_in_order=1 (default), then the sorting is simply not done, because projections has separate plan, and so InterpreterSelectQuery::executeOrder() -> SortingStep has pipeline.getNumStreams() == 0 and it cannot do sorting. Fix this, by adding sorting when creating plan for reading from projections. Signed-off-by: Azat Khuzhin --- src/Interpreters/InterpreterSelectQuery.cpp | 13 ++++++-- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 32 +++++++++++++++---- src/Storages/SelectQueryInfo.h | 7 ++-- ...10_order_by_projections_complete.reference | 13 ++++++++ .../01710_order_by_projections_complete.sql | 14 ++++++++ ..._order_by_projections_incomplete.reference | 13 ++++++++ .../01710_order_by_projections_incomplete.sql | 16 ++++++++++ ...10_projections_order_by_complete.reference | 13 ++++++++ .../01710_projections_order_by_complete.sql | 16 ++++++++++ ..._projections_order_by_incomplete.reference | 13 ++++++++ .../01710_projections_order_by_incomplete.sql | 16 ++++++++++ 11 files changed, 154 insertions(+), 12 deletions(-) create mode 100644 tests/queries/0_stateless/01710_order_by_projections_complete.reference create mode 100644 tests/queries/0_stateless/01710_order_by_projections_complete.sql create mode 100644 tests/queries/0_stateless/01710_order_by_projections_incomplete.reference create mode 100644 tests/queries/0_stateless/01710_order_by_projections_incomplete.sql create mode 100644 tests/queries/0_stateless/01710_projections_order_by_complete.reference create mode 100644 tests/queries/0_stateless/01710_projections_order_by_complete.sql create mode 100644 tests/queries/0_stateless/01710_projections_order_by_incomplete.reference create mode 100644 tests/queries/0_stateless/01710_projections_order_by_incomplete.sql diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index de01115abec..7663b040323 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -748,13 +748,20 @@ BlockIO InterpreterSelectQuery::execute() Block InterpreterSelectQuery::getSampleBlockImpl() { + auto & select_query = getSelectQuery(); + query_info.query = query_ptr; + + /// NOTE: this is required for getQueryProcessingStage(), so should be initialized before ExpressionAnalysisResult. query_info.has_window = query_analyzer->hasWindow(); + /// NOTE: this is required only for IStorage::read(), and to be precise MergeTreeData::read(), in case of projections. + query_info.has_order_by = select_query.orderBy() != nullptr; + query_info.need_aggregate = query_analyzer->hasAggregation(); + if (storage && !options.only_analyze) { - auto & query = getSelectQuery(); - query_analyzer->makeSetsForIndex(query.where()); - query_analyzer->makeSetsForIndex(query.prewhere()); + query_analyzer->makeSetsForIndex(select_query.where()); + query_analyzer->makeSetsForIndex(select_query.prewhere()); query_info.sets = std::move(query_analyzer->getPreparedSets()); query_info.subquery_for_sets = std::move(query_analyzer->getSubqueriesForSets()); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 3916eae1556..d8e96992eca 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -8,13 +8,16 @@ #include #include #include +#include #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -26,7 +29,9 @@ #include #include #include +#include #include +#include #include #include @@ -35,12 +40,7 @@ #include #include #include -#include -#include - -#include -#include #include namespace DB @@ -184,6 +184,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( query_info.projection->desc->type, query_info.projection->desc->name); + const ASTSelectQuery & select_query = query_info.query->as(); QueryPlanResourceHolder resources; auto projection_plan = std::make_unique(); @@ -230,6 +231,25 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( expression_before_aggregation->setStepDescription("Before GROUP BY"); projection_plan->addStep(std::move(expression_before_aggregation)); } + + /// NOTE: input_order_info (for projection and not) is set only if projection is complete + if (query_info.has_order_by && !query_info.need_aggregate && query_info.projection->input_order_info) + { + chassert(query_info.projection->complete); + + SortDescription output_order_descr = InterpreterSelectQuery::getSortDescription(select_query, context); + UInt64 limit = InterpreterSelectQuery::getLimitForSorting(select_query, context); + + auto sorting_step = std::make_unique( + projection_plan->getCurrentDataStream(), + query_info.projection->input_order_info->order_key_prefix_descr, + output_order_descr, + settings.max_block_size, + limit); + + sorting_step->setStepDescription("ORDER BY for projections"); + projection_plan->addStep(std::move(sorting_step)); + } } auto ordinary_query_plan = std::make_unique(); @@ -365,7 +385,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( InputOrderInfoPtr group_by_info = query_info.projection->input_order_info; SortDescription group_by_sort_description; if (group_by_info && settings.optimize_aggregation_in_order) - group_by_sort_description = getSortDescriptionFromGroupBy(query_info.query->as()); + group_by_sort_description = getSortDescriptionFromGroupBy(select_query); else group_by_info = nullptr; diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index 5046a0b6fe0..36eea35e728 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -154,8 +154,6 @@ struct SelectQueryInfoBase TreeRewriterResultPtr syntax_analyzer_result; - PrewhereInfoPtr prewhere_info; - /// This is an additional filer applied to current table. /// It is needed only for additional PK filtering. ASTPtr additional_filter_ast; @@ -168,8 +166,11 @@ struct SelectQueryInfoBase /// Example: x IN (1, 2, 3) PreparedSets sets; - /// Cached value of ExpressionAnalysisResult::has_window + /// Cached value of ExpressionAnalysisResult bool has_window = false; + bool has_order_by = false; + bool need_aggregate = false; + PrewhereInfoPtr prewhere_info; ClusterPtr getCluster() const { return !optimized_cluster ? cluster : optimized_cluster; } diff --git a/tests/queries/0_stateless/01710_order_by_projections_complete.reference b/tests/queries/0_stateless/01710_order_by_projections_complete.reference new file mode 100644 index 00000000000..a51d1ce1f3b --- /dev/null +++ b/tests/queries/0_stateless/01710_order_by_projections_complete.reference @@ -0,0 +1,13 @@ +-- { echoOn } +select t from data_order_by_proj_comp where t > 0 order by t settings optimize_read_in_order=1; +5 +5 +6 +select t from data_order_by_proj_comp where t > 0 order by t settings optimize_read_in_order=0; +5 +5 +6 +select t from data_order_by_proj_comp where t > 0 order by t settings max_threads=1; +5 +5 +6 diff --git a/tests/queries/0_stateless/01710_order_by_projections_complete.sql b/tests/queries/0_stateless/01710_order_by_projections_complete.sql new file mode 100644 index 00000000000..86a553569a2 --- /dev/null +++ b/tests/queries/0_stateless/01710_order_by_projections_complete.sql @@ -0,0 +1,14 @@ +drop table if exists data_order_by_proj_comp; +create table data_order_by_proj_comp (t UInt64, projection tSort (select * order by t)) ENGINE MergeTree() order by t; + +system stop merges data_order_by_proj_comp; + +insert into data_order_by_proj_comp values (5); +insert into data_order_by_proj_comp values (5); +insert into data_order_by_proj_comp values (6); + +-- { echoOn } +select t from data_order_by_proj_comp where t > 0 order by t settings optimize_read_in_order=1; +select t from data_order_by_proj_comp where t > 0 order by t settings optimize_read_in_order=0; +select t from data_order_by_proj_comp where t > 0 order by t settings max_threads=1; +-- { echoOff } diff --git a/tests/queries/0_stateless/01710_order_by_projections_incomplete.reference b/tests/queries/0_stateless/01710_order_by_projections_incomplete.reference new file mode 100644 index 00000000000..4a90aad2d6f --- /dev/null +++ b/tests/queries/0_stateless/01710_order_by_projections_incomplete.reference @@ -0,0 +1,13 @@ +-- { echoOn } +select t from data_order_by_proj_incomp where t > 0 order by t settings optimize_read_in_order=1; +5 +5 +6 +select t from data_order_by_proj_incomp where t > 0 order by t settings optimize_read_in_order=0; +5 +5 +6 +select t from data_order_by_proj_incomp where t > 0 order by t settings max_threads=1; +5 +5 +6 diff --git a/tests/queries/0_stateless/01710_order_by_projections_incomplete.sql b/tests/queries/0_stateless/01710_order_by_projections_incomplete.sql new file mode 100644 index 00000000000..17ae7128694 --- /dev/null +++ b/tests/queries/0_stateless/01710_order_by_projections_incomplete.sql @@ -0,0 +1,16 @@ +drop table if exists data_order_by_proj_incomp; +create table data_order_by_proj_incomp (t UInt64) ENGINE MergeTree() order by t; + +system stop merges data_order_by_proj_incomp; + +insert into data_order_by_proj_incomp values (5); +insert into data_order_by_proj_incomp values (5); + +alter table data_order_by_proj_incomp add projection tSort (select * order by t); +insert into data_order_by_proj_incomp values (6); + +-- { echoOn } +select t from data_order_by_proj_incomp where t > 0 order by t settings optimize_read_in_order=1; +select t from data_order_by_proj_incomp where t > 0 order by t settings optimize_read_in_order=0; +select t from data_order_by_proj_incomp where t > 0 order by t settings max_threads=1; +-- { echoOff } diff --git a/tests/queries/0_stateless/01710_projections_order_by_complete.reference b/tests/queries/0_stateless/01710_projections_order_by_complete.reference new file mode 100644 index 00000000000..f8d6ccd5f11 --- /dev/null +++ b/tests/queries/0_stateless/01710_projections_order_by_complete.reference @@ -0,0 +1,13 @@ +-- { echoOn } +select t from data_proj_order_by_comp where t > 0 order by t settings optimize_read_in_order=1; +5 +5 +6 +select t from data_proj_order_by_comp where t > 0 order by t settings optimize_read_in_order=0; +5 +5 +6 +select t from data_proj_order_by_comp where t > 0 order by t settings max_threads=1; +5 +5 +6 diff --git a/tests/queries/0_stateless/01710_projections_order_by_complete.sql b/tests/queries/0_stateless/01710_projections_order_by_complete.sql new file mode 100644 index 00000000000..d8126d32b43 --- /dev/null +++ b/tests/queries/0_stateless/01710_projections_order_by_complete.sql @@ -0,0 +1,16 @@ +-- Test from https://github.com/ClickHouse/ClickHouse/issues/37673 + +drop table if exists data_proj_order_by_comp; +create table data_proj_order_by_comp (t UInt64, projection tSort (select * order by t)) ENGINE MergeTree() order by tuple(); + +system stop merges data_proj_order_by_comp; + +insert into data_proj_order_by_comp values (5); +insert into data_proj_order_by_comp values (5); +insert into data_proj_order_by_comp values (6); + +-- { echoOn } +select t from data_proj_order_by_comp where t > 0 order by t settings optimize_read_in_order=1; +select t from data_proj_order_by_comp where t > 0 order by t settings optimize_read_in_order=0; +select t from data_proj_order_by_comp where t > 0 order by t settings max_threads=1; +-- { echoOff } diff --git a/tests/queries/0_stateless/01710_projections_order_by_incomplete.reference b/tests/queries/0_stateless/01710_projections_order_by_incomplete.reference new file mode 100644 index 00000000000..368a11a5273 --- /dev/null +++ b/tests/queries/0_stateless/01710_projections_order_by_incomplete.reference @@ -0,0 +1,13 @@ +-- { echoOn } +select t from data_proj_order_by_incomp where t > 0 order by t settings optimize_read_in_order=1; +5 +5 +6 +select t from data_proj_order_by_incomp where t > 0 order by t settings optimize_read_in_order=0; +5 +5 +6 +select t from data_proj_order_by_incomp where t > 0 order by t settings max_threads=1; +5 +5 +6 diff --git a/tests/queries/0_stateless/01710_projections_order_by_incomplete.sql b/tests/queries/0_stateless/01710_projections_order_by_incomplete.sql new file mode 100644 index 00000000000..3ac5f22afa7 --- /dev/null +++ b/tests/queries/0_stateless/01710_projections_order_by_incomplete.sql @@ -0,0 +1,16 @@ +drop table if exists data_proj_order_by_incomp; +create table data_proj_order_by_incomp (t UInt64) ENGINE MergeTree() order by tuple(); + +system stop merges data_proj_order_by_incomp; + +insert into data_proj_order_by_incomp values (5); +insert into data_proj_order_by_incomp values (5); + +alter table data_proj_order_by_incomp add projection tSort (select * order by t); +insert into data_proj_order_by_incomp values (6); + +-- { echoOn } +select t from data_proj_order_by_incomp where t > 0 order by t settings optimize_read_in_order=1; +select t from data_proj_order_by_incomp where t > 0 order by t settings optimize_read_in_order=0; +select t from data_proj_order_by_incomp where t > 0 order by t settings max_threads=1; +-- { echoOff } From 41e72aac838273ae4c1529e9af242b5ffc1bf38b Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Sun, 24 Jul 2022 21:35:36 +0000 Subject: [PATCH 053/139] Fix: DistinctSortedTransform doesn't take advantage of sorting clearing_columns are set incorrectly, so we never clear HashSet --- .../Transforms/DistinctSortedTransform.cpp | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/Processors/Transforms/DistinctSortedTransform.cpp b/src/Processors/Transforms/DistinctSortedTransform.cpp index 73e89c774ae..1a524c73e57 100644 --- a/src/Processors/Transforms/DistinctSortedTransform.cpp +++ b/src/Processors/Transforms/DistinctSortedTransform.cpp @@ -143,16 +143,23 @@ ColumnRawPtrs DistinctSortedTransform::getClearingColumns(const ColumnRawPtrs & { ColumnRawPtrs clearing_hint_columns; clearing_hint_columns.reserve(description.size()); - for (const auto & sort_column_description : description) + try { - const auto * sort_column_ptr = header.getByName(sort_column_description.column_name).column.get(); - const auto it = std::find(key_columns.cbegin(), key_columns.cend(), sort_column_ptr); - if (it != key_columns.cend()) /// if found in key_columns - clearing_hint_columns.emplace_back(sort_column_ptr); - else - return clearing_hint_columns; /// We will use common prefix of sort description and requested DISTINCT key. + const size_t max_num_of_clearing_columns = std::min(description.size(), column_positions.size()); + for (size_t i = 0; i < max_num_of_clearing_columns; ++i) + { + const auto pos = header.getPositionByName(description[i].column_name); // can throw + if (std::find(begin(column_positions), end(column_positions), pos) == column_positions.end()) + break; + + clearing_hint_columns.emplace_back(key_columns[pos]); + } + return clearing_hint_columns; + } + catch (const Exception &) + { + return clearing_hint_columns; } - return clearing_hint_columns; } bool DistinctSortedTransform::rowsEqual(const ColumnRawPtrs & lhs, size_t n, const ColumnRawPtrs & rhs, size_t m) From 5829e47adb215f5548ce3017f8ff28acef392c94 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Sun, 24 Jul 2022 22:27:02 +0000 Subject: [PATCH 054/139] Add test: check the optimized DISTINCT output is the same as for ordinary one --- .../00174_distinct_in_order.reference | 3 +++ .../1_stateful/00174_distinct_in_order.sql | 25 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 tests/queries/1_stateful/00174_distinct_in_order.reference create mode 100644 tests/queries/1_stateful/00174_distinct_in_order.sql diff --git a/tests/queries/1_stateful/00174_distinct_in_order.reference b/tests/queries/1_stateful/00174_distinct_in_order.reference new file mode 100644 index 00000000000..c0b2aadc20e --- /dev/null +++ b/tests/queries/1_stateful/00174_distinct_in_order.reference @@ -0,0 +1,3 @@ +-- check that distinct with and w/o optimization produce the same result +-- DISTINCT colums are the same as in ORDER BY +-- DISTINCT colums has prefix in ORDER BY columns diff --git a/tests/queries/1_stateful/00174_distinct_in_order.sql b/tests/queries/1_stateful/00174_distinct_in_order.sql new file mode 100644 index 00000000000..6cc21f8a445 --- /dev/null +++ b/tests/queries/1_stateful/00174_distinct_in_order.sql @@ -0,0 +1,25 @@ +select '-- check that distinct with and w/o optimization produce the same result'; + +drop table if exists distinct_in_order sync; +drop table if exists ordinary_distinct sync; + +select '-- DISTINCT colums are the same as in ORDER BY'; +create table distinct_in_order (CounterID UInt32, EventDate Date) engine=MergeTree() order by (CounterID, EventDate); +insert into distinct_in_order select distinct CounterID, EventDate from hits_v1 order by CounterID, EventDate settings optimize_distinct_in_order=1; +create table ordinary_distinct (CounterID UInt32, EventDate Date) engine=MergeTree() order by (CounterID, EventDate); +insert into ordinary_distinct select distinct CounterID, EventDate from hits_v1 order by CounterID, EventDate settings optimize_distinct_in_order=0; +select distinct * from distinct_in_order except select * from ordinary_distinct; + +drop table if exists distinct_in_order sync; +drop table if exists ordinary_distinct sync; + +select '-- DISTINCT colums has prefix in ORDER BY columns'; +create table distinct_in_order (CounterID UInt32, EventDate Date) engine=MergeTree() order by (CounterID, EventDate); +insert into distinct_in_order select distinct CounterID, EventDate from hits_v1 order by CounterID settings optimize_distinct_in_order=1; +create table ordinary_distinct (CounterID UInt32, EventDate Date) engine=MergeTree() order by (CounterID, EventDate); +insert into ordinary_distinct select distinct CounterID, EventDate from hits_v1 order by CounterID settings optimize_distinct_in_order=0; +select distinct * from distinct_in_order except select * from ordinary_distinct; + +drop table if exists distinct_in_order sync; +drop table if exists ordinary_distinct sync; + From f574a00e7b42510696093ee0a4050323e4541832 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Mon, 25 Jul 2022 07:56:50 +0000 Subject: [PATCH 055/139] Fix test --- tests/queries/1_stateful/00174_distinct_in_order.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/1_stateful/00174_distinct_in_order.sql b/tests/queries/1_stateful/00174_distinct_in_order.sql index 6cc21f8a445..e9084afc849 100644 --- a/tests/queries/1_stateful/00174_distinct_in_order.sql +++ b/tests/queries/1_stateful/00174_distinct_in_order.sql @@ -5,9 +5,9 @@ drop table if exists ordinary_distinct sync; select '-- DISTINCT colums are the same as in ORDER BY'; create table distinct_in_order (CounterID UInt32, EventDate Date) engine=MergeTree() order by (CounterID, EventDate); -insert into distinct_in_order select distinct CounterID, EventDate from hits_v1 order by CounterID, EventDate settings optimize_distinct_in_order=1; +insert into distinct_in_order select distinct CounterID, EventDate from test.hits order by CounterID, EventDate settings optimize_distinct_in_order=1; create table ordinary_distinct (CounterID UInt32, EventDate Date) engine=MergeTree() order by (CounterID, EventDate); -insert into ordinary_distinct select distinct CounterID, EventDate from hits_v1 order by CounterID, EventDate settings optimize_distinct_in_order=0; +insert into ordinary_distinct select distinct CounterID, EventDate from test.hits order by CounterID, EventDate settings optimize_distinct_in_order=0; select distinct * from distinct_in_order except select * from ordinary_distinct; drop table if exists distinct_in_order sync; @@ -15,9 +15,9 @@ drop table if exists ordinary_distinct sync; select '-- DISTINCT colums has prefix in ORDER BY columns'; create table distinct_in_order (CounterID UInt32, EventDate Date) engine=MergeTree() order by (CounterID, EventDate); -insert into distinct_in_order select distinct CounterID, EventDate from hits_v1 order by CounterID settings optimize_distinct_in_order=1; +insert into distinct_in_order select distinct CounterID, EventDate from test.hits order by CounterID settings optimize_distinct_in_order=1; create table ordinary_distinct (CounterID UInt32, EventDate Date) engine=MergeTree() order by (CounterID, EventDate); -insert into ordinary_distinct select distinct CounterID, EventDate from hits_v1 order by CounterID settings optimize_distinct_in_order=0; +insert into ordinary_distinct select distinct CounterID, EventDate from test.hits order by CounterID settings optimize_distinct_in_order=0; select distinct * from distinct_in_order except select * from ordinary_distinct; drop table if exists distinct_in_order sync; From 5503d92d97080b16399b83c2e16274e8207f68f0 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 25 Jul 2022 11:30:23 +0000 Subject: [PATCH 056/139] Fix submodules --- contrib/arrow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/arrow b/contrib/arrow index 87fbdebf43c..4391dbaac4a 160000 --- a/contrib/arrow +++ b/contrib/arrow @@ -1 +1 @@ -Subproject commit 87fbdebf43ccc44ce1c0204ba40b314a1358766a +Subproject commit 4391dbaac4a39f21923004cb7bc31f325bdd2cb9 From 22219c4b7acea3ca54b873481c7ecf41571c635a Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 24 Jul 2022 19:03:43 +0000 Subject: [PATCH 057/139] Merge Woboq code browser page into "Getting Started" document There was too little information on the page to deserve a separate document. --- docs/en/development/browse-code.md | 13 ------------- docs/en/development/developer-instruction.md | 6 ++++++ docs/ru/development/browse-code.md | 13 ------------- docs/ru/development/developer-instruction.md | 8 ++++++++ docs/zh/development/browse-code.md | 12 ------------ docs/zh/development/developer-instruction.md | 8 ++++++++ 6 files changed, 22 insertions(+), 38 deletions(-) delete mode 100644 docs/en/development/browse-code.md delete mode 100644 docs/ru/development/browse-code.md delete mode 100644 docs/zh/development/browse-code.md diff --git a/docs/en/development/browse-code.md b/docs/en/development/browse-code.md deleted file mode 100644 index da924c359ff..00000000000 --- a/docs/en/development/browse-code.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -sidebar_label: Source Code Browser -sidebar_position: 72 -description: Various ways to browse and edit the source code ---- - -# Browse ClickHouse Source Code - -You can use the **Woboq** online code browser available [here](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). It provides code navigation and semantic highlighting, search and indexing. The code snapshot is updated daily. - -Also, you can browse sources on [GitHub](https://github.com/ClickHouse/ClickHouse) as usual. - -If you’re interested what IDE to use, we recommend CLion, QT Creator, VS Code and KDevelop (with caveats). You can use any favorite IDE. Vim and Emacs also count. diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md index ea2ed95fd27..b496f7f0d63 100644 --- a/docs/en/development/developer-instruction.md +++ b/docs/en/development/developer-instruction.md @@ -267,6 +267,12 @@ The system will prepare ClickHouse binary builds for your pull request individua Most probably some of the builds will fail at first times. This is due to the fact that we check builds both with gcc as well as with clang, with almost all of existing warnings (always with the `-Werror` flag) enabled for clang. On that same page, you can find all of the build logs so that you do not have to build ClickHouse in all of the possible ways. +## Browse ClickHouse Source Code + +You can use the **Woboq** online code browser available [here](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). It provides code navigation, semantic highlighting, search and indexing. The code snapshot is updated daily. + +Also, you can browse sources on [GitHub](https://github.com/ClickHouse/ClickHouse) as usual. + ## Faster builds for development: Split build configuration {#split-build} ClickHouse is normally statically linked into a single static `clickhouse` binary with minimal dependencies. This is convenient for distribution, but it means that for every change the entire binary needs to be re-linked, which is slow and inconvenient for development. As an alternative, you can instead build dynamically linked shared libraries and separate binaries `clickhouse-server`, `clickhouse-client` etc., allowing for faster incremental builds. To use it, add the following flags to your `cmake` invocation: diff --git a/docs/ru/development/browse-code.md b/docs/ru/development/browse-code.md deleted file mode 100644 index 7290eed5c6f..00000000000 --- a/docs/ru/development/browse-code.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -sidebar_position: 72 -sidebar_label: "Навигация по коду ClickHouse" ---- - - -# Навигация по коду ClickHouse {#navigatsiia-po-kodu-clickhouse} - -Для навигации по коду онлайн доступен **Woboq**, он расположен [здесь](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). В нём реализовано удобное перемещение между исходными файлами, семантическая подсветка, подсказки, индексация и поиск. Слепок кода обновляется ежедневно. - -Также вы можете просматривать исходники на [GitHub](https://github.com/ClickHouse/ClickHouse). - -Если вы интересуетесь, какую среду разработки выбрать для работы с ClickHouse, мы рекомендуем CLion, QT Creator, VSCode или KDevelop (с некоторыми предостережениями). Вы можете использовать свою любимую среду разработки, Vim и Emacs тоже считаются. diff --git a/docs/ru/development/developer-instruction.md b/docs/ru/development/developer-instruction.md index 4c1981198a0..f5ab32f4bed 100644 --- a/docs/ru/development/developer-instruction.md +++ b/docs/ru/development/developer-instruction.md @@ -285,3 +285,11 @@ Pull request можно создать, даже если работа над з Система подготовит сборки ClickHouse специально для вашего pull request. Для их получения, нажмите на ссылку «Details» у проверки «Clickhouse build check». Там вы сможете найти прямые ссылки на собранные .deb пакеты ClickHouse, которые, при желании, вы даже сможете установить на свои продакшен серверы (если не страшно). Вероятнее всего, часть сборок не будет успешной с первого раза. Ведь мы проверяем сборку кода и gcc и clang, а при сборке с помощью clang включаются почти все существующие в природе warnings (всегда с флагом `-Werror`). На той же странице, вы сможете найти логи сборки - вам не обязательно самому собирать ClickHouse всеми возможными способами. + +## Навигация по коду ClickHouse {#navigatsiia-po-kodu-clickhouse} + +Для навигации по коду онлайн доступен **Woboq**, он расположен [здесь](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). В нём реализовано удобное перемещение между исходными файлами, семантическая подсветка, подсказки, индексация и поиск. Слепок кода обновляется ежедневно. + +Также вы можете просматривать исходники на [GitHub](https://github.com/ClickHouse/ClickHouse). + +Если вы интересуетесь, какую среду разработки выбрать для работы с ClickHouse, мы рекомендуем CLion, QT Creator, VSCode или KDevelop (с некоторыми предостережениями). Вы можете использовать свою любимую среду разработки, Vim и Emacs тоже считаются. diff --git a/docs/zh/development/browse-code.md b/docs/zh/development/browse-code.md deleted file mode 100644 index 7610589e171..00000000000 --- a/docs/zh/development/browse-code.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -sidebar_position: 63 -sidebar_label: "\u6D4F\u89C8\u6E90\u4EE3\u7801" ---- - -# 浏览ClickHouse源代码 {#browse-clickhouse-source-code} - -您可以使用 **Woboq** 在线代码浏览器 [点击这里](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). 它提供了代码导航和语义突出显示、搜索和索引。 代码快照每天更新。 - -此外，您还可以像往常一样浏览源代码 [GitHub](https://github.com/ClickHouse/ClickHouse) - -如果你希望了解哪种IDE较好，我们推荐使用CLion，QT Creator，VS Code和KDevelop（有注意事项）。 您可以使用任何您喜欢的IDE。 Vim和Emacs也可以。 diff --git a/docs/zh/development/developer-instruction.md b/docs/zh/development/developer-instruction.md index 7ade3ad57fb..5ba6e31099e 100644 --- a/docs/zh/development/developer-instruction.md +++ b/docs/zh/development/developer-instruction.md @@ -264,3 +264,11 @@ ClickHouse成员一旦在您的拉取请求上贴上«可以测试»标签，就 系统将分别为您的拉取请求准备ClickHouse二进制版本。若要检索这些构建信息，请在检查列表中单击« ClickHouse构建检查»旁边的«详细信息»链接。在这里，您会找到指向ClickHouse的.deb软件包的直接链接，此外，甚至可以将其部署在生产服务器上（如果您不担心）。 某些构建项很可能会在首次构建时失败。这是因为我们同时检查了基于gcc和clang的构建，几乎所有现有的被clang启用的警告（总是带有`-Werror`标志）。在同一页面上，您可以找到所有构建的日志，因此不必以所有可能的方式构建ClickHouse。 + +## 浏览ClickHouse源代码 {#browse-clickhouse-source-code} + +您可以使用 **Woboq** 在线代码浏览器 [点击这里](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). 它提供了代码导航和语义突出显示、搜索和索引。 代码快照每天更新。 + +此外，您还可以像往常一样浏览源代码 [GitHub](https://github.com/ClickHouse/ClickHouse) + +如果你希望了解哪种IDE较好，我们推荐使用CLion，QT Creator，VS Code和KDevelop（有注意事项）。 您可以使用任何您喜欢的IDE。 Vim和Emacs也可以。 From 9302d67fc950ea8b5463c3438cf6aa8d7962180e Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 26 Jul 2022 10:30:12 +0000 Subject: [PATCH 058/139] Fix special build --- contrib/arrow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/arrow b/contrib/arrow index 4391dbaac4a..14976b7b205 160000 --- a/contrib/arrow +++ b/contrib/arrow @@ -1 +1 @@ -Subproject commit 4391dbaac4a39f21923004cb7bc31f325bdd2cb9 +Subproject commit 14976b7b2057ad0c0368705378682c625742b2fd From c5bbf4449250df51c75e50e09f7ad2f1ef4d5872 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 26 Jul 2022 10:58:10 +0000 Subject: [PATCH 059/139] Update NuRaft --- contrib/NuRaft | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/NuRaft b/contrib/NuRaft index 1334b9ae725..b924bb097a9 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit 1334b9ae72576821a698d657d08838861cf33007 +Subproject commit b924bb097a9bc1c19f52a60a2265bede0a567b2f From 5ad83143674a2f1bbcbb0598ffdd304e41135519 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 26 Jul 2022 23:55:13 +0800 Subject: [PATCH 060/139] Combining sumIf->countIf and multiIf->if opt. --- src/Interpreters/TreeOptimizer.cpp | 6 +++--- .../02374_combine_multi_if_and_count_if_opt.reference | 2 ++ .../02374_combine_multi_if_and_count_if_opt.sql | 9 +++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.reference create mode 100644 tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.sql diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index cf79cd158e8..d8ebc836a05 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -823,12 +823,12 @@ void TreeOptimizer::apply(ASTPtr & query, TreeRewriterResult & result, if (settings.optimize_normalize_count_variants) optimizeCountConstantAndSumOne(query); - if (settings.optimize_rewrite_sum_if_to_count_if) - optimizeSumIfFunctions(query); - if (settings.optimize_multiif_to_if) optimizeMultiIfToIf(query); + if (settings.optimize_rewrite_sum_if_to_count_if) + optimizeSumIfFunctions(query); + /// Remove injective functions inside uniq if (settings.optimize_injective_functions_inside_uniq) optimizeInjectiveFunctionsInsideUniq(query, context); diff --git a/tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.reference b/tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.reference new file mode 100644 index 00000000000..d7efa9ef54c --- /dev/null +++ b/tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.reference @@ -0,0 +1,2 @@ +SELECT countIf(a = 1) +FROM m diff --git a/tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.sql b/tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.sql new file mode 100644 index 00000000000..06e1e4bfd55 --- /dev/null +++ b/tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.sql @@ -0,0 +1,9 @@ +drop table if exists m; + +create table m (a int) engine Log; + +insert into m values (1); + +explain syntax select sum(multiIf(a = 1, 1, 0)) from m; + +drop table m; From 3bc9e1bd162647b0c2649eb4359c556e69c7c77d Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Tue, 26 Jul 2022 20:06:23 +0300 Subject: [PATCH 061/139] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 1d0146582a6..8c132d5d167 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,7 @@ ClickHouse® is an open-source column-oriented database management system that a * [Contacts](https://clickhouse.com/company/#contact) can help to get your questions answered if there are any. ## Upcoming events -* [v22.7 Release Webinar](https://clickhouse.com/company/events/v22-7-release-webinar/) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap. -* [ClickHouse Meetup at the Cloudflare office in London](https://www.meetup.com/clickhouse-london-user-group/events/286891586/) ClickHouse meetup at the Cloudflare office space in central London + * [ClickHouse Meetup at the Metoda office in Munich](https://www.meetup.com/clickhouse-meetup-munich/events/286891667/) ClickHouse meetup at the Metoda office in Munich From d196ab24d4cc9d532d7be4ac2cb73e7c353a4c6f Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Tue, 26 Jul 2022 19:55:29 +0000 Subject: [PATCH 062/139] Calculate DISTINCT columns positions which form sort prefix in sort description once --- .../Transforms/DistinctSortedTransform.cpp | 36 ++++++++++--------- .../Transforms/DistinctSortedTransform.h | 4 ++- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/Processors/Transforms/DistinctSortedTransform.cpp b/src/Processors/Transforms/DistinctSortedTransform.cpp index 1a524c73e57..6d7cf79ce07 100644 --- a/src/Processors/Transforms/DistinctSortedTransform.cpp +++ b/src/Processors/Transforms/DistinctSortedTransform.cpp @@ -27,6 +27,21 @@ DistinctSortedTransform::DistinctSortedTransform( if (col && !isColumnConst(*col)) column_positions.emplace_back(pos); } + /// pre-calculate DISTINCT column positions which form sort prefix of sort description + sort_prefix_positions.reserve(description.size()); + for (const auto & column_sort_descr : description) + { + /// check if there is such column in header + if (!header.has(column_sort_descr.column_name)) + break; + + /// check if sorted column position matches any DISTINCT column + const auto pos = header.getPositionByName(column_sort_descr.column_name); + if (std::find(begin(column_positions), end(column_positions), pos) == column_positions.end()) + break; + + sort_prefix_positions.emplace_back(pos); + } } void DistinctSortedTransform::transform(Chunk & chunk) @@ -142,24 +157,11 @@ ColumnRawPtrs DistinctSortedTransform::getKeyColumns(const Chunk & chunk) const ColumnRawPtrs DistinctSortedTransform::getClearingColumns(const ColumnRawPtrs & key_columns) const { ColumnRawPtrs clearing_hint_columns; - clearing_hint_columns.reserve(description.size()); - try - { - const size_t max_num_of_clearing_columns = std::min(description.size(), column_positions.size()); - for (size_t i = 0; i < max_num_of_clearing_columns; ++i) - { - const auto pos = header.getPositionByName(description[i].column_name); // can throw - if (std::find(begin(column_positions), end(column_positions), pos) == column_positions.end()) - break; + clearing_hint_columns.reserve(sort_prefix_positions.size()); + for (const auto pos : sort_prefix_positions) + clearing_hint_columns.emplace_back(key_columns[pos]); - clearing_hint_columns.emplace_back(key_columns[pos]); - } - return clearing_hint_columns; - } - catch (const Exception &) - { - return clearing_hint_columns; - } + return clearing_hint_columns; } bool DistinctSortedTransform::rowsEqual(const ColumnRawPtrs & lhs, size_t n, const ColumnRawPtrs & rhs, size_t m) diff --git a/src/Processors/Transforms/DistinctSortedTransform.h b/src/Processors/Transforms/DistinctSortedTransform.h index 72acde0716b..3ba1db5d429 100644 --- a/src/Processors/Transforms/DistinctSortedTransform.h +++ b/src/Processors/Transforms/DistinctSortedTransform.h @@ -59,7 +59,9 @@ private: PreviousChunk prev_chunk; Names column_names; - ColumnNumbers column_positions; + ColumnNumbers column_positions; /// DISTINCT columns positions in header + ColumnNumbers sort_prefix_positions; /// DISTINCT columns positions which form sort prefix of sort description + ClearableSetVariants data; Sizes key_sizes; UInt64 limit_hint; From 589104fa6e47aa1632ac444fb161c40494d7be96 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 27 Jul 2022 07:44:42 +0000 Subject: [PATCH 063/139] Make building column arrays for chunk processing straightforward --- .../Transforms/DistinctSortedTransform.cpp | 121 +++++++++--------- .../Transforms/DistinctSortedTransform.h | 4 - 2 files changed, 57 insertions(+), 68 deletions(-) diff --git a/src/Processors/Transforms/DistinctSortedTransform.cpp b/src/Processors/Transforms/DistinctSortedTransform.cpp index 6d7cf79ce07..cefb12b2252 100644 --- a/src/Processors/Transforms/DistinctSortedTransform.cpp +++ b/src/Processors/Transforms/DistinctSortedTransform.cpp @@ -46,58 +46,73 @@ DistinctSortedTransform::DistinctSortedTransform( void DistinctSortedTransform::transform(Chunk & chunk) { - const ColumnRawPtrs column_ptrs(getKeyColumns(chunk)); - if (column_ptrs.empty()) - return; + if (!chunk.hasRows()) + return; - ColumnRawPtrs clearing_hint_columns(getClearingColumns(column_ptrs)); + /// get DISTINCT columns from chunk + ColumnRawPtrs column_ptrs; + column_ptrs.reserve(column_positions.size()); + for (const auto pos : column_positions) + { + const auto & column = chunk.getColumns()[pos]; + column_ptrs.emplace_back(column.get()); + } - if (data.type == ClearableSetVariants::Type::EMPTY) - data.init(ClearableSetVariants::chooseMethod(column_ptrs, key_sizes)); + /// get DISTINCT columns from chunk which form sort prefix of sort description + ColumnRawPtrs clearing_hint_columns; + clearing_hint_columns.reserve(sort_prefix_positions.size()); + for (const auto pos : sort_prefix_positions) + { + const auto & column = chunk.getColumns()[pos]; + clearing_hint_columns.emplace_back(column.get()); + } - const size_t rows = chunk.getNumRows(); - IColumn::Filter filter(rows); + if (data.type == ClearableSetVariants::Type::EMPTY) + data.init(ClearableSetVariants::chooseMethod(column_ptrs, key_sizes)); - bool has_new_data = false; - switch (data.type) - { - case ClearableSetVariants::Type::EMPTY: - break; - #define M(NAME) \ - case ClearableSetVariants::Type::NAME: \ - has_new_data = buildFilter(*data.NAME, column_ptrs, clearing_hint_columns, filter, rows, data); \ - break; - APPLY_FOR_SET_VARIANTS(M) - #undef M - } + const size_t rows = chunk.getNumRows(); + IColumn::Filter filter(rows); - /// Just go to the next block if there isn't any new record in the current one. - if (!has_new_data) - { - chunk.clear(); - return; - } + bool has_new_data = false; + switch (data.type) + { + case ClearableSetVariants::Type::EMPTY: + break; +#define M(NAME) \ + case ClearableSetVariants::Type::NAME: \ + has_new_data = buildFilter(*data.NAME, column_ptrs, clearing_hint_columns, filter, rows, data); \ + break; + APPLY_FOR_SET_VARIANTS(M) +#undef M + } - if (!set_size_limits.check(data.getTotalRowCount(), data.getTotalByteCount(), "DISTINCT", ErrorCodes::SET_SIZE_LIMIT_EXCEEDED)) - { - stopReading(); - chunk.clear(); - return; - } + /// Just go to the next block if there isn't any new record in the current one. + if (!has_new_data) + { + chunk.clear(); + return; + } - /// Stop reading if we already reached the limit. - if (limit_hint && data.getTotalRowCount() >= limit_hint) - stopReading(); + if (!set_size_limits.check(data.getTotalRowCount(), data.getTotalByteCount(), "DISTINCT", ErrorCodes::SET_SIZE_LIMIT_EXCEEDED)) + { + stopReading(); + chunk.clear(); + return; + } - prev_chunk.chunk = std::move(chunk); - prev_chunk.clearing_hint_columns = std::move(clearing_hint_columns); + /// Stop reading if we already reached the limit. + if (limit_hint && data.getTotalRowCount() >= limit_hint) + stopReading(); - size_t all_columns = prev_chunk.chunk.getNumColumns(); - Chunk res_chunk; - for (size_t i = 0; i < all_columns; ++i) - res_chunk.addColumn(prev_chunk.chunk.getColumns().at(i)->filter(filter, -1)); + prev_chunk.chunk = std::move(chunk); + prev_chunk.clearing_hint_columns = std::move(clearing_hint_columns); - chunk = std::move(res_chunk); + size_t all_columns = prev_chunk.chunk.getNumColumns(); + Chunk res_chunk; + for (size_t i = 0; i < all_columns; ++i) + res_chunk.addColumn(prev_chunk.chunk.getColumns().at(i)->filter(filter, -1)); + + chunk = std::move(res_chunk); } @@ -142,28 +157,6 @@ bool DistinctSortedTransform::buildFilter( return has_new_data; } -ColumnRawPtrs DistinctSortedTransform::getKeyColumns(const Chunk & chunk) const -{ - ColumnRawPtrs column_ptrs; - column_ptrs.reserve(column_positions.size()); - for (const auto pos : column_positions) - { - const auto & column = chunk.getColumns()[pos]; - column_ptrs.emplace_back(column.get()); - } - return column_ptrs; -} - -ColumnRawPtrs DistinctSortedTransform::getClearingColumns(const ColumnRawPtrs & key_columns) const -{ - ColumnRawPtrs clearing_hint_columns; - clearing_hint_columns.reserve(sort_prefix_positions.size()); - for (const auto pos : sort_prefix_positions) - clearing_hint_columns.emplace_back(key_columns[pos]); - - return clearing_hint_columns; -} - bool DistinctSortedTransform::rowsEqual(const ColumnRawPtrs & lhs, size_t n, const ColumnRawPtrs & rhs, size_t m) { for (size_t column_index = 0, num_columns = lhs.size(); column_index < num_columns; ++column_index) diff --git a/src/Processors/Transforms/DistinctSortedTransform.h b/src/Processors/Transforms/DistinctSortedTransform.h index 3ba1db5d429..11e3c22f708 100644 --- a/src/Processors/Transforms/DistinctSortedTransform.h +++ b/src/Processors/Transforms/DistinctSortedTransform.h @@ -32,10 +32,6 @@ protected: void transform(Chunk & chunk) override; private: - ColumnRawPtrs getKeyColumns(const Chunk & chunk) const; - /// When clearing_columns changed, we can clean HashSet to memory optimization - /// clearing_columns is a left-prefix of SortDescription exists in key_columns - ColumnRawPtrs getClearingColumns(const ColumnRawPtrs & key_columns) const; static bool rowsEqual(const ColumnRawPtrs & lhs, size_t n, const ColumnRawPtrs & rhs, size_t m); /// return true if has new data From 12a75674024b04bb94960361e53a0c8c988874d3 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 27 Jul 2022 07:58:54 +0000 Subject: [PATCH 064/139] Some polishing --- .../Transforms/DistinctSortedTransform.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Processors/Transforms/DistinctSortedTransform.cpp b/src/Processors/Transforms/DistinctSortedTransform.cpp index cefb12b2252..d34da9d72c9 100644 --- a/src/Processors/Transforms/DistinctSortedTransform.cpp +++ b/src/Processors/Transforms/DistinctSortedTransform.cpp @@ -46,7 +46,7 @@ DistinctSortedTransform::DistinctSortedTransform( void DistinctSortedTransform::transform(Chunk & chunk) { - if (!chunk.hasRows()) + if (unlikely(!chunk.hasRows())) return; /// get DISTINCT columns from chunk @@ -78,12 +78,15 @@ void DistinctSortedTransform::transform(Chunk & chunk) { case ClearableSetVariants::Type::EMPTY: break; + // clang-format off #define M(NAME) \ - case ClearableSetVariants::Type::NAME: \ - has_new_data = buildFilter(*data.NAME, column_ptrs, clearing_hint_columns, filter, rows, data); \ - break; - APPLY_FOR_SET_VARIANTS(M) + case ClearableSetVariants::Type::NAME: \ + has_new_data = buildFilter(*data.NAME, column_ptrs, clearing_hint_columns, filter, rows, data); \ + break; + + APPLY_FOR_SET_VARIANTS(M) #undef M + // clang-format on } /// Just go to the next block if there isn't any new record in the current one. From 64e51e56e7c944cb9e01d75c913f6ee68b74efca Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 27 Jul 2022 08:15:51 +0000 Subject: [PATCH 065/139] Allocate memory for column arrays once --- .../Transforms/DistinctSortedTransform.cpp | 15 ++++++++------- .../Transforms/DistinctSortedTransform.h | 2 ++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Processors/Transforms/DistinctSortedTransform.cpp b/src/Processors/Transforms/DistinctSortedTransform.cpp index d34da9d72c9..3762504fda5 100644 --- a/src/Processors/Transforms/DistinctSortedTransform.cpp +++ b/src/Processors/Transforms/DistinctSortedTransform.cpp @@ -27,6 +27,8 @@ DistinctSortedTransform::DistinctSortedTransform( if (col && !isColumnConst(*col)) column_positions.emplace_back(pos); } + column_ptrs.reserve(column_positions.size()); + /// pre-calculate DISTINCT column positions which form sort prefix of sort description sort_prefix_positions.reserve(description.size()); for (const auto & column_sort_descr : description) @@ -42,6 +44,7 @@ DistinctSortedTransform::DistinctSortedTransform( sort_prefix_positions.emplace_back(pos); } + sort_prefix_columns.reserve(sort_prefix_positions.size()); } void DistinctSortedTransform::transform(Chunk & chunk) @@ -50,8 +53,7 @@ void DistinctSortedTransform::transform(Chunk & chunk) return; /// get DISTINCT columns from chunk - ColumnRawPtrs column_ptrs; - column_ptrs.reserve(column_positions.size()); + column_ptrs.clear(); for (const auto pos : column_positions) { const auto & column = chunk.getColumns()[pos]; @@ -59,12 +61,11 @@ void DistinctSortedTransform::transform(Chunk & chunk) } /// get DISTINCT columns from chunk which form sort prefix of sort description - ColumnRawPtrs clearing_hint_columns; - clearing_hint_columns.reserve(sort_prefix_positions.size()); + sort_prefix_columns.clear(); for (const auto pos : sort_prefix_positions) { const auto & column = chunk.getColumns()[pos]; - clearing_hint_columns.emplace_back(column.get()); + sort_prefix_columns.emplace_back(column.get()); } if (data.type == ClearableSetVariants::Type::EMPTY) @@ -81,7 +82,7 @@ void DistinctSortedTransform::transform(Chunk & chunk) // clang-format off #define M(NAME) \ case ClearableSetVariants::Type::NAME: \ - has_new_data = buildFilter(*data.NAME, column_ptrs, clearing_hint_columns, filter, rows, data); \ + has_new_data = buildFilter(*data.NAME, column_ptrs, sort_prefix_columns, filter, rows, data); \ break; APPLY_FOR_SET_VARIANTS(M) @@ -108,7 +109,7 @@ void DistinctSortedTransform::transform(Chunk & chunk) stopReading(); prev_chunk.chunk = std::move(chunk); - prev_chunk.clearing_hint_columns = std::move(clearing_hint_columns); + prev_chunk.clearing_hint_columns = std::move(sort_prefix_columns); size_t all_columns = prev_chunk.chunk.getNumColumns(); Chunk res_chunk; diff --git a/src/Processors/Transforms/DistinctSortedTransform.h b/src/Processors/Transforms/DistinctSortedTransform.h index 11e3c22f708..2fe40408683 100644 --- a/src/Processors/Transforms/DistinctSortedTransform.h +++ b/src/Processors/Transforms/DistinctSortedTransform.h @@ -57,6 +57,8 @@ private: Names column_names; ColumnNumbers column_positions; /// DISTINCT columns positions in header ColumnNumbers sort_prefix_positions; /// DISTINCT columns positions which form sort prefix of sort description + ColumnRawPtrs column_ptrs; /// DISTINCT columns from chunk + ColumnRawPtrs sort_prefix_columns; /// DISTINCT columns from chunk which form sort prefix of sort description ClearableSetVariants data; Sizes key_sizes; From c9ad914164462f9b48952058e112fac1c9785b03 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 27 Jul 2022 09:32:54 +0000 Subject: [PATCH 066/139] Update arrow to fix special build --- contrib/arrow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/arrow b/contrib/arrow index 14976b7b205..3e03c6de41a 160000 --- a/contrib/arrow +++ b/contrib/arrow @@ -1 +1 @@ -Subproject commit 14976b7b2057ad0c0368705378682c625742b2fd +Subproject commit 3e03c6de41a86df2fc54a61e9be1abaefeff6b0e From 01a309d4e37131475d405b7ce17c369dab528b2b Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 27 Jul 2022 12:02:27 +0000 Subject: [PATCH 067/139] Fix strings in dictionary in Arrow format --- src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp | 3 ++- .../02376_arrow_dict_with_string.reference | 11 +++++++++++ .../0_stateless/02376_arrow_dict_with_string.sql | 4 ++++ 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02376_arrow_dict_with_string.reference create mode 100644 tests/queries/0_stateless/02376_arrow_dict_with_string.sql diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index 34c120c3f52..28f51b9b922 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -47,7 +47,8 @@ M(INT64, arrow::Int64Type) \ M(FLOAT, arrow::FloatType) \ M(DOUBLE, arrow::DoubleType) \ - M(BINARY, arrow::BinaryType) + M(BINARY, arrow::BinaryType) \ + M(STRING, arrow::StringType) namespace DB { diff --git a/tests/queries/0_stateless/02376_arrow_dict_with_string.reference b/tests/queries/0_stateless/02376_arrow_dict_with_string.reference new file mode 100644 index 00000000000..6f6ed89497c --- /dev/null +++ b/tests/queries/0_stateless/02376_arrow_dict_with_string.reference @@ -0,0 +1,11 @@ +x LowCardinality(Nullable(String)) +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 diff --git a/tests/queries/0_stateless/02376_arrow_dict_with_string.sql b/tests/queries/0_stateless/02376_arrow_dict_with_string.sql new file mode 100644 index 00000000000..c1be023e8e6 --- /dev/null +++ b/tests/queries/0_stateless/02376_arrow_dict_with_string.sql @@ -0,0 +1,4 @@ +-- Tags: no-parallel +insert into function file(02376_data.arrow) select toLowCardinality(toString(number)) as x from numbers(10) settings output_format_arrow_string_as_string=1, output_format_arrow_low_cardinality_as_dictionary=1, engine_file_truncate_on_insert=1; +desc file (02376_data.arrow); +select * from file(02376_data.arrow); From 0209d10569802fa71b05a55ad6313794c1d8b577 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 27 Jul 2022 15:37:43 +0300 Subject: [PATCH 068/139] Fix test --- .../02364_window_view_segfault.reference | 1 + .../0_stateless/02364_window_view_segfault.sh | 14 ++++++++++++++ .../0_stateless/02364_window_view_segfault.sql | 4 ---- 3 files changed, 15 insertions(+), 4 deletions(-) create mode 100755 tests/queries/0_stateless/02364_window_view_segfault.sh delete mode 100644 tests/queries/0_stateless/02364_window_view_segfault.sql diff --git a/tests/queries/0_stateless/02364_window_view_segfault.reference b/tests/queries/0_stateless/02364_window_view_segfault.reference index e69de29bb2d..9766475a418 100644 --- a/tests/queries/0_stateless/02364_window_view_segfault.reference +++ b/tests/queries/0_stateless/02364_window_view_segfault.reference @@ -0,0 +1 @@ +ok diff --git a/tests/queries/0_stateless/02364_window_view_segfault.sh b/tests/queries/0_stateless/02364_window_view_segfault.sh new file mode 100755 index 00000000000..d23ffc4c076 --- /dev/null +++ b/tests/queries/0_stateless/02364_window_view_segfault.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel +# Tag no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --multiquery --multiline --query """ +DROP TABLE IF EXISTS mt ON CLUSTER test_shard_localhost; +DROP TABLE IF EXISTS wv ON CLUSTER test_shard_localhost; +CREATE TABLE mt ON CLUSTER test_shard_localhost (a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple(); +CREATE WINDOW VIEW wv ON CLUSTER test_shard_localhost TO input_deduplicated INNER ENGINE Memory WATERMARK=INTERVAL '1' SECOND AS SELECT count(a), hopStart(wid) AS w_start, hopEnd(wid) AS w_end FROM mt GROUP BY hop(timestamp, INTERVAL '3' SECOND, INTERVAL '5' SECOND) AS wid; +""" 2>&1 | grep -q -e "Code: 344" -e "Code: 60" && echo 'ok' || echo 'fail' ||: diff --git a/tests/queries/0_stateless/02364_window_view_segfault.sql b/tests/queries/0_stateless/02364_window_view_segfault.sql deleted file mode 100644 index 3b454d3006e..00000000000 --- a/tests/queries/0_stateless/02364_window_view_segfault.sql +++ /dev/null @@ -1,4 +0,0 @@ -DROP TABLE IF EXISTS mt; -DROP TABLE IF EXISTS wv; -CREATE TABLE mt(a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple(); -CREATE WINDOW VIEW wv ON CLUSTER test_shard_localhost TO input_deduplicated INNER ENGINE Memory WATERMARK=INTERVAL '1' SECOND AS SELECT count(a), hopStart(wid) AS w_start, hopEnd(wid) AS w_end FROM mt GROUP BY hop(timestamp, INTERVAL '3' SECOND, INTERVAL '5' SECOND) AS wid; -- { serverError 344 } From 6a8407380193beb5fb4d375c282e12a3b4c986a9 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 27 Jul 2022 13:48:17 +0000 Subject: [PATCH 069/139] Fix test --- tests/queries/1_stateful/00174_distinct_in_order.reference | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/queries/1_stateful/00174_distinct_in_order.reference b/tests/queries/1_stateful/00174_distinct_in_order.reference index e3585746360..c0b2aadc20e 100644 --- a/tests/queries/1_stateful/00174_distinct_in_order.reference +++ b/tests/queries/1_stateful/00174_distinct_in_order.reference @@ -1,6 +1,3 @@ -- check that distinct with and w/o optimization produce the same result -<<<<<<< HEAD -- DISTINCT colums are the same as in ORDER BY -- DISTINCT colums has prefix in ORDER BY columns -======= ->>>>>>> origin/master From 3639ecfc562f14b044338ae10bcc94e7be894086 Mon Sep 17 00:00:00 2001 From: HarryLeeIBM Date: Wed, 27 Jul 2022 06:56:04 -0700 Subject: [PATCH 070/139] Fix Endian issue in BitHelpers --- src/IO/BitHelpers.h | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/IO/BitHelpers.h b/src/IO/BitHelpers.h index 1648746e2b6..c67b44bc325 100644 --- a/src/IO/BitHelpers.h +++ b/src/IO/BitHelpers.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -140,7 +141,8 @@ private: memcpy(&tmp_buffer, source_current, bytes_to_read); source_current += bytes_to_read; - tmp_buffer = __builtin_bswap64(tmp_buffer); + if constexpr (std::endian::native == std::endian::little) + tmp_buffer = __builtin_bswap64(tmp_buffer); bits_buffer |= BufferType(tmp_buffer) << ((sizeof(BufferType) - sizeof(tmp_buffer)) * 8 - bits_count); bits_count += static_cast(bytes_to_read) * 8; @@ -223,8 +225,11 @@ private: "Can not write past end of buffer. Space available {} bytes, required to write {} bytes.", available, to_write); } - - const auto tmp_buffer = __builtin_bswap64(static_cast(bits_buffer >> (sizeof(bits_buffer) - sizeof(UInt64)) * 8)); + UInt64 tmp_buffer = 0; + if constexpr (std::endian::native == std::endian::little) + tmp_buffer = __builtin_bswap64(static_cast(bits_buffer >> (sizeof(bits_buffer) - sizeof(UInt64)) * 8)); + else + tmp_buffer = static_cast(bits_buffer >> (sizeof(bits_buffer) - sizeof(UInt64)) * 8); memcpy(dest_current, &tmp_buffer, to_write); dest_current += to_write; From 1cb42c2c18bb71969abad5cdc00861989f1d66bf Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 27 Jul 2022 22:09:15 +0300 Subject: [PATCH 071/139] Update 02364_window_view_segfault.sh --- tests/queries/0_stateless/02364_window_view_segfault.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02364_window_view_segfault.sh b/tests/queries/0_stateless/02364_window_view_segfault.sh index d23ffc4c076..d03a1e5ae3e 100755 --- a/tests/queries/0_stateless/02364_window_view_segfault.sh +++ b/tests/queries/0_stateless/02364_window_view_segfault.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest, no-parallel -# Tag no-fasttest +# Tags: no-fasttest, no-parallel, no-replicated-database CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From 0ff9fc2eb902f388fe6aec50749ca7bba59a55f6 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 27 Jul 2022 08:00:25 +0000 Subject: [PATCH 072/139] Use commit with init --- contrib/NuRaft | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/NuRaft b/contrib/NuRaft index b924bb097a9..e1dc47c1cfd 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit b924bb097a9bc1c19f52a60a2265bede0a567b2f +Subproject commit e1dc47c1cfd529801a8c94a396a3921a71ae3ccf From c6fd8364a4c86fe03385db6df6353deb0dab6f35 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 27 Jul 2022 14:24:14 +0200 Subject: [PATCH 073/139] Increase ccache max size for clang tidy builds --- docker/packager/packager | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index 0b00bc4e9c0..3b307ada1b9 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -236,7 +236,11 @@ def parse_env_variables( result.append("CCACHE_BASEDIR=/build") result.append("CCACHE_NOHASHDIR=true") result.append("CCACHE_COMPILERCHECK=content") - result.append("CCACHE_MAXSIZE=15G") + cache_maxsize = "15G" + if clang_tidy: + # 15G is not enough for tidy build + cache_maxsize = "25G" + result.append(f"CCACHE_MAXSIZE={cache_maxsize}") # result.append("CCACHE_UMASK=777") if distcc_hosts: From e1f8cff3b4791c5a2ea62239af9da9aae21961b3 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 28 Jul 2022 22:03:13 +0300 Subject: [PATCH 074/139] Add duration_ms into system.zookeeper_log Signed-off-by: Azat Khuzhin --- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 20 +++++++++++-------- src/Common/ZooKeeper/ZooKeeperImpl.h | 2 +- src/Interpreters/ZooKeeperLog.cpp | 2 ++ src/Interpreters/ZooKeeperLog.h | 2 ++ .../01158_zookeeper_log_long.reference | 2 ++ .../0_stateless/01158_zookeeper_log_long.sql | 4 ++++ 6 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 6e62fccb6fd..1d0f1fdb1a2 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -701,6 +701,7 @@ void ZooKeeper::receiveEvent() RequestInfo request_info; ZooKeeperResponsePtr response; + UInt64 elapsed_ms = 0; if (xid == PING_XID) { @@ -761,8 +762,8 @@ void ZooKeeper::receiveEvent() CurrentMetrics::sub(CurrentMetrics::ZooKeeperRequest); } - auto elapsed_microseconds = std::chrono::duration_cast(clock::now() - request_info.time).count(); - ProfileEvents::increment(ProfileEvents::ZooKeeperWaitMicroseconds, elapsed_microseconds); + elapsed_ms = std::chrono::duration_cast(clock::now() - request_info.time).count(); + ProfileEvents::increment(ProfileEvents::ZooKeeperWaitMicroseconds, elapsed_ms); } try @@ -812,7 +813,7 @@ void ZooKeeper::receiveEvent() if (length != actual_length) throw Exception("Response length doesn't match. Expected: " + DB::toString(length) + ", actual: " + DB::toString(actual_length), Error::ZMARSHALLINGERROR); - logOperationIfNeeded(request_info.request, response); //-V614 + logOperationIfNeeded(request_info.request, response, /* finalize= */ false, elapsed_ms); //-V614 } catch (...) { @@ -831,7 +832,7 @@ void ZooKeeper::receiveEvent() if (request_info.callback) request_info.callback(*response); - logOperationIfNeeded(request_info.request, response); + logOperationIfNeeded(request_info.request, response, /* finalize= */ false, elapsed_ms); } catch (...) { @@ -919,13 +920,14 @@ void ZooKeeper::finalize(bool error_send, bool error_receive, const String & rea ? Error::ZCONNECTIONLOSS : Error::ZSESSIONEXPIRED; response->xid = request_info.request->xid; + UInt64 elapsed_ms = std::chrono::duration_cast(clock::now() - request_info.time).count(); if (request_info.callback) { try { request_info.callback(*response); - logOperationIfNeeded(request_info.request, response, true); + logOperationIfNeeded(request_info.request, response, true, elapsed_ms); } catch (...) { @@ -985,7 +987,8 @@ void ZooKeeper::finalize(bool error_send, bool error_receive, const String & rea try { info.callback(*response); - logOperationIfNeeded(info.request, response, true); + UInt64 elapsed_ms = std::chrono::duration_cast(clock::now() - info.time).count(); + logOperationIfNeeded(info.request, response, true, elapsed_ms); } catch (...) { @@ -1310,7 +1313,7 @@ void ZooKeeper::setZooKeeperLog(std::shared_ptr zk_log_) } #ifdef ZOOKEEPER_LOG -void ZooKeeper::logOperationIfNeeded(const ZooKeeperRequestPtr & request, const ZooKeeperResponsePtr & response, bool finalize) +void ZooKeeper::logOperationIfNeeded(const ZooKeeperRequestPtr & request, const ZooKeeperResponsePtr & response, bool finalize, UInt64 elapsed_ms) { auto maybe_zk_log = std::atomic_load(&zk_log); if (!maybe_zk_log) @@ -1348,6 +1351,7 @@ void ZooKeeper::logOperationIfNeeded(const ZooKeeperRequestPtr & request, const elem.event_time = event_time; elem.address = socket_address; elem.session_id = session_id; + elem.duration_ms = elapsed_ms; if (request) { elem.thread_id = request->thread_id; @@ -1357,7 +1361,7 @@ void ZooKeeper::logOperationIfNeeded(const ZooKeeperRequestPtr & request, const } } #else -void ZooKeeper::logOperationIfNeeded(const ZooKeeperRequestPtr &, const ZooKeeperResponsePtr &, bool) +void ZooKeeper::logOperationIfNeeded(const ZooKeeperRequestPtr &, const ZooKeeperResponsePtr &, bool, UInt64) {} #endif diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.h b/src/Common/ZooKeeper/ZooKeeperImpl.h index 4f2098f25ca..e00250c1517 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.h +++ b/src/Common/ZooKeeper/ZooKeeperImpl.h @@ -276,7 +276,7 @@ private: template void read(T &); - void logOperationIfNeeded(const ZooKeeperRequestPtr & request, const ZooKeeperResponsePtr & response = nullptr, bool finalize = false); + void logOperationIfNeeded(const ZooKeeperRequestPtr & request, const ZooKeeperResponsePtr & response = nullptr, bool finalize = false, UInt64 elapsed_ms = 0); void initApiVersion(); diff --git a/src/Interpreters/ZooKeeperLog.cpp b/src/Interpreters/ZooKeeperLog.cpp index 4f01ad8b5fb..7a59aa9ac9b 100644 --- a/src/Interpreters/ZooKeeperLog.cpp +++ b/src/Interpreters/ZooKeeperLog.cpp @@ -122,6 +122,7 @@ NamesAndTypesList ZooKeeperLogElement::getNamesAndTypes() {"address", DataTypeFactory::instance().get("IPv6")}, {"port", std::make_shared()}, {"session_id", std::make_shared()}, + {"duration_ms", std::make_shared()}, {"xid", std::make_shared()}, {"has_watch", std::make_shared()}, @@ -172,6 +173,7 @@ void ZooKeeperLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insertData(IPv6ToBinary(address.host()).data(), 16); columns[i++]->insert(address.port()); columns[i++]->insert(session_id); + columns[i++]->insert(duration_ms); columns[i++]->insert(xid); columns[i++]->insert(has_watch); diff --git a/src/Interpreters/ZooKeeperLog.h b/src/Interpreters/ZooKeeperLog.h index 43cdebced2d..d79b75ec85f 100644 --- a/src/Interpreters/ZooKeeperLog.h +++ b/src/Interpreters/ZooKeeperLog.h @@ -27,6 +27,8 @@ struct ZooKeeperLogElement Poco::Net::SocketAddress address; Int64 session_id = 0; + UInt64 duration_ms = 0; + /// Common request info Int32 xid = 0; bool has_watch = false; diff --git a/tests/queries/0_stateless/01158_zookeeper_log_long.reference b/tests/queries/0_stateless/01158_zookeeper_log_long.reference index 44451d82783..ee2b465b3d3 100644 --- a/tests/queries/0_stateless/01158_zookeeper_log_long.reference +++ b/tests/queries/0_stateless/01158_zookeeper_log_long.reference @@ -38,3 +38,5 @@ Response 0 Error /test/01158/default/rmt/blocks/all_6308706741995381342_24957917 Response 0 Error /test/01158/default/rmt/temp/abandonable_lock- 1 1 \N 0 3 ZRUNTIMEINCONSISTENCY \N \N 0 0 0 0 Request 0 Get /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 \N 0 0 \N \N \N 0 0 0 0 Response 0 Get /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 \N 0 0 ZOK \N \N 0 0 9 0 +duration_ms +1 diff --git a/tests/queries/0_stateless/01158_zookeeper_log_long.sql b/tests/queries/0_stateless/01158_zookeeper_log_long.sql index 6048169a3d8..501ab805a5a 100644 --- a/tests/queries/0_stateless/01158_zookeeper_log_long.sql +++ b/tests/queries/0_stateless/01158_zookeeper_log_long.sql @@ -31,3 +31,7 @@ where (session_id, xid) in (select session_id, xid from system.zookeeper_log whe order by xid, type, request_idx; drop table rmt; + +system flush logs; +select 'duration_ms'; +select count()>0 from system.zookeeper_log where path like '/test/01158/' || currentDatabase() || '/rmt%' and duration_ms > 0; From 3b6bacbf0b8e70dda63eacb9e4111e401be31a4f Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 28 Jul 2022 22:08:53 +0200 Subject: [PATCH 075/139] Add tests. --- src/IO/tests/gtest_file_encryption.cpp | 18 ++++++++++++++++++ tests/integration/test_encrypted_disk/test.py | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/IO/tests/gtest_file_encryption.cpp b/src/IO/tests/gtest_file_encryption.cpp index e9affee4add..f53c85c422a 100644 --- a/src/IO/tests/gtest_file_encryption.cpp +++ b/src/IO/tests/gtest_file_encryption.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -241,6 +242,23 @@ TEST(FileEncryptionPositionUpdateTest, Decryption) rb.ignore(5); rb.ignore(5); ASSERT_EQ(rb.getPosition(), 15); + + String res; + readStringUntilEOF(res, rb); + ASSERT_EQ(res, data.substr(15)); + res.clear(); + + rb.seek(0, SEEK_SET); + ASSERT_EQ(rb.getPosition(), 0); + res.resize(5); + rb.read(res.data(), res.size()); + ASSERT_EQ(res, data.substr(0, 5)); + res.clear(); + + rb.seek(1, SEEK_CUR); + ASSERT_EQ(rb.getPosition(), 6); + readStringUntilEOF(res, rb); + ASSERT_EQ(res, data.substr(6)); } #endif diff --git a/tests/integration/test_encrypted_disk/test.py b/tests/integration/test_encrypted_disk/test.py index 4e6d1db9e99..17a30676f7f 100644 --- a/tests/integration/test_encrypted_disk/test.py +++ b/tests/integration/test_encrypted_disk/test.py @@ -252,3 +252,21 @@ EOF""".format( # Detach the part encrypted with the wrong key and check that another part containing "(2,'data'),(3,'data')" still can be read. node.query("ALTER TABLE encrypted_test DETACH PART '{}'".format(FIRST_PART_NAME)) assert node.query(select_query) == "(2,'data'),(3,'data')" + + +def test_read_in_order(): + node.query( + "CREATE TABLE encrypted_test(`a` UInt64, `b` String(150)) ENGINE = MergeTree() ORDER BY (a, b) SETTINGS storage_policy='encrypted_policy'" + ) + + node.query( + "INSERT INTO encrypted_test SELECT * FROM generateRandom('a UInt64, b FixedString(150)') LIMIT 100000" + ) + + node.query( + "SELECT * FROM encrypted_test ORDER BY a, b SETTINGS optimize_read_in_order=1 FORMAT Null" + ) + + node.query( + "SELECT * FROM encrypted_test ORDER BY a, b SETTINGS optimize_read_in_order=0 FORMAT Null" + ) From 4af435bddae5e3e8c0382d2ef6f5c1fedf91ba3d Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 28 Jul 2022 21:22:06 +0000 Subject: [PATCH 076/139] Fix: handle all const columns case correctly --- .../Transforms/DistinctSortedTransform.cpp | 29 +++++++++++++++++-- .../Transforms/DistinctSortedTransform.h | 1 + ...7_distinct_in_order_optimization.reference | 10 +++++++ .../02317_distinct_in_order_optimization.sql | 8 +++++ 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/src/Processors/Transforms/DistinctSortedTransform.cpp b/src/Processors/Transforms/DistinctSortedTransform.cpp index 3762504fda5..a5f058aaa8e 100644 --- a/src/Processors/Transforms/DistinctSortedTransform.cpp +++ b/src/Processors/Transforms/DistinctSortedTransform.cpp @@ -8,6 +8,20 @@ namespace ErrorCodes extern const int SET_SIZE_LIMIT_EXCEEDED; } +static void handleAllColumnsConst(Chunk & chunk) +{ + const size_t rows = chunk.getNumRows(); + IColumn::Filter filter(rows); + + Chunk res_chunk; + std::fill(filter.begin(), filter.end(), 0); + filter[0] = 1; + for (const auto & column : chunk.getColumns()) + res_chunk.addColumn(column->filter(filter, -1)); + + chunk = std::move(res_chunk); +} + DistinctSortedTransform::DistinctSortedTransform( Block header_, SortDescription sort_description, const SizeLimits & set_size_limits_, UInt64 limit_hint_, const Names & columns) : ISimpleTransform(header_, header_, true) @@ -23,9 +37,12 @@ DistinctSortedTransform::DistinctSortedTransform( for (size_t i = 0; i < num_columns; ++i) { auto pos = column_names.empty() ? i : header.getPositionByName(column_names[i]); - const auto & col = header.getByPosition(pos).column; - if (col && !isColumnConst(*col)) + const auto & column = header.getByPosition(pos).column; + if (column && !isColumnConst(*column)) + { column_positions.emplace_back(pos); + all_columns_const = false; + } } column_ptrs.reserve(column_positions.size()); @@ -52,6 +69,14 @@ void DistinctSortedTransform::transform(Chunk & chunk) if (unlikely(!chunk.hasRows())) return; + /// special case - all column constant + if (unlikely(all_columns_const)) + { + handleAllColumnsConst(chunk); + stopReading(); + return; + } + /// get DISTINCT columns from chunk column_ptrs.clear(); for (const auto pos : column_positions) diff --git a/src/Processors/Transforms/DistinctSortedTransform.h b/src/Processors/Transforms/DistinctSortedTransform.h index 2fe40408683..440754ce6a4 100644 --- a/src/Processors/Transforms/DistinctSortedTransform.h +++ b/src/Processors/Transforms/DistinctSortedTransform.h @@ -66,6 +66,7 @@ private: /// Restrictions on the maximum size of the output data. SizeLimits set_size_limits; + bool all_columns_const = true; }; } diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference b/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference index a5ae3759d5d..79927cf8884 100644 --- a/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference +++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference @@ -77,4 +77,14 @@ 2 2 1 1 0 0 +-- distinct with constants columns +-- { echoOn } +select distinct 1 as a, 2 as b from distinct_in_order; +1 2 +select distinct 1 as a, 2 as b from distinct_in_order order by a; +1 2 +select distinct 1 as a, 2 as b from distinct_in_order order by a, b; +1 2 +select distinct x, y from (select 1 as x, 2 as y from distinct_in_order order by x) order by x; +1 2 -- check that distinct in order has the same result as ordinary distinct diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql b/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql index f1de07e3db2..945c9625e99 100644 --- a/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql +++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql @@ -43,6 +43,14 @@ select distinct b,c from distinct_in_order order by c; select '-- distinct with non-key prefix and non-sorted column, order by non-sorted desc'; select distinct b,c from distinct_in_order order by c desc; +select '-- distinct with constants columns'; +-- { echoOn } +select distinct 1 as a, 2 as b from distinct_in_order; +select distinct 1 as a, 2 as b from distinct_in_order order by a; +select distinct 1 as a, 2 as b from distinct_in_order order by a, b; +select distinct x, y from (select 1 as x, 2 as y from distinct_in_order order by x) order by x; +-- { echoOff } + drop table if exists distinct_in_order sync; select '-- check that distinct in order has the same result as ordinary distinct'; From d3e5199372cb3110e24651a61ae11d928321ca26 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 28 Jul 2022 23:54:59 +0200 Subject: [PATCH 077/139] Fix build of keeper-data-dumper --- utils/keeper-data-dumper/main.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/keeper-data-dumper/main.cpp b/utils/keeper-data-dumper/main.cpp index b55dab8399b..bcc3f6daf26 100644 --- a/utils/keeper-data-dumper/main.cpp +++ b/utils/keeper-data-dumper/main.cpp @@ -62,7 +62,8 @@ int main(int argc, char *argv[]) ResponsesQueue queue(std::numeric_limits::max()); SnapshotsQueue snapshots_queue{1}; CoordinationSettingsPtr settings = std::make_shared(); - auto state_machine = std::make_shared(queue, snapshots_queue, argv[1], settings); + auto keeper_context = std::make_shared(); + auto state_machine = std::make_shared(queue, snapshots_queue, argv[1], settings, keeper_context); state_machine->init(); size_t last_commited_index = state_machine->last_commit_index(); From 552b517e59e52540f66e7e66adf68a20bf35103f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 29 Jul 2022 00:13:02 +0200 Subject: [PATCH 078/139] Allow to hide addresses in stack traces --- programs/server/Server.cpp | 2 ++ programs/server/config.xml | 6 ++++++ src/Common/StackTrace.cpp | 43 ++++++++++++++++++++++++++++---------- src/Common/StackTrace.h | 5 +++++ 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 9effc23e107..14108d7931d 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -627,6 +627,8 @@ int Server::main(const std::vector & /*args*/) MainThreadStatus::getInstance(); + StackTrace::setShowAddresses(config().getBool("show_addresses_in_stack_traces", true)); + registerFunctions(); registerAggregateFunctions(); registerTableFunctions(); diff --git a/programs/server/config.xml b/programs/server/config.xml index 84930230397..c49d2a83499 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -1386,4 +1386,10 @@ 268435456 true + + + true diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index f41a646f31e..01d8bccc1f5 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -1,6 +1,5 @@ #include -#include #include #include #include @@ -8,6 +7,7 @@ #include #include +#include #include #include #include @@ -19,6 +19,20 @@ # include #endif + +namespace +{ + /// Currently this variable is set up once on server startup. + /// But we use atomic just in case, so it is possible to be modified at runtime. + std::atomic show_addresses = true; +} + +void StackTrace::setShowAddresses(bool show) +{ + show_addresses.store(show, std::memory_order_relaxed); +} + + std::string signalToErrorMessage(int sig, const siginfo_t & info, [[maybe_unused]] const ucontext_t & context) { std::stringstream error; // STYLE_CHECK_ALLOW_STD_STRING_STREAM @@ -30,7 +44,7 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, [[maybe_unused /// Print info about address and reason. if (nullptr == info.si_addr) error << "Address: NULL pointer."; - else + else if (show_addresses.load(std::memory_order_relaxed)) error << "Address: " << info.si_addr; #if defined(__x86_64__) && !defined(OS_FREEBSD) && !defined(OS_DARWIN) && !defined(__arm__) && !defined(__powerpc__) @@ -372,7 +386,9 @@ static void toStringEveryLineImpl( else out << "?"; - out << " @ " << physical_addr; + if (show_addresses.load(std::memory_order_relaxed)) + out << " @ " << physical_addr; + out << " in " << (object ? object->name : "?"); for (size_t j = 0; j < inline_frames.size(); ++j) @@ -387,17 +403,22 @@ static void toStringEveryLineImpl( out.str({}); } #else - std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - out.exceptions(std::ios::failbit); - - for (size_t i = offset; i < size; ++i) + if (show_addresses.load(std::memory_order_relaxed)) { - const void * addr = frame_pointers[i]; - out << i << ". " << addr; + std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + out.exceptions(std::ios::failbit); - callback(out.str()); - out.str({}); + for (size_t i = offset; i < size; ++i) + { + const void * addr = frame_pointers[i]; + out << i << ". " << addr; + + callback(out.str()); + out.str({}); + } } + else + callback("Addresses are hidden for security reasons."); #endif } diff --git a/src/Common/StackTrace.h b/src/Common/StackTrace.h index 84a2e9d1f7f..f07c05107ee 100644 --- a/src/Common/StackTrace.h +++ b/src/Common/StackTrace.h @@ -67,6 +67,11 @@ public: void toStringEveryLine(std::function callback) const; + /// Displaying the addresses can be disabled for security reasons. + /// If you turn off addresses, it will be more secure, but we will be unable to help you with debugging. + /// Please note: addresses are also available in the system.stack_trace and system.trace_log tables. + static void setShowAddresses(bool show); + protected: void tryCapture(); From d231627a24c48816cfc3cbf164a0a2ddc789602c Mon Sep 17 00:00:00 2001 From: Yuko Takagi <70714860+yukotakagi@users.noreply.github.com> Date: Thu, 28 Jul 2022 16:49:00 -0600 Subject: [PATCH 079/139] Update README.md --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1d0146582a6..03617fb5c2a 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,9 @@ ClickHouse® is an open-source column-oriented database management system that a * [Blog](https://clickhouse.com/blog/en/) contains various ClickHouse-related articles, as well as announcements and reports about events. * [Code Browser (Woboq)](https://clickhouse.com/codebrowser/ClickHouse/index.html) with syntax highlight and navigation. * [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlight, powered by github.dev. -* [Contacts](https://clickhouse.com/company/#contact) can help to get your questions answered if there are any. +* [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any. ## Upcoming events -* [v22.7 Release Webinar](https://clickhouse.com/company/events/v22-7-release-webinar/) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap. -* [ClickHouse Meetup at the Cloudflare office in London](https://www.meetup.com/clickhouse-london-user-group/events/286891586/) ClickHouse meetup at the Cloudflare office space in central London -* [ClickHouse Meetup at the Metoda office in Munich](https://www.meetup.com/clickhouse-meetup-munich/events/286891667/) ClickHouse meetup at the Metoda office in Munich +* [v22.8 Release Webinar](https://clickhouse.com/company/events/v22-8-release-webinar/) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap. From c5837dd6f5a9c6e31c50380208d3f8d3bf3cc3dc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 29 Jul 2022 02:59:54 +0300 Subject: [PATCH 080/139] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 03617fb5c2a..db31b283cab 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,6 @@ ClickHouse® is an open-source column-oriented database management system that a * [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any. ## Upcoming events -* [v22.8 Release Webinar](https://clickhouse.com/company/events/v22-8-release-webinar/) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap. +* **v22.8 Release Webinar** Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap. From 4047a865cbeb0eb080853c27958bfd3f288f7396 Mon Sep 17 00:00:00 2001 From: qinghuan wang <1095193290@qq.com> Date: Fri, 29 Jul 2022 10:26:21 +0800 Subject: [PATCH 081/139] update clickcat --- docs/en/interfaces/third-party/gui.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/interfaces/third-party/gui.md b/docs/en/interfaces/third-party/gui.md index c5cc8c7c34b..e23d68cbc2e 100644 --- a/docs/en/interfaces/third-party/gui.md +++ b/docs/en/interfaces/third-party/gui.md @@ -149,7 +149,7 @@ Features: ### ClickCat {#clickcat} -[ClickCat](https://github.com/open-botech/ClickCat) is a firendly user interface that lets you search, explore and visualize your ClickHouse Data. +[ClickCat](https://github.com/clickcat-project/ClickCat) is a firendly user interface that lets you search, explore and visualize your ClickHouse Data. Features: From 4088c0a7f324b98c44f26a20462ff5d8e04d6cac Mon Sep 17 00:00:00 2001 From: Li Yin Date: Mon, 4 Jul 2022 15:01:39 +0800 Subject: [PATCH 082/139] Automated function registration Automated register all functions with below naming convention by iterating through the symbols: void DB::registerXXX(DB::FunctionFactory &) --- src/CMakeLists.txt | 4 +- src/Common/register_objects.cpp | 12 ++ src/Common/register_objects.h | 33 ++++ src/Functions/CMakeLists.txt | 57 ++++-- src/Functions/CRC.cpp | 2 +- src/Functions/CastOverloadResolver.cpp | 2 +- src/Functions/FunctionChar.cpp | 2 +- src/Functions/FunctionFQDN.cpp | 2 +- src/Functions/FunctionFactory.h | 1 + src/Functions/FunctionFile.cpp | 2 +- src/Functions/FunctionHashID.cpp | 2 +- src/Functions/FunctionJoinGet.cpp | 2 +- src/Functions/FunctionSQLJSON.cpp | 2 +- src/Functions/FunctionShowCertificate.cpp | 2 +- src/Functions/FunctionsBase58.cpp | 4 +- .../FunctionsBinaryRepresentation.cpp | 2 +- src/Functions/FunctionsBitToArray.cpp | 2 +- src/Functions/FunctionsBitmap.cpp | 2 +- .../FunctionsCharsetClassification.cpp | 2 +- src/Functions/FunctionsCodingIP.cpp | 2 +- src/Functions/FunctionsCodingUUID.cpp | 2 +- src/Functions/FunctionsConversion.cpp | 10 +- .../FunctionsEmbeddedDictionaries.cpp | 2 +- .../FunctionsExternalDictionaries.cpp | 2 +- src/Functions/FunctionsHashing.cpp | 2 +- src/Functions/FunctionsJSON.cpp | 2 +- .../FunctionsLanguageClassification.cpp | 2 +- src/Functions/FunctionsLogical.cpp | 2 +- .../FunctionsProgrammingClassification.cpp | 2 +- src/Functions/FunctionsRound.cpp | 2 +- src/Functions/FunctionsStringArray.cpp | 2 +- src/Functions/FunctionsStringHash.cpp | 2 +- src/Functions/FunctionsStringSimilarity.cpp | 2 +- src/Functions/FunctionsTimeWindow.cpp | 2 +- .../FunctionsTonalityClassification.cpp | 2 +- .../FunctionsTransactionCounters.cpp | 2 +- src/Functions/JSONPath/CMakeLists.txt | 1 - src/Functions/SubtractSubSeconds.cpp | 6 +- src/Functions/URL/CMakeLists.txt | 3 +- src/Functions/URL/URLHierarchy.cpp | 2 +- src/Functions/URL/URLPathHierarchy.cpp | 2 +- src/Functions/URL/basename.cpp | 2 +- src/Functions/URL/cutFragment.cpp | 2 +- src/Functions/URL/cutQueryString.cpp | 2 +- .../URL/cutQueryStringAndFragment.cpp | 2 +- .../URL/cutToFirstSignificantSubdomain.cpp | 2 +- .../cutToFirstSignificantSubdomainCustom.cpp | 2 +- src/Functions/URL/cutURLParameter.cpp | 2 +- src/Functions/URL/cutWWW.cpp | 2 +- src/Functions/URL/decodeURLComponent.cpp | 2 +- src/Functions/URL/domain.cpp | 2 +- src/Functions/URL/domainWithoutWWW.cpp | 2 +- src/Functions/URL/extractURLParameter.cpp | 2 +- .../URL/extractURLParameterNames.cpp | 2 +- src/Functions/URL/extractURLParameters.cpp | 2 +- .../URL/firstSignificantSubdomain.cpp | 2 +- .../URL/firstSignificantSubdomainCustom.cpp | 2 +- src/Functions/URL/fragment.cpp | 2 +- src/Functions/URL/netloc.cpp | 2 +- src/Functions/URL/path.cpp | 2 +- src/Functions/URL/pathFull.cpp | 2 +- src/Functions/URL/port.cpp | 2 +- src/Functions/URL/protocol.cpp | 2 +- src/Functions/URL/queryString.cpp | 2 +- src/Functions/URL/queryStringAndFragment.cpp | 2 +- src/Functions/URL/registerFunctionsURL.cpp | 64 ------ src/Functions/URL/topLevelDomain.cpp | 2 +- src/Functions/abs.cpp | 2 +- src/Functions/acos.cpp | 2 +- src/Functions/acosh.cpp | 2 +- src/Functions/addDays.cpp | 2 +- src/Functions/addHours.cpp | 2 +- src/Functions/addMinutes.cpp | 2 +- src/Functions/addMonths.cpp | 2 +- src/Functions/addQuarters.cpp | 2 +- src/Functions/addSeconds.cpp | 2 +- src/Functions/addSubSeconds.cpp | 6 +- src/Functions/addWeeks.cpp | 2 +- src/Functions/addYears.cpp | 2 +- src/Functions/addressToLine.cpp | 2 +- src/Functions/addressToLineWithInlines.cpp | 2 +- src/Functions/addressToSymbol.cpp | 2 +- src/Functions/aes_decrypt_mysql.cpp | 2 +- src/Functions/aes_encrypt_mysql.cpp | 2 +- src/Functions/appendTrailingCharIfAbsent.cpp | 2 +- src/Functions/array/CMakeLists.txt | 2 +- src/Functions/array/array.cpp | 2 +- src/Functions/array/arrayAUC.cpp | 2 +- src/Functions/array/arrayAggregation.cpp | 2 +- src/Functions/array/arrayAll.cpp | 2 +- src/Functions/array/arrayCompact.cpp | 2 +- src/Functions/array/arrayConcat.cpp | 2 +- src/Functions/array/arrayCount.cpp | 2 +- src/Functions/array/arrayCumSum.cpp | 2 +- .../array/arrayCumSumNonNegative.cpp | 2 +- src/Functions/array/arrayDifference.cpp | 2 +- src/Functions/array/arrayDistinct.cpp | 2 +- src/Functions/array/arrayElement.cpp | 2 +- src/Functions/array/arrayEnumerate.cpp | 2 +- src/Functions/array/arrayEnumerateDense.cpp | 2 +- .../array/arrayEnumerateDenseRanked.cpp | 2 +- src/Functions/array/arrayEnumerateUniq.cpp | 2 +- .../array/arrayEnumerateUniqRanked.cpp | 2 +- src/Functions/array/arrayExists.cpp | 2 +- src/Functions/array/arrayFill.cpp | 2 +- src/Functions/array/arrayFilter.cpp | 2 +- src/Functions/array/arrayFirstLast.cpp | 2 +- src/Functions/array/arrayFirstLastIndex.cpp | 2 +- src/Functions/array/arrayFlatten.cpp | 2 +- src/Functions/array/arrayIntersect.cpp | 2 +- src/Functions/array/arrayJoin.cpp | 2 +- src/Functions/array/arrayMap.cpp | 2 +- src/Functions/array/arrayPopBack.cpp | 2 +- src/Functions/array/arrayPopFront.cpp | 2 +- src/Functions/array/arrayPushBack.cpp | 2 +- src/Functions/array/arrayPushFront.cpp | 2 +- src/Functions/array/arrayReduce.cpp | 2 +- src/Functions/array/arrayReduceInRanges.cpp | 2 +- src/Functions/array/arrayResize.cpp | 2 +- src/Functions/array/arrayReverse.cpp | 2 +- src/Functions/array/arraySlice.cpp | 2 +- src/Functions/array/arraySort.cpp | 2 +- src/Functions/array/arraySplit.cpp | 2 +- src/Functions/array/arrayUniq.cpp | 2 +- src/Functions/array/arrayWithConstant.cpp | 2 +- src/Functions/array/arrayZip.cpp | 2 +- src/Functions/array/countEqual.cpp | 2 +- src/Functions/array/emptyArray.cpp | 2 +- src/Functions/array/emptyArrayToSingle.cpp | 2 +- src/Functions/array/has.cpp | 2 +- src/Functions/array/hasAll.cpp | 2 +- src/Functions/array/hasAny.cpp | 2 +- src/Functions/array/hasSubstr.cpp | 2 +- src/Functions/array/indexOf.cpp | 2 +- src/Functions/array/length.cpp | 2 +- src/Functions/array/mapOp.cpp | 2 +- src/Functions/array/mapPopulateSeries.cpp | 2 +- src/Functions/array/range.cpp | 2 +- .../array/registerFunctionsArray.cpp | 80 -------- src/Functions/asin.cpp | 2 +- src/Functions/asinh.cpp | 2 +- src/Functions/assumeNotNull.cpp | 2 +- src/Functions/atan.cpp | 2 +- src/Functions/atan2.cpp | 2 +- src/Functions/atanh.cpp | 2 +- src/Functions/bar.cpp | 2 +- src/Functions/base64Decode.cpp | 2 +- src/Functions/base64Encode.cpp | 2 +- src/Functions/bitAnd.cpp | 2 +- src/Functions/bitBoolMaskAnd.cpp | 2 +- src/Functions/bitBoolMaskOr.cpp | 2 +- src/Functions/bitCount.cpp | 2 +- src/Functions/bitHammingDistance.cpp | 2 +- src/Functions/bitNot.cpp | 2 +- src/Functions/bitOr.cpp | 2 +- src/Functions/bitRotateLeft.cpp | 2 +- src/Functions/bitRotateRight.cpp | 2 +- src/Functions/bitShiftLeft.cpp | 2 +- src/Functions/bitShiftRight.cpp | 2 +- src/Functions/bitSlice.cpp | 2 +- src/Functions/bitSwapLastTwo.cpp | 2 +- src/Functions/bitTest.cpp | 2 +- src/Functions/bitTestAll.cpp | 2 +- src/Functions/bitTestAny.cpp | 2 +- src/Functions/bitWrapperFunc.cpp | 2 +- src/Functions/bitXor.cpp | 2 +- src/Functions/blockNumber.cpp | 2 +- src/Functions/blockSerializedSize.cpp | 2 +- src/Functions/blockSize.cpp | 2 +- src/Functions/byteSize.cpp | 2 +- src/Functions/caseWithExpression.cpp | 2 +- src/Functions/castOrDefault.cpp | 2 +- src/Functions/cbrt.cpp | 2 +- src/Functions/coalesce.cpp | 2 +- src/Functions/concat.cpp | 2 +- src/Functions/connectionId.cpp | 2 +- src/Functions/convertCharset.cpp | 2 +- src/Functions/cos.cpp | 2 +- src/Functions/cosh.cpp | 2 +- src/Functions/countDigits.cpp | 2 +- src/Functions/countMatches.cpp | 2 +- src/Functions/countSubstrings.cpp | 2 +- .../countSubstringsCaseInsensitive.cpp | 2 +- .../countSubstringsCaseInsensitiveUTF8.cpp | 2 +- src/Functions/currentDatabase.cpp | 2 +- src/Functions/currentProfiles.cpp | 2 +- src/Functions/currentRoles.cpp | 2 +- src/Functions/currentUser.cpp | 2 +- src/Functions/dateDiff.cpp | 2 +- src/Functions/dateName.cpp | 2 +- src/Functions/date_trunc.cpp | 2 +- src/Functions/decodeXMLComponent.cpp | 2 +- src/Functions/decrypt.cpp | 2 +- src/Functions/defaultValueOfArgumentType.cpp | 2 +- src/Functions/defaultValueOfTypeName.cpp | 2 +- src/Functions/degrees.cpp | 2 +- src/Functions/demange.cpp | 2 +- src/Functions/divide.cpp | 2 +- src/Functions/dumpColumnStructure.cpp | 2 +- src/Functions/empty.cpp | 2 +- src/Functions/encodeXMLComponent.cpp | 2 +- src/Functions/encrypt.cpp | 2 +- src/Functions/endsWith.cpp | 2 +- src/Functions/equals.cpp | 2 +- src/Functions/erf.cpp | 2 +- src/Functions/erfc.cpp | 2 +- src/Functions/errorCodeToName.cpp | 2 +- src/Functions/evalMLMethod.cpp | 2 +- src/Functions/exp.cpp | 2 +- src/Functions/exp10.cpp | 2 +- src/Functions/exp2.cpp | 2 +- src/Functions/extract.cpp | 2 +- src/Functions/extractAllGroupsHorizontal.cpp | 2 +- src/Functions/extractAllGroupsVertical.cpp | 2 +- src/Functions/extractGroups.cpp | 2 +- src/Functions/extractTextFromHTML.cpp | 2 +- src/Functions/filesystem.cpp | 2 +- src/Functions/finalizeAggregation.cpp | 2 +- src/Functions/flattenTuple.cpp | 2 +- src/Functions/formatDateTime.cpp | 2 +- src/Functions/formatReadableQuantity.cpp | 2 +- src/Functions/formatReadableSize.cpp | 2 +- src/Functions/formatReadableTimeDelta.cpp | 2 +- src/Functions/formatRow.cpp | 2 +- src/Functions/formatString.cpp | 2 +- src/Functions/fromModifiedJulianDay.cpp | 2 +- src/Functions/fromUnixTimestamp64Micro.cpp | 2 +- src/Functions/fromUnixTimestamp64Milli.cpp | 2 +- src/Functions/fromUnixTimestamp64Nano.cpp | 2 +- src/Functions/fuzzBits.cpp | 2 +- src/Functions/gcd.cpp | 2 +- src/Functions/generateUUIDv4.cpp | 2 +- src/Functions/geoToH3.cpp | 2 +- src/Functions/geoToS2.cpp | 2 +- src/Functions/geohashDecode.cpp | 2 +- src/Functions/geohashEncode.cpp | 2 +- src/Functions/geohashesInBox.cpp | 2 +- src/Functions/getFuzzerData.cpp | 4 +- src/Functions/getMacro.cpp | 2 +- src/Functions/getScalar.cpp | 2 +- src/Functions/getServerPort.cpp | 2 +- src/Functions/getSetting.cpp | 2 +- src/Functions/getSizeOfEnumType.cpp | 2 +- src/Functions/getTypeSerializationStreams.cpp | 2 +- src/Functions/globalVariable.cpp | 2 +- src/Functions/greatCircleDistance.cpp | 2 +- src/Functions/greater.cpp | 2 +- src/Functions/greaterOrEquals.cpp | 2 +- src/Functions/greatest.cpp | 2 +- src/Functions/h3CellAreaM2.cpp | 2 +- src/Functions/h3CellAreaRads2.cpp | 2 +- src/Functions/h3Distance.cpp | 2 +- src/Functions/h3EdgeAngle.cpp | 2 +- src/Functions/h3EdgeLengthKm.cpp | 2 +- src/Functions/h3EdgeLengthM.cpp | 2 +- src/Functions/h3ExactEdgeLengthKm.cpp | 2 +- src/Functions/h3ExactEdgeLengthM.cpp | 2 +- src/Functions/h3ExactEdgeLengthRads.cpp | 2 +- src/Functions/h3GetBaseCell.cpp | 2 +- ...DestinationIndexFromUnidirectionalEdge.cpp | 2 +- src/Functions/h3GetFaces.cpp | 2 +- .../h3GetIndexesFromUnidirectionalEdge.cpp | 2 +- ...h3GetOriginIndexFromUnidirectionalEdge.cpp | 2 +- src/Functions/h3GetPentagonIndexes.cpp | 2 +- src/Functions/h3GetRes0Indexes.cpp | 2 +- src/Functions/h3GetResolution.cpp | 2 +- src/Functions/h3GetUnidirectionalEdge.cpp | 2 +- .../h3GetUnidirectionalEdgeBoundary.cpp | 2 +- .../h3GetUnidirectionalEdgesFromHexagon.cpp | 2 +- src/Functions/h3HexAreaKm2.cpp | 2 +- src/Functions/h3HexAreaM2.cpp | 2 +- src/Functions/h3HexRing.cpp | 2 +- src/Functions/h3IndexesAreNeighbors.cpp | 2 +- src/Functions/h3IsPentagon.cpp | 2 +- src/Functions/h3IsResClassIII.cpp | 2 +- src/Functions/h3IsValid.cpp | 2 +- src/Functions/h3Line.cpp | 2 +- src/Functions/h3NumHexagons.cpp | 2 +- src/Functions/h3PointDist.cpp | 6 +- src/Functions/h3ToCenterChild.cpp | 2 +- src/Functions/h3ToChildren.cpp | 2 +- src/Functions/h3ToGeoBoundary.cpp | 2 +- src/Functions/h3ToParent.cpp | 2 +- src/Functions/h3ToString.cpp | 2 +- src/Functions/h3UnidirectionalEdgeIsValid.cpp | 2 +- src/Functions/h3kRing.cpp | 2 +- src/Functions/h3toGeo.cpp | 2 +- src/Functions/hasColumnInTable.cpp | 2 +- src/Functions/hasThreadFuzzer.cpp | 2 +- src/Functions/hasToken.cpp | 2 +- src/Functions/hasTokenCaseInsensitive.cpp | 2 +- src/Functions/hypot.cpp | 2 +- src/Functions/identity.cpp | 2 +- src/Functions/if.cpp | 2 +- src/Functions/ifNotFinite.cpp | 2 +- src/Functions/ifNull.cpp | 2 +- src/Functions/ignore.cpp | 2 +- src/Functions/ilike.cpp | 2 +- src/Functions/in.cpp | 2 +- src/Functions/indexHint.cpp | 2 +- src/Functions/initialQueryID.cpp | 2 +- src/Functions/initializeAggregation.cpp | 2 +- src/Functions/intDiv.cpp | 2 +- src/Functions/intDivOrZero.cpp | 2 +- src/Functions/intExp10.cpp | 2 +- src/Functions/intExp2.cpp | 2 +- src/Functions/isConstant.cpp | 2 +- src/Functions/isDecimalOverflow.cpp | 2 +- src/Functions/isFinite.cpp | 2 +- src/Functions/isIPAddressContainedIn.cpp | 2 +- src/Functions/isInfinite.cpp | 2 +- src/Functions/isNaN.cpp | 2 +- src/Functions/isNotNull.cpp | 2 +- src/Functions/isNull.cpp | 2 +- src/Functions/isNullable.cpp | 2 +- src/Functions/isValidUTF8.cpp | 2 +- src/Functions/isZeroOrNull.cpp | 2 +- src/Functions/jumpConsistentHash.cpp | 2 +- src/Functions/kostikConsistentHash.cpp | 2 +- src/Functions/lcm.cpp | 2 +- src/Functions/least.cpp | 2 +- src/Functions/left.cpp | 2 +- src/Functions/lemmatize.cpp | 2 +- src/Functions/lengthUTF8.cpp | 2 +- src/Functions/less.cpp | 2 +- src/Functions/lessOrEquals.cpp | 2 +- src/Functions/lgamma.cpp | 2 +- src/Functions/like.cpp | 2 +- src/Functions/log.cpp | 2 +- src/Functions/log10.cpp | 2 +- src/Functions/log1p.cpp | 2 +- src/Functions/log2.cpp | 2 +- src/Functions/logTrace.cpp | 2 +- src/Functions/lowCardinalityIndices.cpp | 2 +- src/Functions/lowCardinalityKeys.cpp | 2 +- src/Functions/lower.cpp | 2 +- src/Functions/lowerUTF8.cpp | 2 +- src/Functions/makeDate.cpp | 2 +- src/Functions/map.cpp | 2 +- src/Functions/mapFilter.cpp | 2 +- src/Functions/match.cpp | 2 +- src/Functions/materialize.cpp | 2 +- src/Functions/mathConstants.cpp | 4 +- src/Functions/max2.cpp | 2 +- src/Functions/meiliMatch.cpp | 2 +- src/Functions/min2.cpp | 2 +- src/Functions/minSampleSize.cpp | 2 +- src/Functions/minus.cpp | 2 +- src/Functions/modelEvaluate.cpp | 2 +- src/Functions/modulo.cpp | 4 +- src/Functions/moduloOrZero.cpp | 2 +- src/Functions/monthName.cpp | 2 +- src/Functions/multiFuzzyMatchAllIndices.cpp | 2 +- src/Functions/multiFuzzyMatchAny.cpp | 2 +- src/Functions/multiFuzzyMatchAnyIndex.cpp | 2 +- src/Functions/multiIf.cpp | 2 +- src/Functions/multiMatchAllIndices.cpp | 2 +- src/Functions/multiMatchAny.cpp | 2 +- src/Functions/multiMatchAnyIndex.cpp | 2 +- src/Functions/multiSearchAllPositions.cpp | 2 +- ...multiSearchAllPositionsCaseInsensitive.cpp | 2 +- ...iSearchAllPositionsCaseInsensitiveUTF8.cpp | 2 +- src/Functions/multiSearchAllPositionsUTF8.cpp | 2 +- src/Functions/multiSearchAny.cpp | 2 +- .../multiSearchAnyCaseInsensitive.cpp | 2 +- .../multiSearchAnyCaseInsensitiveUTF8.cpp | 2 +- src/Functions/multiSearchAnyUTF8.cpp | 2 +- src/Functions/multiSearchFirstIndex.cpp | 2 +- .../multiSearchFirstIndexCaseInsensitive.cpp | 2 +- ...ltiSearchFirstIndexCaseInsensitiveUTF8.cpp | 2 +- src/Functions/multiSearchFirstIndexUTF8.cpp | 2 +- src/Functions/multiSearchFirstPosition.cpp | 2 +- ...ultiSearchFirstPositionCaseInsensitive.cpp | 2 +- ...SearchFirstPositionCaseInsensitiveUTF8.cpp | 2 +- .../multiSearchFirstPositionUTF8.cpp | 2 +- src/Functions/multiply.cpp | 2 +- src/Functions/negate.cpp | 2 +- src/Functions/neighbor.cpp | 2 +- src/Functions/normalizeQuery.cpp | 2 +- src/Functions/normalizeString.cpp | 2 +- src/Functions/normalizedQueryHash.cpp | 2 +- src/Functions/notEmpty.cpp | 2 +- src/Functions/notEquals.cpp | 2 +- src/Functions/notILike.cpp | 2 +- src/Functions/notLike.cpp | 2 +- src/Functions/now.cpp | 2 +- src/Functions/now64.cpp | 2 +- src/Functions/nowInBlock.cpp | 2 +- src/Functions/nullIf.cpp | 2 +- src/Functions/padString.cpp | 2 +- src/Functions/parseTimeDelta.cpp | 2 +- src/Functions/partitionId.cpp | 2 +- src/Functions/plus.cpp | 2 +- src/Functions/pointInEllipses.cpp | 2 +- src/Functions/pointInPolygon.cpp | 2 +- src/Functions/polygonArea.cpp | 2 +- src/Functions/polygonConvexHull.cpp | 2 +- src/Functions/polygonPerimeter.cpp | 2 +- src/Functions/polygonsDistance.cpp | 2 +- src/Functions/polygonsEquals.cpp | 2 +- src/Functions/polygonsIntersection.cpp | 2 +- src/Functions/polygonsSymDifference.cpp | 2 +- src/Functions/polygonsUnion.cpp | 2 +- src/Functions/polygonsWithin.cpp | 2 +- src/Functions/position.cpp | 2 +- src/Functions/positionCaseInsensitive.cpp | 2 +- src/Functions/positionCaseInsensitiveUTF8.cpp | 2 +- src/Functions/positionUTF8.cpp | 2 +- src/Functions/pow.cpp | 2 +- src/Functions/queryID.cpp | 2 +- src/Functions/radians.cpp | 2 +- src/Functions/rand.cpp | 2 +- src/Functions/rand64.cpp | 2 +- src/Functions/randConstant.cpp | 2 +- src/Functions/randomFixedString.cpp | 2 +- src/Functions/randomPrintableASCII.cpp | 2 +- src/Functions/randomString.cpp | 2 +- src/Functions/randomStringUTF8.cpp | 2 +- src/Functions/readWkt.cpp | 2 +- src/Functions/regexpQuoteMeta.cpp | 2 +- src/Functions/registerFunctions.cpp | 132 +------------ src/Functions/registerFunctionsArithmetic.cpp | 92 --------- src/Functions/registerFunctionsComparison.cpp | 24 --- .../registerFunctionsConditional.cpp | 18 -- .../registerFunctionsConsistentHashing.cpp | 14 -- src/Functions/registerFunctionsDateTime.cpp | 170 ---------------- src/Functions/registerFunctionsFormatting.cpp | 21 -- src/Functions/registerFunctionsGeo.cpp | 166 ---------------- .../registerFunctionsHigherOrder.cpp | 43 ---- .../registerFunctionsIntrospection.cpp | 27 --- src/Functions/registerFunctionsMath.cpp | 85 -------- .../registerFunctionsMiscellaneous.cpp | 187 ------------------ src/Functions/registerFunctionsNull.cpp | 30 --- src/Functions/registerFunctionsRandom.cpp | 28 --- .../registerFunctionsReinterpret.cpp | 13 -- src/Functions/registerFunctionsSnowflake.cpp | 22 --- src/Functions/registerFunctionsString.cpp | 126 ------------ .../registerFunctionsStringRegexp.cpp | 50 ----- .../registerFunctionsStringSearch.cpp | 74 ------- src/Functions/registerFunctionsTuple.cpp | 17 -- .../registerFunctionsUnixTimestamp64.cpp | 25 --- src/Functions/registerFunctionsVisitParam.cpp | 25 --- src/Functions/reinterpretAs.cpp | 2 +- src/Functions/repeat.cpp | 2 +- src/Functions/replaceAll.cpp | 2 +- src/Functions/replaceOne.cpp | 2 +- src/Functions/replaceRegexpAll.cpp | 2 +- src/Functions/replaceRegexpOne.cpp | 2 +- src/Functions/replicate.cpp | 2 +- src/Functions/reverse.cpp | 2 +- src/Functions/reverseUTF8.cpp | 2 +- src/Functions/right.cpp | 2 +- src/Functions/roundAge.cpp | 2 +- src/Functions/roundDuration.cpp | 2 +- src/Functions/roundToExp2.cpp | 2 +- src/Functions/rowNumberInAllBlocks.cpp | 2 +- src/Functions/rowNumberInBlock.cpp | 2 +- src/Functions/runningAccumulate.cpp | 2 +- src/Functions/runningConcurrency.cpp | 2 +- src/Functions/runningDifference.cpp | 2 +- ...unningDifferenceStartingWithFirstValue.cpp | 2 +- src/Functions/s2CapContains.cpp | 2 +- src/Functions/s2CapUnion.cpp | 2 +- src/Functions/s2CellsIntersect.cpp | 2 +- src/Functions/s2GetNeighbors.cpp | 2 +- src/Functions/s2RectAdd.cpp | 2 +- src/Functions/s2RectContains.cpp | 2 +- src/Functions/s2RectIntersection.cpp | 2 +- src/Functions/s2RectUnion.cpp | 2 +- src/Functions/s2ToGeo.cpp | 2 +- src/Functions/serverConstants.cpp | 29 ++- src/Functions/sigmoid.cpp | 2 +- src/Functions/sign.cpp | 2 +- src/Functions/sin.cpp | 2 +- src/Functions/sinh.cpp | 2 +- src/Functions/sleep.cpp | 2 +- src/Functions/sleepEachRow.cpp | 2 +- src/Functions/snowflake.cpp | 8 +- src/Functions/sqrt.cpp | 2 +- src/Functions/startsWith.cpp | 2 +- src/Functions/stem.cpp | 2 +- src/Functions/stringCutToZero.cpp | 2 +- src/Functions/stringToH3.cpp | 2 +- src/Functions/substring.cpp | 2 +- src/Functions/subtractDays.cpp | 2 +- src/Functions/subtractHours.cpp | 2 +- src/Functions/subtractMinutes.cpp | 2 +- src/Functions/subtractMonths.cpp | 2 +- src/Functions/subtractQuarters.cpp | 2 +- src/Functions/subtractSeconds.cpp | 2 +- src/Functions/subtractWeeks.cpp | 2 +- src/Functions/subtractYears.cpp | 2 +- src/Functions/svg.cpp | 2 +- src/Functions/synonyms.cpp | 2 +- src/Functions/tan.cpp | 2 +- src/Functions/tanh.cpp | 2 +- src/Functions/tgamma.cpp | 2 +- src/Functions/throwIf.cpp | 2 +- src/Functions/tid.cpp | 2 +- src/Functions/timeSlot.cpp | 2 +- src/Functions/timeSlots.cpp | 2 +- src/Functions/timezoneOf.cpp | 2 +- src/Functions/timezoneOffset.cpp | 2 +- src/Functions/toBool.cpp | 2 +- src/Functions/toColumnTypeName.cpp | 2 +- src/Functions/toCustomWeek.cpp | 2 +- src/Functions/toDayOfMonth.cpp | 2 +- src/Functions/toDayOfWeek.cpp | 2 +- src/Functions/toDayOfYear.cpp | 2 +- src/Functions/toFixedString.cpp | 2 +- src/Functions/toHour.cpp | 2 +- src/Functions/toISOWeek.cpp | 2 +- src/Functions/toISOYear.cpp | 2 +- src/Functions/toJSONString.cpp | 2 +- src/Functions/toLastDayOfMonth.cpp | 2 +- src/Functions/toLowCardinality.cpp | 2 +- src/Functions/toMinute.cpp | 2 +- src/Functions/toModifiedJulianDay.cpp | 2 +- src/Functions/toMonday.cpp | 2 +- src/Functions/toMonth.cpp | 2 +- src/Functions/toNullable.cpp | 2 +- src/Functions/toQuarter.cpp | 2 +- src/Functions/toRelativeDayNum.cpp | 2 +- src/Functions/toRelativeHourNum.cpp | 2 +- src/Functions/toRelativeMinuteNum.cpp | 2 +- src/Functions/toRelativeMonthNum.cpp | 2 +- src/Functions/toRelativeQuarterNum.cpp | 2 +- src/Functions/toRelativeSecondNum.cpp | 2 +- src/Functions/toRelativeWeekNum.cpp | 2 +- src/Functions/toRelativeYearNum.cpp | 2 +- src/Functions/toSecond.cpp | 2 +- src/Functions/toStartOfDay.cpp | 2 +- src/Functions/toStartOfFifteenMinutes.cpp | 2 +- src/Functions/toStartOfFiveMinutes.cpp | 2 +- src/Functions/toStartOfHour.cpp | 2 +- src/Functions/toStartOfISOYear.cpp | 2 +- src/Functions/toStartOfInterval.cpp | 2 +- src/Functions/toStartOfMinute.cpp | 2 +- src/Functions/toStartOfMonth.cpp | 2 +- src/Functions/toStartOfQuarter.cpp | 2 +- src/Functions/toStartOfSecond.cpp | 2 +- src/Functions/toStartOfSubsecond.cpp | 6 +- src/Functions/toStartOfTenMinutes.cpp | 2 +- src/Functions/toStartOfYear.cpp | 2 +- src/Functions/toTime.cpp | 2 +- src/Functions/toTimezone.cpp | 2 +- src/Functions/toTypeName.cpp | 2 +- src/Functions/toUnixTimestamp64Micro.cpp | 2 +- src/Functions/toUnixTimestamp64Milli.cpp | 2 +- src/Functions/toUnixTimestamp64Nano.cpp | 2 +- src/Functions/toValidUTF8.cpp | 2 +- src/Functions/toYYYYMM.cpp | 2 +- src/Functions/toYYYYMMDD.cpp | 2 +- src/Functions/toYYYYMMDDhhmmss.cpp | 2 +- src/Functions/toYear.cpp | 2 +- src/Functions/today.cpp | 2 +- src/Functions/tokenExtractors.cpp | 2 +- src/Functions/transform.cpp | 2 +- src/Functions/translate.cpp | 2 +- src/Functions/trap.cpp | 10 +- src/Functions/trim.cpp | 2 +- src/Functions/tryBase64Decode.cpp | 2 +- src/Functions/tuple.cpp | 2 +- src/Functions/tupleElement.cpp | 2 +- src/Functions/tupleHammingDistance.cpp | 2 +- src/Functions/tupleToNameValuePairs.cpp | 2 +- src/Functions/upper.cpp | 2 +- src/Functions/upperUTF8.cpp | 2 +- src/Functions/validateNestedArraySizes.cpp | 2 +- src/Functions/vectorFunctions.cpp | 2 +- src/Functions/visibleWidth.cpp | 2 +- src/Functions/visitParamExtractBool.cpp | 2 +- src/Functions/visitParamExtractFloat.cpp | 2 +- src/Functions/visitParamExtractInt.cpp | 2 +- src/Functions/visitParamExtractRaw.cpp | 2 +- src/Functions/visitParamExtractString.cpp | 2 +- src/Functions/visitParamExtractUInt.cpp | 2 +- src/Functions/visitParamHas.cpp | 2 +- src/Functions/wkt.cpp | 2 +- src/Functions/yesterday.cpp | 2 +- src/Functions/ztest.cpp | 2 +- 581 files changed, 667 insertions(+), 2150 deletions(-) create mode 100644 src/Common/register_objects.cpp create mode 100644 src/Common/register_objects.h delete mode 100644 src/Functions/URL/registerFunctionsURL.cpp delete mode 100644 src/Functions/array/registerFunctionsArray.cpp delete mode 100644 src/Functions/registerFunctionsArithmetic.cpp delete mode 100644 src/Functions/registerFunctionsComparison.cpp delete mode 100644 src/Functions/registerFunctionsConditional.cpp delete mode 100644 src/Functions/registerFunctionsConsistentHashing.cpp delete mode 100644 src/Functions/registerFunctionsDateTime.cpp delete mode 100644 src/Functions/registerFunctionsFormatting.cpp delete mode 100644 src/Functions/registerFunctionsGeo.cpp delete mode 100644 src/Functions/registerFunctionsHigherOrder.cpp delete mode 100644 src/Functions/registerFunctionsIntrospection.cpp delete mode 100644 src/Functions/registerFunctionsMath.cpp delete mode 100644 src/Functions/registerFunctionsMiscellaneous.cpp delete mode 100644 src/Functions/registerFunctionsNull.cpp delete mode 100644 src/Functions/registerFunctionsRandom.cpp delete mode 100644 src/Functions/registerFunctionsReinterpret.cpp delete mode 100644 src/Functions/registerFunctionsSnowflake.cpp delete mode 100644 src/Functions/registerFunctionsString.cpp delete mode 100644 src/Functions/registerFunctionsStringRegexp.cpp delete mode 100644 src/Functions/registerFunctionsStringSearch.cpp delete mode 100644 src/Functions/registerFunctionsTuple.cpp delete mode 100644 src/Functions/registerFunctionsUnixTimestamp64.cpp delete mode 100644 src/Functions/registerFunctionsVisitParam.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f4d3be14da6..49acf55892a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -144,8 +144,8 @@ endif () list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD}) list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON}) -list (APPEND dbms_sources Functions/IFunction.cpp Functions/FunctionFactory.cpp Functions/FunctionHelpers.cpp Functions/extractTimeZoneFromFunctionArguments.cpp Functions/replicate.cpp Functions/FunctionsLogical.cpp) -list (APPEND dbms_headers Functions/IFunction.h Functions/FunctionFactory.h Functions/FunctionHelpers.h Functions/extractTimeZoneFromFunctionArguments.h Functions/replicate.h Functions/FunctionsLogical.h) +list (APPEND dbms_sources Functions/IFunction.cpp Functions/FunctionFactory.cpp Functions/FunctionHelpers.cpp Functions/extractTimeZoneFromFunctionArguments.cpp Functions/FunctionsLogical.cpp) +list (APPEND dbms_headers Functions/IFunction.h Functions/FunctionFactory.h Functions/FunctionHelpers.h Functions/extractTimeZoneFromFunctionArguments.h Functions/FunctionsLogical.h) list (APPEND dbms_sources AggregateFunctions/IAggregateFunction.cpp diff --git a/src/Common/register_objects.cpp b/src/Common/register_objects.cpp new file mode 100644 index 00000000000..93b0feb5f8d --- /dev/null +++ b/src/Common/register_objects.cpp @@ -0,0 +1,12 @@ +#include + +namespace DB +{ + +FunctionRegisterMap & FunctionRegisterMap::instance() +{ + static FunctionRegisterMap map; + return map; +} + +} diff --git a/src/Common/register_objects.h b/src/Common/register_objects.h new file mode 100644 index 00000000000..0a08fdf43a4 --- /dev/null +++ b/src/Common/register_objects.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include + +namespace DB +{ + +class FunctionFactory; + +using FunctionRegisterFunctionPtr = void (*)(::DB::FunctionFactory &); + +struct FunctionRegisterMap : public std::unordered_map +{ + static FunctionRegisterMap & instance(); +}; + +struct FunctionRegister +{ + FunctionRegister(std::string_view name, FunctionRegisterFunctionPtr func_ptr) + { + FunctionRegisterMap::instance().emplace(std::move(name), func_ptr); + } +}; + +} + +#define REGISTER_FUNCTION_IMPL(fn, func_name, register_name) \ + void func_name(::DB::FunctionFactory & factory); \ + static ::DB::FunctionRegister register_name(#fn, func_name); \ + void func_name(::DB::FunctionFactory & factory) + +#define REGISTER_FUNCTION(fn) REGISTER_FUNCTION_IMPL(fn, registerFunction##fn, REGISTER_FUNCTION_##fn) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 208b3e6a77e..1bc95b49dbe 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -6,13 +6,14 @@ add_subdirectory(divide) include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") add_headers_and_sources(clickhouse_functions .) -list(REMOVE_ITEM clickhouse_functions_sources IFunction.cpp FunctionFactory.cpp FunctionHelpers.cpp) -list(REMOVE_ITEM clickhouse_functions_headers IFunction.h FunctionFactory.h FunctionHelpers.h) +list(REMOVE_ITEM clickhouse_functions_sources IFunction.cpp FunctionFactory.cpp FunctionHelpers.cpp extractTimeZoneFromFunctionArguments.cpp FunctionsLogical.cpp) +list(REMOVE_ITEM clickhouse_functions_headers IFunction.h FunctionFactory.h FunctionHelpers.h extractTimeZoneFromFunctionArguments.h FunctionsLogical.h) -add_library(clickhouse_functions ${clickhouse_functions_sources}) +add_library(clickhouse_functions_obj OBJECT ${clickhouse_functions_sources}) -target_link_libraries(clickhouse_functions - PUBLIC +list (APPEND OBJECT_LIBS $) + +list (APPEND PUBLIC_LIBS ch_contrib::wyhash ch_contrib::cityhash ch_contrib::farmhash @@ -24,27 +25,28 @@ target_link_libraries(clickhouse_functions ch_contrib::metrohash ch_contrib::murmurhash ch_contrib::hashidsxx +) - PRIVATE +list (APPEND PRIVATE_LIBS ch_contrib::zlib boost::filesystem divide_impl ) if (TARGET OpenSSL::Crypto) - target_link_libraries(clickhouse_functions PUBLIC OpenSSL::Crypto) + list (APPEND PUBLIC_LIBS OpenSSL::Crypto) endif() if (OMIT_HEAVY_DEBUG_SYMBOLS) - target_compile_options(clickhouse_functions PRIVATE "-g0") + target_compile_options(clickhouse_functions_obj PRIVATE "-g0") endif() if (TARGET ch_contrib::icu) - target_link_libraries (clickhouse_functions PRIVATE ch_contrib::icu) + list (APPEND PRIVATE_LIBS ch_contrib::icu) endif () if (TARGET ch_contrib::fastops) - target_link_libraries (clickhouse_functions PRIVATE ch_contrib::fastops) + list (APPEND PRIVATE_LIBS ch_contrib::fastops) endif () if (ENABLE_EXAMPLES) @@ -52,45 +54,46 @@ if (ENABLE_EXAMPLES) endif () if (TARGET ch_contrib::llvm) - target_link_libraries(clickhouse_functions PRIVATE ch_contrib::llvm) + list (APPEND PRIVATE_LIBS ch_contrib::llvm) endif () if (TARGET ch_contrib::base64) - target_link_libraries(clickhouse_functions PRIVATE ch_contrib::base64) + list (APPEND PRIVATE_LIBS ch_contrib::base64) endif() -target_link_libraries(clickhouse_functions PRIVATE ch_contrib::lz4) +list (APPEND PRIVATE_LIBS ch_contrib::lz4) if (ENABLE_NLP) - target_link_libraries(clickhouse_functions PRIVATE ch_contrib::cld2) + list (APPEND PRIVATE_LIBS ch_contrib::cld2) endif() if (TARGET ch_contrib::h3) - target_link_libraries (clickhouse_functions PRIVATE ch_contrib::h3) + list (APPEND PRIVATE_LIBS ch_contrib::h3) endif() if (TARGET ch_contrib::vectorscan) - target_link_libraries(clickhouse_functions PRIVATE ch_contrib::vectorscan) + list (APPEND PRIVATE_LIBS ch_contrib::vectorscan) endif() if (TARGET ch_contrib::simdjson) - target_link_libraries(clickhouse_functions PRIVATE ch_contrib::simdjson) + list (APPEND PRIVATE_LIBS ch_contrib::simdjson) endif() if (TARGET ch_contrib::rapidjson) - target_link_libraries(clickhouse_functions PRIVATE ch_contrib::rapidjson) + list (APPEND PRIVATE_LIBS ch_contrib::rapidjson) endif() add_subdirectory(GatherUtils) -target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_gatherutils) +list (APPEND PRIVATE_LIBS clickhouse_functions_gatherutils) add_subdirectory(URL) -target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_url) +list (APPEND OBJECT_LIBS $) add_subdirectory(array) -target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_array) +list (APPEND OBJECT_LIBS $) add_subdirectory(JSONPath) +list (APPEND PRIVATE_LIBS clickhouse_functions_jsonpath) # Signed integer overflow on user-provided data inside boost::geometry - ignore. set_source_files_properties("pointInPolygon.cpp" PROPERTIES COMPILE_FLAGS -fno-sanitize=signed-integer-overflow) @@ -98,3 +101,15 @@ set_source_files_properties("pointInPolygon.cpp" PROPERTIES COMPILE_FLAGS -fno-s if (ENABLE_FUZZING) add_compile_definitions(FUZZING_MODE=1) endif () + +target_link_libraries(clickhouse_functions_obj PUBLIC ${PUBLIC_LIBS} PRIVATE ${PRIVATE_LIBS}) + +if (USE_STATIC_LIBRARIES OR NOT SPLIT_SHARED_LIBRARIES) + # Used to forward the linking information to the final binaries such as clickhouse / unit_tests_dbms, + # since such information are lost after we convert to OBJECT target + add_library(clickhouse_functions INTERFACE) + target_link_libraries(clickhouse_functions INTERFACE ${OBJECT_LIBS} ${PUBLIC_LIBS} ${PRIVATE_LIBS}) +else() + add_library(clickhouse_functions SHARED ${OBJECT_LIBS}) + target_link_libraries(clickhouse_functions PUBLIC ${PUBLIC_LIBS} PRIVATE ${PRIVATE_LIBS}) +endif () diff --git a/src/Functions/CRC.cpp b/src/Functions/CRC.cpp index b7c6c1195ea..10045a246c0 100644 --- a/src/Functions/CRC.cpp +++ b/src/Functions/CRC.cpp @@ -143,7 +143,7 @@ void registerFunctionCRCImpl(FunctionFactory & factory) factory.registerFunction(T::name, FunctionFactory::CaseInsensitive); } -void registerFunctionCRC(FunctionFactory & factory) +REGISTER_FUNCTION(CRC) { registerFunctionCRCImpl(factory); registerFunctionCRCImpl(factory); diff --git a/src/Functions/CastOverloadResolver.cpp b/src/Functions/CastOverloadResolver.cpp index fd6fecc37d6..761e49fe7bd 100644 --- a/src/Functions/CastOverloadResolver.cpp +++ b/src/Functions/CastOverloadResolver.cpp @@ -5,7 +5,7 @@ namespace DB { -void registerCastOverloadResolvers(FunctionFactory & factory) +REGISTER_FUNCTION(CastOverloadResolvers) { factory.registerFunction>(FunctionFactory::CaseInsensitive); factory.registerFunction>(); diff --git a/src/Functions/FunctionChar.cpp b/src/Functions/FunctionChar.cpp index f50e212d9b2..c022fda04c8 100644 --- a/src/Functions/FunctionChar.cpp +++ b/src/Functions/FunctionChar.cpp @@ -113,7 +113,7 @@ private: } }; -void registerFunctionChar(FunctionFactory & factory) +REGISTER_FUNCTION(Char) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/FunctionFQDN.cpp b/src/Functions/FunctionFQDN.cpp index 7385de807a1..c4ac409ca04 100644 --- a/src/Functions/FunctionFQDN.cpp +++ b/src/Functions/FunctionFQDN.cpp @@ -44,7 +44,7 @@ public: }; -void registerFunctionFQDN(FunctionFactory & factory) +REGISTER_FUNCTION(FQDN) { factory.registerFunction(FunctionFactory::CaseInsensitive); factory.registerFunction("fullHostName"); diff --git a/src/Functions/FunctionFactory.h b/src/Functions/FunctionFactory.h index 13b14559ec4..6758592558c 100644 --- a/src/Functions/FunctionFactory.h +++ b/src/Functions/FunctionFactory.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include diff --git a/src/Functions/FunctionFile.cpp b/src/Functions/FunctionFile.cpp index c85051e61c0..d0f87942395 100644 --- a/src/Functions/FunctionFile.cpp +++ b/src/Functions/FunctionFile.cpp @@ -94,7 +94,7 @@ public: }; -void registerFunctionFile(FunctionFactory & factory) +REGISTER_FUNCTION(File) { factory.registerFunction(); } diff --git a/src/Functions/FunctionHashID.cpp b/src/Functions/FunctionHashID.cpp index bd875a9d4ff..829b3d9d2f6 100644 --- a/src/Functions/FunctionHashID.cpp +++ b/src/Functions/FunctionHashID.cpp @@ -4,7 +4,7 @@ namespace DB { -void registerFunctionHashID(FunctionFactory & factory) +REGISTER_FUNCTION(HashID) { factory.registerFunction(); } diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp index da1061e4b3e..825ae6a665a 100644 --- a/src/Functions/FunctionJoinGet.cpp +++ b/src/Functions/FunctionJoinGet.cpp @@ -96,7 +96,7 @@ FunctionBasePtr JoinGetOverloadResolver::buildImpl(const ColumnsWithTyp return std::make_unique>(getContext(), table_lock, storage_join, attr_name, argument_types, return_type); } -void registerFunctionJoinGet(FunctionFactory & factory) +REGISTER_FUNCTION(JoinGet) { // joinGet factory.registerFunction>(); diff --git a/src/Functions/FunctionSQLJSON.cpp b/src/Functions/FunctionSQLJSON.cpp index a316d9de7ab..e9d179ad6ae 100644 --- a/src/Functions/FunctionSQLJSON.cpp +++ b/src/Functions/FunctionSQLJSON.cpp @@ -5,7 +5,7 @@ namespace DB { -void registerFunctionsSQLJSON(FunctionFactory & factory) +REGISTER_FUNCTION(SQLJSON) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/FunctionShowCertificate.cpp b/src/Functions/FunctionShowCertificate.cpp index e978f77244c..2e2b7506803 100644 --- a/src/Functions/FunctionShowCertificate.cpp +++ b/src/Functions/FunctionShowCertificate.cpp @@ -4,7 +4,7 @@ namespace DB { -void registerFunctionShowCertificate(FunctionFactory & factory) +REGISTER_FUNCTION(ShowCertificate) { factory.registerFunction(); } diff --git a/src/Functions/FunctionsBase58.cpp b/src/Functions/FunctionsBase58.cpp index a336fbe6b07..3aa141c3dde 100644 --- a/src/Functions/FunctionsBase58.cpp +++ b/src/Functions/FunctionsBase58.cpp @@ -3,12 +3,12 @@ namespace DB { -void registerFunctionBase58Encode(FunctionFactory & factory) +REGISTER_FUNCTION(Base58Encode) { factory.registerFunction>(); } -void registerFunctionBase58Decode(FunctionFactory & factory) +REGISTER_FUNCTION(Base58Decode) { factory.registerFunction>(); } diff --git a/src/Functions/FunctionsBinaryRepresentation.cpp b/src/Functions/FunctionsBinaryRepresentation.cpp index a13558133d3..d53963ace8a 100644 --- a/src/Functions/FunctionsBinaryRepresentation.cpp +++ b/src/Functions/FunctionsBinaryRepresentation.cpp @@ -621,7 +621,7 @@ public: } }; -void registerFunctionsBinaryRepr(FunctionFactory & factory) +REGISTER_FUNCTION(BinaryRepr) { factory.registerFunction>(FunctionFactory::CaseInsensitive); factory.registerFunction>(FunctionFactory::CaseInsensitive); diff --git a/src/Functions/FunctionsBitToArray.cpp b/src/Functions/FunctionsBitToArray.cpp index 76496b102cd..22a56ba35e6 100644 --- a/src/Functions/FunctionsBitToArray.cpp +++ b/src/Functions/FunctionsBitToArray.cpp @@ -329,7 +329,7 @@ public: } -void registerFunctionsBitToArray(FunctionFactory & factory) +REGISTER_FUNCTION(BitToArray) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/FunctionsBitmap.cpp b/src/Functions/FunctionsBitmap.cpp index 159d0ff6777..5dcc20c3dfb 100644 --- a/src/Functions/FunctionsBitmap.cpp +++ b/src/Functions/FunctionsBitmap.cpp @@ -7,7 +7,7 @@ namespace DB { -void registerFunctionsBitmap(FunctionFactory & factory) +REGISTER_FUNCTION(Bitmap) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/FunctionsCharsetClassification.cpp b/src/Functions/FunctionsCharsetClassification.cpp index af6c1de2768..a25da8f6c13 100644 --- a/src/Functions/FunctionsCharsetClassification.cpp +++ b/src/Functions/FunctionsCharsetClassification.cpp @@ -143,7 +143,7 @@ struct NameDetectLanguageUnknown using FunctionDetectCharset = FunctionTextClassificationString, NameDetectCharset>; using FunctionDetectLanguageUnknown = FunctionTextClassificationString, NameDetectLanguageUnknown>; -void registerFunctionDetectCharset(FunctionFactory & factory) +REGISTER_FUNCTION(DetectCharset) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/FunctionsCodingIP.cpp b/src/Functions/FunctionsCodingIP.cpp index 1fdbbf3e9bb..eaf62e232f7 100644 --- a/src/Functions/FunctionsCodingIP.cpp +++ b/src/Functions/FunctionsCodingIP.cpp @@ -1128,7 +1128,7 @@ public: struct NameFunctionIPv4NumToString { static constexpr auto name = "IPv4NumToString"; }; struct NameFunctionIPv4NumToStringClassC { static constexpr auto name = "IPv4NumToStringClassC"; }; -void registerFunctionsCoding(FunctionFactory & factory) +REGISTER_FUNCTION(Coding) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/FunctionsCodingUUID.cpp b/src/Functions/FunctionsCodingUUID.cpp index 5ac1d585325..9309c4cdbeb 100644 --- a/src/Functions/FunctionsCodingUUID.cpp +++ b/src/Functions/FunctionsCodingUUID.cpp @@ -229,7 +229,7 @@ public: } }; -void registerFunctionsCodingUUID(FunctionFactory & factory) +REGISTER_FUNCTION(CodingUUID) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index 7f8e9148032..ae6ad0a6034 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -5,11 +5,7 @@ namespace DB { -void registerFunctionFixedString(FunctionFactory & factory); - -void registerCastOverloadResolvers(FunctionFactory & factory); - -void registerFunctionsConversion(FunctionFactory & factory) +REGISTER_FUNCTION(Conversion) { factory.registerFunction(); factory.registerFunction(); @@ -41,12 +37,8 @@ void registerFunctionsConversion(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); - registerFunctionFixedString(factory); - factory.registerFunction(); - registerCastOverloadResolvers(factory); - factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/FunctionsEmbeddedDictionaries.cpp b/src/Functions/FunctionsEmbeddedDictionaries.cpp index eeaea9a32a5..570f717f3f6 100644 --- a/src/Functions/FunctionsEmbeddedDictionaries.cpp +++ b/src/Functions/FunctionsEmbeddedDictionaries.cpp @@ -5,7 +5,7 @@ namespace DB { -void registerFunctionsEmbeddedDictionaries(FunctionFactory & factory) +REGISTER_FUNCTION(EmbeddedDictionaries) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/FunctionsExternalDictionaries.cpp b/src/Functions/FunctionsExternalDictionaries.cpp index 0cd56f87df4..371b838635b 100644 --- a/src/Functions/FunctionsExternalDictionaries.cpp +++ b/src/Functions/FunctionsExternalDictionaries.cpp @@ -5,7 +5,7 @@ namespace DB { -void registerFunctionsExternalDictionaries(FunctionFactory & factory) +REGISTER_FUNCTION(ExternalDictionaries) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/FunctionsHashing.cpp b/src/Functions/FunctionsHashing.cpp index 901234e5443..31ffae2d98a 100644 --- a/src/Functions/FunctionsHashing.cpp +++ b/src/Functions/FunctionsHashing.cpp @@ -6,7 +6,7 @@ namespace DB { -void registerFunctionsHashing(FunctionFactory & factory) +REGISTER_FUNCTION(Hashing) { #if USE_SSL factory.registerFunction(); diff --git a/src/Functions/FunctionsJSON.cpp b/src/Functions/FunctionsJSON.cpp index 9ab27c2976a..6beb8155965 100644 --- a/src/Functions/FunctionsJSON.cpp +++ b/src/Functions/FunctionsJSON.cpp @@ -1443,7 +1443,7 @@ public: } }; -void registerFunctionsJSON(FunctionFactory & factory) +REGISTER_FUNCTION(JSON) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/FunctionsLanguageClassification.cpp b/src/Functions/FunctionsLanguageClassification.cpp index 521a4b0301e..18579e674ec 100644 --- a/src/Functions/FunctionsLanguageClassification.cpp +++ b/src/Functions/FunctionsLanguageClassification.cpp @@ -221,7 +221,7 @@ struct NameDetectLanguage using FunctionDetectLanguage = FunctionTextClassificationString; -void registerFunctionsDetectLanguage(FunctionFactory & factory) +REGISTER_FUNCTION(DetectLanguage) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp index 392cbac8a1b..05ec1e54c94 100644 --- a/src/Functions/FunctionsLogical.cpp +++ b/src/Functions/FunctionsLogical.cpp @@ -22,7 +22,7 @@ namespace DB { -void registerFunctionsLogical(FunctionFactory & factory) +REGISTER_FUNCTION(Logical) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/FunctionsProgrammingClassification.cpp b/src/Functions/FunctionsProgrammingClassification.cpp index 2c95e22f239..8a552a30e65 100644 --- a/src/Functions/FunctionsProgrammingClassification.cpp +++ b/src/Functions/FunctionsProgrammingClassification.cpp @@ -112,7 +112,7 @@ struct NameDetectProgrammingLanguage using FunctionDetectProgrammingLanguage = FunctionTextClassificationString; -void registerFunctionDetectProgrammingLanguage(FunctionFactory & factory) +REGISTER_FUNCTION(DetectProgrammingLanguage) { factory.registerFunction(); } diff --git a/src/Functions/FunctionsRound.cpp b/src/Functions/FunctionsRound.cpp index c5ad27a0b90..a08ebbaf038 100644 --- a/src/Functions/FunctionsRound.cpp +++ b/src/Functions/FunctionsRound.cpp @@ -5,7 +5,7 @@ namespace DB { -void registerFunctionsRound(FunctionFactory & factory) +REGISTER_FUNCTION(Round) { factory.registerFunction("round", FunctionFactory::CaseInsensitive); factory.registerFunction("roundBankers", FunctionFactory::CaseSensitive); diff --git a/src/Functions/FunctionsStringArray.cpp b/src/Functions/FunctionsStringArray.cpp index 0e73d6a33f5..660de15973f 100644 --- a/src/Functions/FunctionsStringArray.cpp +++ b/src/Functions/FunctionsStringArray.cpp @@ -27,7 +27,7 @@ DataTypePtr FunctionArrayStringConcat::getReturnTypeImpl(const DataTypes & argum return std::make_shared(); } -void registerFunctionsStringArray(FunctionFactory & factory) +REGISTER_FUNCTION(StringArray) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/FunctionsStringHash.cpp b/src/Functions/FunctionsStringHash.cpp index 46b5562ae75..e7dbe4087f2 100644 --- a/src/Functions/FunctionsStringHash.cpp +++ b/src/Functions/FunctionsStringHash.cpp @@ -743,7 +743,7 @@ using FunctionWordShingleMinHashArgUTF8 using FunctionWordShingleMinHashArgCaseInsensitiveUTF8 = FunctionsStringHash, NameWordShingleMinHashArgCaseInsensitiveUTF8, false, true>; -void registerFunctionsStringHash(FunctionFactory & factory) +REGISTER_FUNCTION(StringHash) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/FunctionsStringSimilarity.cpp b/src/Functions/FunctionsStringSimilarity.cpp index ee3bce4c0db..d1bb77d6d40 100644 --- a/src/Functions/FunctionsStringSimilarity.cpp +++ b/src/Functions/FunctionsStringSimilarity.cpp @@ -530,7 +530,7 @@ using FunctionNgramSearchUTF8 = FunctionsStringSimilarity, NameNgramSearchUTF8CaseInsensitive>; -void registerFunctionsStringSimilarity(FunctionFactory & factory) +REGISTER_FUNCTION(StringSimilarity) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/FunctionsTimeWindow.cpp b/src/Functions/FunctionsTimeWindow.cpp index 5022c42bc79..61d4f694a61 100644 --- a/src/Functions/FunctionsTimeWindow.cpp +++ b/src/Functions/FunctionsTimeWindow.cpp @@ -684,7 +684,7 @@ ColumnPtr FunctionTimeWindow::executeImpl(const ColumnsWithTypeAndName & a return TimeWindowImpl::dispatchForColumns(arguments, name); } -void registerFunctionsTimeWindow(FunctionFactory& factory) +REGISTER_FUNCTION(TimeWindow) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/FunctionsTonalityClassification.cpp b/src/Functions/FunctionsTonalityClassification.cpp index 2d8e47b9bcb..308f7264c49 100644 --- a/src/Functions/FunctionsTonalityClassification.cpp +++ b/src/Functions/FunctionsTonalityClassification.cpp @@ -81,7 +81,7 @@ struct NameDetectTonality using FunctionDetectTonality = FunctionTextClassificationFloat; -void registerFunctionDetectTonality(FunctionFactory & factory) +REGISTER_FUNCTION(DetectTonality) { factory.registerFunction(); } diff --git a/src/Functions/FunctionsTransactionCounters.cpp b/src/Functions/FunctionsTransactionCounters.cpp index f28cfb60dfa..b60a59a3432 100644 --- a/src/Functions/FunctionsTransactionCounters.cpp +++ b/src/Functions/FunctionsTransactionCounters.cpp @@ -61,7 +61,7 @@ public: } -void registerFunctionsTransactionCounters(FunctionFactory & factory) +REGISTER_FUNCTION(TransactionCounters) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/JSONPath/CMakeLists.txt b/src/Functions/JSONPath/CMakeLists.txt index bdc826008eb..3a3e313fe52 100644 --- a/src/Functions/JSONPath/CMakeLists.txt +++ b/src/Functions/JSONPath/CMakeLists.txt @@ -6,7 +6,6 @@ add_library(clickhouse_functions_jsonpath ${clickhouse_functions_jsonpath_source target_link_libraries(clickhouse_functions_jsonpath PRIVATE dbms) target_link_libraries(clickhouse_functions_jsonpath PRIVATE clickhouse_parsers) -target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_jsonpath) if (OMIT_HEAVY_DEBUG_SYMBOLS) target_compile_options(clickhouse_functions_jsonpath PRIVATE "-g0") diff --git a/src/Functions/SubtractSubSeconds.cpp b/src/Functions/SubtractSubSeconds.cpp index ac3a66bfc2d..b1c47700d13 100644 --- a/src/Functions/SubtractSubSeconds.cpp +++ b/src/Functions/SubtractSubSeconds.cpp @@ -6,19 +6,19 @@ namespace DB { using FunctionSubtractNanoseconds = FunctionDateOrDateTimeAddInterval; -void registerFunctionSubtractNanoseconds(FunctionFactory & factory) +REGISTER_FUNCTION(SubtractNanoseconds) { factory.registerFunction(); } using FunctionSubtractMicroseconds = FunctionDateOrDateTimeAddInterval; -void registerFunctionSubtractMicroseconds(FunctionFactory & factory) +REGISTER_FUNCTION(SubtractMicroseconds) { factory.registerFunction(); } using FunctionSubtractMilliseconds = FunctionDateOrDateTimeAddInterval; -void registerFunctionSubtractMilliseconds(FunctionFactory & factory) +REGISTER_FUNCTION(SubtractMilliseconds) { factory.registerFunction(); } diff --git a/src/Functions/URL/CMakeLists.txt b/src/Functions/URL/CMakeLists.txt index 28a72651315..6328476543d 100644 --- a/src/Functions/URL/CMakeLists.txt +++ b/src/Functions/URL/CMakeLists.txt @@ -1,6 +1,6 @@ include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") add_headers_and_sources(clickhouse_functions_url .) -add_library(clickhouse_functions_url ${clickhouse_functions_url_sources} ${clickhouse_functions_url_headers}) +add_library(clickhouse_functions_url OBJECT ${clickhouse_functions_url_sources} ${clickhouse_functions_url_headers}) target_link_libraries(clickhouse_functions_url PRIVATE dbms) if (OMIT_HEAVY_DEBUG_SYMBOLS) @@ -10,6 +10,7 @@ endif() # TODO: move Functions/Regexps.h to some lib and use here if (TARGET ch_contrib::vectorscan) target_link_libraries(clickhouse_functions_url PRIVATE ch_contrib::vectorscan) + list (APPEND PRIVATE_LIBS ch_contrib::vectorscan PARENT_SCOPE) endif() if (USE_GPERF) diff --git a/src/Functions/URL/URLHierarchy.cpp b/src/Functions/URL/URLHierarchy.cpp index e21450847b7..c0c3a7e5b37 100644 --- a/src/Functions/URL/URLHierarchy.cpp +++ b/src/Functions/URL/URLHierarchy.cpp @@ -104,7 +104,7 @@ public: struct NameURLPathHierarchy { static constexpr auto name = "URLPathHierarchy"; }; using FunctionURLPathHierarchy = FunctionTokens; -void registerFunctionURLPathHierarchy(FunctionFactory & factory) +REGISTER_FUNCTION(URLPathHierarchy) { factory.registerFunction(); } diff --git a/src/Functions/URL/URLPathHierarchy.cpp b/src/Functions/URL/URLPathHierarchy.cpp index 6f8832ddf65..c00e61e67fd 100644 --- a/src/Functions/URL/URLPathHierarchy.cpp +++ b/src/Functions/URL/URLPathHierarchy.cpp @@ -106,7 +106,7 @@ public: struct NameURLHierarchy { static constexpr auto name = "URLHierarchy"; }; using FunctionURLHierarchy = FunctionTokens; -void registerFunctionURLHierarchy(FunctionFactory & factory) +REGISTER_FUNCTION(URLHierarchy) { factory.registerFunction(); } diff --git a/src/Functions/URL/basename.cpp b/src/Functions/URL/basename.cpp index f97724020cb..6992f924ef2 100644 --- a/src/Functions/URL/basename.cpp +++ b/src/Functions/URL/basename.cpp @@ -34,7 +34,7 @@ struct ExtractBasename struct NameBasename { static constexpr auto name = "basename"; }; using FunctionBasename = FunctionStringToString, NameBasename>; -void registerFunctionBasename(FunctionFactory & factory) +REGISTER_FUNCTION(Basename) { factory.registerFunction(); } diff --git a/src/Functions/URL/cutFragment.cpp b/src/Functions/URL/cutFragment.cpp index 28ab287282d..3b99edf61a3 100644 --- a/src/Functions/URL/cutFragment.cpp +++ b/src/Functions/URL/cutFragment.cpp @@ -8,7 +8,7 @@ namespace DB struct NameCutFragment { static constexpr auto name = "cutFragment"; }; using FunctionCutFragment = FunctionStringToString>, NameCutFragment>; -void registerFunctionCutFragment(FunctionFactory & factory) +REGISTER_FUNCTION(CutFragment) { factory.registerFunction(); } diff --git a/src/Functions/URL/cutQueryString.cpp b/src/Functions/URL/cutQueryString.cpp index 44179a91f20..2886adc2e36 100644 --- a/src/Functions/URL/cutQueryString.cpp +++ b/src/Functions/URL/cutQueryString.cpp @@ -8,7 +8,7 @@ namespace DB struct NameCutQueryString { static constexpr auto name = "cutQueryString"; }; using FunctionCutQueryString = FunctionStringToString>, NameCutQueryString>; -void registerFunctionCutQueryString(FunctionFactory & factory) +REGISTER_FUNCTION(CutQueryString) { factory.registerFunction(); } diff --git a/src/Functions/URL/cutQueryStringAndFragment.cpp b/src/Functions/URL/cutQueryStringAndFragment.cpp index 0678fa8b6e9..4116b352542 100644 --- a/src/Functions/URL/cutQueryStringAndFragment.cpp +++ b/src/Functions/URL/cutQueryStringAndFragment.cpp @@ -8,7 +8,7 @@ namespace DB struct NameCutQueryStringAndFragment { static constexpr auto name = "cutQueryStringAndFragment"; }; using FunctionCutQueryStringAndFragment = FunctionStringToString>, NameCutQueryStringAndFragment>; -void registerFunctionCutQueryStringAndFragment(FunctionFactory & factory) +REGISTER_FUNCTION(CutQueryStringAndFragment) { factory.registerFunction(); } diff --git a/src/Functions/URL/cutToFirstSignificantSubdomain.cpp b/src/Functions/URL/cutToFirstSignificantSubdomain.cpp index 82eb366dae6..dddfbe4f4dd 100644 --- a/src/Functions/URL/cutToFirstSignificantSubdomain.cpp +++ b/src/Functions/URL/cutToFirstSignificantSubdomain.cpp @@ -35,7 +35,7 @@ using FunctionCutToFirstSignificantSubdomain = FunctionStringToString>, NameCutToFirstSignificantSubdomainWithWWW>; -void registerFunctionCutToFirstSignificantSubdomain(FunctionFactory & factory) +REGISTER_FUNCTION(CutToFirstSignificantSubdomain) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/URL/cutToFirstSignificantSubdomainCustom.cpp b/src/Functions/URL/cutToFirstSignificantSubdomainCustom.cpp index 7532ddd00f2..a2e51200910 100644 --- a/src/Functions/URL/cutToFirstSignificantSubdomainCustom.cpp +++ b/src/Functions/URL/cutToFirstSignificantSubdomainCustom.cpp @@ -34,7 +34,7 @@ using FunctionCutToFirstSignificantSubdomainCustom = FunctionCutToFirstSignifica struct NameCutToFirstSignificantSubdomainCustomWithWWW { static constexpr auto name = "cutToFirstSignificantSubdomainCustomWithWWW"; }; using FunctionCutToFirstSignificantSubdomainCustomWithWWW = FunctionCutToFirstSignificantSubdomainCustomImpl, NameCutToFirstSignificantSubdomainCustomWithWWW>; -void registerFunctionCutToFirstSignificantSubdomainCustom(FunctionFactory & factory) +REGISTER_FUNCTION(CutToFirstSignificantSubdomainCustom) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/URL/cutURLParameter.cpp b/src/Functions/URL/cutURLParameter.cpp index 2ff7c3e33d9..6077b068bd0 100644 --- a/src/Functions/URL/cutURLParameter.cpp +++ b/src/Functions/URL/cutURLParameter.cpp @@ -77,7 +77,7 @@ struct CutURLParameterImpl struct NameCutURLParameter { static constexpr auto name = "cutURLParameter"; }; using FunctionCutURLParameter = FunctionsStringSearchToString; -void registerFunctionCutURLParameter(FunctionFactory & factory) +REGISTER_FUNCTION(CutURLParameter) { factory.registerFunction(); } diff --git a/src/Functions/URL/cutWWW.cpp b/src/Functions/URL/cutWWW.cpp index 9b55b0929b0..992d5128440 100644 --- a/src/Functions/URL/cutWWW.cpp +++ b/src/Functions/URL/cutWWW.cpp @@ -54,7 +54,7 @@ struct ExtractWWW struct NameCutWWW { static constexpr auto name = "cutWWW"; }; using FunctionCutWWW = FunctionStringToString, NameCutWWW>; -void registerFunctionCutWWW(FunctionFactory & factory) +REGISTER_FUNCTION(CutWWW) { factory.registerFunction(); } diff --git a/src/Functions/URL/decodeURLComponent.cpp b/src/Functions/URL/decodeURLComponent.cpp index a4c9e1f4eec..eb44ca005a8 100644 --- a/src/Functions/URL/decodeURLComponent.cpp +++ b/src/Functions/URL/decodeURLComponent.cpp @@ -172,7 +172,7 @@ using FunctionEncodeURLComponent = FunctionStringToString, NameDecodeURLFormComponent>; using FunctionEncodeURLFormComponent = FunctionStringToString, NameEncodeURLFormComponent>; -void registerFunctionEncodeAndDecodeURLComponent(FunctionFactory & factory) +REGISTER_FUNCTION(EncodeAndDecodeURLComponent) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/URL/domain.cpp b/src/Functions/URL/domain.cpp index 339d3366a34..1d781b37943 100644 --- a/src/Functions/URL/domain.cpp +++ b/src/Functions/URL/domain.cpp @@ -10,7 +10,7 @@ struct NameDomain { static constexpr auto name = "domain"; }; using FunctionDomain = FunctionStringToString>, NameDomain>; -void registerFunctionDomain(FunctionFactory & factory) +REGISTER_FUNCTION(Domain) { factory.registerFunction(); } diff --git a/src/Functions/URL/domainWithoutWWW.cpp b/src/Functions/URL/domainWithoutWWW.cpp index 2fcf88009a1..53ff5bc919e 100644 --- a/src/Functions/URL/domainWithoutWWW.cpp +++ b/src/Functions/URL/domainWithoutWWW.cpp @@ -9,7 +9,7 @@ struct NameDomainWithoutWWW { static constexpr auto name = "domainWithoutWWW"; } using FunctionDomainWithoutWWW = FunctionStringToString>, NameDomainWithoutWWW>; -void registerFunctionDomainWithoutWWW(FunctionFactory & factory) +REGISTER_FUNCTION(DomainWithoutWWW) { factory.registerFunction(); } diff --git a/src/Functions/URL/extractURLParameter.cpp b/src/Functions/URL/extractURLParameter.cpp index 983426ef0fc..f75875e0200 100644 --- a/src/Functions/URL/extractURLParameter.cpp +++ b/src/Functions/URL/extractURLParameter.cpp @@ -92,7 +92,7 @@ struct ExtractURLParameterImpl struct NameExtractURLParameter { static constexpr auto name = "extractURLParameter"; }; using FunctionExtractURLParameter = FunctionsStringSearchToString; -void registerFunctionExtractURLParameter(FunctionFactory & factory) +REGISTER_FUNCTION(ExtractURLParameter) { factory.registerFunction(); } diff --git a/src/Functions/URL/extractURLParameterNames.cpp b/src/Functions/URL/extractURLParameterNames.cpp index 377e969a6b7..f274547ed4e 100644 --- a/src/Functions/URL/extractURLParameterNames.cpp +++ b/src/Functions/URL/extractURLParameterNames.cpp @@ -89,7 +89,7 @@ public: struct NameExtractURLParameterNames { static constexpr auto name = "extractURLParameterNames"; }; using FunctionExtractURLParameterNames = FunctionTokens; -void registerFunctionExtractURLParameterNames(FunctionFactory & factory) +REGISTER_FUNCTION(ExtractURLParameterNames) { factory.registerFunction(); } diff --git a/src/Functions/URL/extractURLParameters.cpp b/src/Functions/URL/extractURLParameters.cpp index fb595c23170..15b564fc947 100644 --- a/src/Functions/URL/extractURLParameters.cpp +++ b/src/Functions/URL/extractURLParameters.cpp @@ -97,7 +97,7 @@ public: struct NameExtractURLParameters { static constexpr auto name = "extractURLParameters"; }; using FunctionExtractURLParameters = FunctionTokens; -void registerFunctionExtractURLParameters(FunctionFactory & factory) +REGISTER_FUNCTION(ExtractURLParameters) { factory.registerFunction(); } diff --git a/src/Functions/URL/firstSignificantSubdomain.cpp b/src/Functions/URL/firstSignificantSubdomain.cpp index 87659940938..d3aeb90771f 100644 --- a/src/Functions/URL/firstSignificantSubdomain.cpp +++ b/src/Functions/URL/firstSignificantSubdomain.cpp @@ -10,7 +10,7 @@ struct NameFirstSignificantSubdomain { static constexpr auto name = "firstSignif using FunctionFirstSignificantSubdomain = FunctionStringToString>, NameFirstSignificantSubdomain>; -void registerFunctionFirstSignificantSubdomain(FunctionFactory & factory) +REGISTER_FUNCTION(FirstSignificantSubdomain) { factory.registerFunction(); } diff --git a/src/Functions/URL/firstSignificantSubdomainCustom.cpp b/src/Functions/URL/firstSignificantSubdomainCustom.cpp index 675b4a346de..f43b42d0309 100644 --- a/src/Functions/URL/firstSignificantSubdomainCustom.cpp +++ b/src/Functions/URL/firstSignificantSubdomainCustom.cpp @@ -10,7 +10,7 @@ struct NameFirstSignificantSubdomainCustom { static constexpr auto name = "first using FunctionFirstSignificantSubdomainCustom = FunctionCutToFirstSignificantSubdomainCustomImpl, NameFirstSignificantSubdomainCustom>; -void registerFunctionFirstSignificantSubdomainCustom(FunctionFactory & factory) +REGISTER_FUNCTION(FirstSignificantSubdomainCustom) { factory.registerFunction(); } diff --git a/src/Functions/URL/fragment.cpp b/src/Functions/URL/fragment.cpp index 181ee6ce72c..262c1a4c7a6 100644 --- a/src/Functions/URL/fragment.cpp +++ b/src/Functions/URL/fragment.cpp @@ -8,7 +8,7 @@ namespace DB struct NameFragment { static constexpr auto name = "fragment"; }; using FunctionFragment = FunctionStringToString>, NameFragment>; -void registerFunctionFragment(FunctionFactory & factory) +REGISTER_FUNCTION(Fragment) { factory.registerFunction(); } diff --git a/src/Functions/URL/netloc.cpp b/src/Functions/URL/netloc.cpp index 723eea138c3..5c23f7cc06d 100644 --- a/src/Functions/URL/netloc.cpp +++ b/src/Functions/URL/netloc.cpp @@ -132,7 +132,7 @@ struct ExtractNetloc struct NameNetloc { static constexpr auto name = "netloc"; }; using FunctionNetloc = FunctionStringToString, NameNetloc>; -void registerFunctionNetloc(FunctionFactory & factory) +REGISTER_FUNCTION(Netloc) { factory.registerFunction(); } diff --git a/src/Functions/URL/path.cpp b/src/Functions/URL/path.cpp index c3037956118..ccc7dedb724 100644 --- a/src/Functions/URL/path.cpp +++ b/src/Functions/URL/path.cpp @@ -11,7 +11,7 @@ namespace DB struct NamePath { static constexpr auto name = "path"; }; using FunctionPath = FunctionStringToString>, NamePath>; -void registerFunctionPath(FunctionFactory & factory) +REGISTER_FUNCTION(Path) { factory.registerFunction(); } diff --git a/src/Functions/URL/pathFull.cpp b/src/Functions/URL/pathFull.cpp index f37c42e3753..002770e812a 100644 --- a/src/Functions/URL/pathFull.cpp +++ b/src/Functions/URL/pathFull.cpp @@ -10,7 +10,7 @@ namespace DB struct NamePathFull { static constexpr auto name = "pathFull"; }; using FunctionPathFull = FunctionStringToString>, NamePathFull>; -void registerFunctionPathFull(FunctionFactory & factory) +REGISTER_FUNCTION(PathFull) { factory.registerFunction(); } diff --git a/src/Functions/URL/port.cpp b/src/Functions/URL/port.cpp index 3bbcdcea560..85b060ca987 100644 --- a/src/Functions/URL/port.cpp +++ b/src/Functions/URL/port.cpp @@ -121,7 +121,7 @@ private: } }; -void registerFunctionPort(FunctionFactory & factory) +REGISTER_FUNCTION(Port) { factory.registerFunction(); } diff --git a/src/Functions/URL/protocol.cpp b/src/Functions/URL/protocol.cpp index 2948b044d70..6bed71207f6 100644 --- a/src/Functions/URL/protocol.cpp +++ b/src/Functions/URL/protocol.cpp @@ -9,7 +9,7 @@ namespace DB struct NameProtocol { static constexpr auto name = "protocol"; }; using FunctionProtocol = FunctionStringToString, NameProtocol>; -void registerFunctionProtocol(FunctionFactory & factory) +REGISTER_FUNCTION(Protocol) { factory.registerFunction(); } diff --git a/src/Functions/URL/queryString.cpp b/src/Functions/URL/queryString.cpp index 7edf212415d..7069ce46b38 100644 --- a/src/Functions/URL/queryString.cpp +++ b/src/Functions/URL/queryString.cpp @@ -8,7 +8,7 @@ namespace DB struct NameQueryString { static constexpr auto name = "queryString"; }; using FunctionQueryString = FunctionStringToString>, NameQueryString>; -void registerFunctionQueryString(FunctionFactory & factory) +REGISTER_FUNCTION(QueryString) { factory.registerFunction(); } diff --git a/src/Functions/URL/queryStringAndFragment.cpp b/src/Functions/URL/queryStringAndFragment.cpp index 18f14f9a2b9..367a95acbdc 100644 --- a/src/Functions/URL/queryStringAndFragment.cpp +++ b/src/Functions/URL/queryStringAndFragment.cpp @@ -8,7 +8,7 @@ namespace DB struct NameQueryStringAndFragment { static constexpr auto name = "queryStringAndFragment"; }; using FunctionQueryStringAndFragment = FunctionStringToString>, NameQueryStringAndFragment>; -void registerFunctionQueryStringAndFragment(FunctionFactory & factory) +REGISTER_FUNCTION(QueryStringAndFragment) { factory.registerFunction(); } diff --git a/src/Functions/URL/registerFunctionsURL.cpp b/src/Functions/URL/registerFunctionsURL.cpp deleted file mode 100644 index 91142a593f2..00000000000 --- a/src/Functions/URL/registerFunctionsURL.cpp +++ /dev/null @@ -1,64 +0,0 @@ -namespace DB -{ - -class FunctionFactory; - -void registerFunctionProtocol(FunctionFactory & factory); -void registerFunctionDomain(FunctionFactory & factory); -void registerFunctionDomainWithoutWWW(FunctionFactory & factory); -void registerFunctionFirstSignificantSubdomain(FunctionFactory & factory); -void registerFunctionFirstSignificantSubdomainCustom(FunctionFactory & factory); -void registerFunctionTopLevelDomain(FunctionFactory & factory); -void registerFunctionPort(FunctionFactory & factory); -void registerFunctionPath(FunctionFactory & factory); -void registerFunctionPathFull(FunctionFactory & factory); -void registerFunctionQueryString(FunctionFactory & factory); -void registerFunctionFragment(FunctionFactory & factory); -void registerFunctionQueryStringAndFragment(FunctionFactory & factory); -void registerFunctionExtractURLParameter(FunctionFactory & factory); -void registerFunctionExtractURLParameters(FunctionFactory & factory); -void registerFunctionExtractURLParameterNames(FunctionFactory & factory); -void registerFunctionURLHierarchy(FunctionFactory & factory); -void registerFunctionURLPathHierarchy(FunctionFactory & factory); -void registerFunctionCutToFirstSignificantSubdomain(FunctionFactory & factory); -void registerFunctionCutToFirstSignificantSubdomainCustom(FunctionFactory & factory); -void registerFunctionCutWWW(FunctionFactory & factory); -void registerFunctionCutQueryString(FunctionFactory & factory); -void registerFunctionCutFragment(FunctionFactory & factory); -void registerFunctionCutQueryStringAndFragment(FunctionFactory & factory); -void registerFunctionCutURLParameter(FunctionFactory & factory); -void registerFunctionEncodeAndDecodeURLComponent(FunctionFactory & factory); -void registerFunctionNetloc(FunctionFactory & factory); - -void registerFunctionsURL(FunctionFactory & factory) -{ - registerFunctionProtocol(factory); - registerFunctionDomain(factory); - registerFunctionDomainWithoutWWW(factory); - registerFunctionFirstSignificantSubdomain(factory); - registerFunctionFirstSignificantSubdomainCustom(factory); - registerFunctionTopLevelDomain(factory); - registerFunctionPort(factory); - registerFunctionPath(factory); - registerFunctionPathFull(factory); - registerFunctionQueryString(factory); - registerFunctionFragment(factory); - registerFunctionQueryStringAndFragment(factory); - registerFunctionExtractURLParameter(factory); - registerFunctionExtractURLParameters(factory); - registerFunctionExtractURLParameterNames(factory); - registerFunctionURLHierarchy(factory); - registerFunctionURLPathHierarchy(factory); - registerFunctionCutToFirstSignificantSubdomain(factory); - registerFunctionCutToFirstSignificantSubdomainCustom(factory); - registerFunctionCutWWW(factory); - registerFunctionCutQueryString(factory); - registerFunctionCutFragment(factory); - registerFunctionCutQueryStringAndFragment(factory); - registerFunctionCutURLParameter(factory); - registerFunctionEncodeAndDecodeURLComponent(factory); - registerFunctionNetloc(factory); -} - -} - diff --git a/src/Functions/URL/topLevelDomain.cpp b/src/Functions/URL/topLevelDomain.cpp index ade9439d8ec..23ba5201c18 100644 --- a/src/Functions/URL/topLevelDomain.cpp +++ b/src/Functions/URL/topLevelDomain.cpp @@ -43,7 +43,7 @@ struct ExtractTopLevelDomain struct NameTopLevelDomain { static constexpr auto name = "topLevelDomain"; }; using FunctionTopLevelDomain = FunctionStringToString, NameTopLevelDomain>; -void registerFunctionTopLevelDomain(FunctionFactory & factory) +REGISTER_FUNCTION(TopLevelDomain) { factory.registerFunction(); } diff --git a/src/Functions/abs.cpp b/src/Functions/abs.cpp index f8c68c0ded2..67aa5ec6e90 100644 --- a/src/Functions/abs.cpp +++ b/src/Functions/abs.cpp @@ -50,7 +50,7 @@ template <> struct FunctionUnaryArithmeticMonotonicity } }; -void registerFunctionAbs(FunctionFactory & factory) +REGISTER_FUNCTION(Abs) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/acos.cpp b/src/Functions/acos.cpp index 62e68b5c17b..1fbd636f14e 100644 --- a/src/Functions/acos.cpp +++ b/src/Functions/acos.cpp @@ -12,7 +12,7 @@ using FunctionAcos = FunctionMathUnary>; } -void registerFunctionAcos(FunctionFactory & factory) +REGISTER_FUNCTION(Acos) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/acosh.cpp b/src/Functions/acosh.cpp index 20473faafc7..5b071da9c40 100644 --- a/src/Functions/acosh.cpp +++ b/src/Functions/acosh.cpp @@ -13,7 +13,7 @@ namespace } -void registerFunctionAcosh(FunctionFactory & factory) +REGISTER_FUNCTION(Acosh) { factory.registerFunction(); } diff --git a/src/Functions/addDays.cpp b/src/Functions/addDays.cpp index c86d7cfb2c1..75559963379 100644 --- a/src/Functions/addDays.cpp +++ b/src/Functions/addDays.cpp @@ -7,7 +7,7 @@ namespace DB using FunctionAddDays = FunctionDateOrDateTimeAddInterval; -void registerFunctionAddDays(FunctionFactory & factory) +REGISTER_FUNCTION(AddDays) { factory.registerFunction(); } diff --git a/src/Functions/addHours.cpp b/src/Functions/addHours.cpp index cc526e6c8dc..d72a8624431 100644 --- a/src/Functions/addHours.cpp +++ b/src/Functions/addHours.cpp @@ -7,7 +7,7 @@ namespace DB using FunctionAddHours = FunctionDateOrDateTimeAddInterval; -void registerFunctionAddHours(FunctionFactory & factory) +REGISTER_FUNCTION(AddHours) { factory.registerFunction(); } diff --git a/src/Functions/addMinutes.cpp b/src/Functions/addMinutes.cpp index b7920490a83..afd10219b1e 100644 --- a/src/Functions/addMinutes.cpp +++ b/src/Functions/addMinutes.cpp @@ -7,7 +7,7 @@ namespace DB using FunctionAddMinutes = FunctionDateOrDateTimeAddInterval; -void registerFunctionAddMinutes(FunctionFactory & factory) +REGISTER_FUNCTION(AddMinutes) { factory.registerFunction(); } diff --git a/src/Functions/addMonths.cpp b/src/Functions/addMonths.cpp index c74a8ecb08b..c604d64bec6 100644 --- a/src/Functions/addMonths.cpp +++ b/src/Functions/addMonths.cpp @@ -7,7 +7,7 @@ namespace DB using FunctionAddMonths = FunctionDateOrDateTimeAddInterval; -void registerFunctionAddMonths(FunctionFactory & factory) +REGISTER_FUNCTION(AddMonths) { factory.registerFunction(); } diff --git a/src/Functions/addQuarters.cpp b/src/Functions/addQuarters.cpp index 9b5e3556ff3..36a9a56987b 100644 --- a/src/Functions/addQuarters.cpp +++ b/src/Functions/addQuarters.cpp @@ -7,7 +7,7 @@ namespace DB using FunctionAddQuarters = FunctionDateOrDateTimeAddInterval; -void registerFunctionAddQuarters(FunctionFactory & factory) +REGISTER_FUNCTION(AddQuarters) { factory.registerFunction(); } diff --git a/src/Functions/addSeconds.cpp b/src/Functions/addSeconds.cpp index 8ba229e238c..9a869236e97 100644 --- a/src/Functions/addSeconds.cpp +++ b/src/Functions/addSeconds.cpp @@ -7,7 +7,7 @@ namespace DB using FunctionAddSeconds = FunctionDateOrDateTimeAddInterval; -void registerFunctionAddSeconds(FunctionFactory & factory) +REGISTER_FUNCTION(AddSeconds) { factory.registerFunction(); } diff --git a/src/Functions/addSubSeconds.cpp b/src/Functions/addSubSeconds.cpp index cb5ffce61e3..fa901ad4dcd 100644 --- a/src/Functions/addSubSeconds.cpp +++ b/src/Functions/addSubSeconds.cpp @@ -6,19 +6,19 @@ namespace DB { using FunctionAddNanoseconds = FunctionDateOrDateTimeAddInterval; -void registerFunctionAddNanoseconds(FunctionFactory & factory) +REGISTER_FUNCTION(AddNanoseconds) { factory.registerFunction(); } using FunctionAddMicroseconds = FunctionDateOrDateTimeAddInterval; -void registerFunctionAddMicroseconds(FunctionFactory & factory) +REGISTER_FUNCTION(AddMicroseconds) { factory.registerFunction(); } using FunctionAddMilliseconds = FunctionDateOrDateTimeAddInterval; -void registerFunctionAddMilliseconds(FunctionFactory & factory) +REGISTER_FUNCTION(AddMilliseconds) { factory.registerFunction(); } diff --git a/src/Functions/addWeeks.cpp b/src/Functions/addWeeks.cpp index 2b83f61f45c..366d85c4def 100644 --- a/src/Functions/addWeeks.cpp +++ b/src/Functions/addWeeks.cpp @@ -7,7 +7,7 @@ namespace DB using FunctionAddWeeks = FunctionDateOrDateTimeAddInterval; -void registerFunctionAddWeeks(FunctionFactory & factory) +REGISTER_FUNCTION(AddWeeks) { factory.registerFunction(); } diff --git a/src/Functions/addYears.cpp b/src/Functions/addYears.cpp index 24364154c23..ca2b55e9da2 100644 --- a/src/Functions/addYears.cpp +++ b/src/Functions/addYears.cpp @@ -7,7 +7,7 @@ namespace DB using FunctionAddYears = FunctionDateOrDateTimeAddInterval; -void registerFunctionAddYears(FunctionFactory & factory) +REGISTER_FUNCTION(AddYears) { factory.registerFunction(); } diff --git a/src/Functions/addressToLine.cpp b/src/Functions/addressToLine.cpp index 2e731539636..42f9be23c38 100644 --- a/src/Functions/addressToLine.cpp +++ b/src/Functions/addressToLine.cpp @@ -58,7 +58,7 @@ protected: } -void registerFunctionAddressToLine(FunctionFactory & factory) +REGISTER_FUNCTION(AddressToLine) { factory.registerFunction(); } diff --git a/src/Functions/addressToLineWithInlines.cpp b/src/Functions/addressToLineWithInlines.cpp index ea2adcd140c..9a7b0858b87 100644 --- a/src/Functions/addressToLineWithInlines.cpp +++ b/src/Functions/addressToLineWithInlines.cpp @@ -89,7 +89,7 @@ private: } -void registerFunctionAddressToLineWithInlines(FunctionFactory & factory) +REGISTER_FUNCTION(AddressToLineWithInlines) { factory.registerFunction(); } diff --git a/src/Functions/addressToSymbol.cpp b/src/Functions/addressToSymbol.cpp index fd26ebce0f1..99988ee82f6 100644 --- a/src/Functions/addressToSymbol.cpp +++ b/src/Functions/addressToSymbol.cpp @@ -94,7 +94,7 @@ public: } -void registerFunctionAddressToSymbol(FunctionFactory & factory) +REGISTER_FUNCTION(AddressToSymbol) { factory.registerFunction(); } diff --git a/src/Functions/aes_decrypt_mysql.cpp b/src/Functions/aes_decrypt_mysql.cpp index 764fcf06c1a..8ba7eaa8e80 100644 --- a/src/Functions/aes_decrypt_mysql.cpp +++ b/src/Functions/aes_decrypt_mysql.cpp @@ -19,7 +19,7 @@ struct DecryptMySQLModeImpl namespace DB { -void registerFunctionAESDecryptMysql(FunctionFactory & factory) +REGISTER_FUNCTION(AESDecryptMysql) { factory.registerFunction>(); } diff --git a/src/Functions/aes_encrypt_mysql.cpp b/src/Functions/aes_encrypt_mysql.cpp index 1d84824d9d6..24d618b5a86 100644 --- a/src/Functions/aes_encrypt_mysql.cpp +++ b/src/Functions/aes_encrypt_mysql.cpp @@ -19,7 +19,7 @@ struct EncryptMySQLModeImpl namespace DB { -void registerFunctionAESEncryptMysql(FunctionFactory & factory) +REGISTER_FUNCTION(AESEncryptMysql) { factory.registerFunction>(); } diff --git a/src/Functions/appendTrailingCharIfAbsent.cpp b/src/Functions/appendTrailingCharIfAbsent.cpp index 1661ec8fb0e..9fe141458fb 100644 --- a/src/Functions/appendTrailingCharIfAbsent.cpp +++ b/src/Functions/appendTrailingCharIfAbsent.cpp @@ -115,7 +115,7 @@ private: } -void registerFunctionAppendTrailingCharIfAbsent(FunctionFactory & factory) +REGISTER_FUNCTION(AppendTrailingCharIfAbsent) { factory.registerFunction(); } diff --git a/src/Functions/array/CMakeLists.txt b/src/Functions/array/CMakeLists.txt index 6b4a8122d16..98335dbb8e7 100644 --- a/src/Functions/array/CMakeLists.txt +++ b/src/Functions/array/CMakeLists.txt @@ -1,6 +1,6 @@ include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") add_headers_and_sources(clickhouse_functions_array .) -add_library(clickhouse_functions_array ${clickhouse_functions_array_sources} ${clickhouse_functions_array_headers}) +add_library(clickhouse_functions_array OBJECT ${clickhouse_functions_array_sources} ${clickhouse_functions_array_headers}) target_link_libraries(clickhouse_functions_array PRIVATE dbms clickhouse_functions_gatherutils) if (OMIT_HEAVY_DEBUG_SYMBOLS) diff --git a/src/Functions/array/array.cpp b/src/Functions/array/array.cpp index f220117407c..551f0a6625b 100644 --- a/src/Functions/array/array.cpp +++ b/src/Functions/array/array.cpp @@ -100,7 +100,7 @@ private: }; -void registerFunctionArray(FunctionFactory & factory) +REGISTER_FUNCTION(Array) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayAUC.cpp b/src/Functions/array/arrayAUC.cpp index 1af0ca331a7..78c6fae59ca 100644 --- a/src/Functions/array/arrayAUC.cpp +++ b/src/Functions/array/arrayAUC.cpp @@ -140,7 +140,7 @@ public: /// auc(array_score, array_label) - Calculate AUC with array of score and label using FunctionArrayAUC = FunctionArrayScalarProduct; -void registerFunctionArrayAUC(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayAUC) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayAggregation.cpp b/src/Functions/array/arrayAggregation.cpp index 9c18c6213de..7b72060f0c0 100644 --- a/src/Functions/array/arrayAggregation.cpp +++ b/src/Functions/array/arrayAggregation.cpp @@ -387,7 +387,7 @@ using FunctionArrayAverage = FunctionArrayMapped, NameArrayProduct>; -void registerFunctionArrayAggregation(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayAggregation) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/array/arrayAll.cpp b/src/Functions/array/arrayAll.cpp index 0f7ae797dc9..960f1f59fc0 100644 --- a/src/Functions/array/arrayAll.cpp +++ b/src/Functions/array/arrayAll.cpp @@ -86,7 +86,7 @@ struct ArrayAllImpl struct NameArrayAll { static constexpr auto name = "arrayAll"; }; using FunctionArrayAll = FunctionArrayMapped; -void registerFunctionArrayAll(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayAll) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayCompact.cpp b/src/Functions/array/arrayCompact.cpp index 8abce7288d2..5c899d4d97e 100644 --- a/src/Functions/array/arrayCompact.cpp +++ b/src/Functions/array/arrayCompact.cpp @@ -162,7 +162,7 @@ struct ArrayCompactImpl struct NameArrayCompact { static constexpr auto name = "arrayCompact"; }; using FunctionArrayCompact = FunctionArrayMapped; -void registerFunctionArrayCompact(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayCompact) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayConcat.cpp b/src/Functions/array/arrayConcat.cpp index 0a8f289ab3f..1878cc27323 100644 --- a/src/Functions/array/arrayConcat.cpp +++ b/src/Functions/array/arrayConcat.cpp @@ -98,7 +98,7 @@ public: }; -void registerFunctionArrayConcat(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayConcat) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayCount.cpp b/src/Functions/array/arrayCount.cpp index df45783323b..cb902206e8b 100644 --- a/src/Functions/array/arrayCount.cpp +++ b/src/Functions/array/arrayCount.cpp @@ -83,7 +83,7 @@ struct ArrayCountImpl struct NameArrayCount { static constexpr auto name = "arrayCount"; }; using FunctionArrayCount = FunctionArrayMapped; -void registerFunctionArrayCount(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayCount) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayCumSum.cpp b/src/Functions/array/arrayCumSum.cpp index 98ffa09820b..3bb733482b6 100644 --- a/src/Functions/array/arrayCumSum.cpp +++ b/src/Functions/array/arrayCumSum.cpp @@ -162,7 +162,7 @@ struct ArrayCumSumImpl struct NameArrayCumSum { static constexpr auto name = "arrayCumSum"; }; using FunctionArrayCumSum = FunctionArrayMapped; -void registerFunctionArrayCumSum(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayCumSum) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayCumSumNonNegative.cpp b/src/Functions/array/arrayCumSumNonNegative.cpp index cd8393b7a5f..1e27b773141 100644 --- a/src/Functions/array/arrayCumSumNonNegative.cpp +++ b/src/Functions/array/arrayCumSumNonNegative.cpp @@ -126,7 +126,7 @@ struct ArrayCumSumNonNegativeImpl struct NameArrayCumSumNonNegative { static constexpr auto name = "arrayCumSumNonNegative"; }; using FunctionArrayCumSumNonNegative = FunctionArrayMapped; -void registerFunctionArrayCumSumNonNegative(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayCumSumNonNegative) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayDifference.cpp b/src/Functions/array/arrayDifference.cpp index 8af0e8b04f9..a19f04f4e02 100644 --- a/src/Functions/array/arrayDifference.cpp +++ b/src/Functions/array/arrayDifference.cpp @@ -155,7 +155,7 @@ struct ArrayDifferenceImpl struct NameArrayDifference { static constexpr auto name = "arrayDifference"; }; using FunctionArrayDifference = FunctionArrayMapped; -void registerFunctionArrayDifference(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayDifference) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayDistinct.cpp b/src/Functions/array/arrayDistinct.cpp index e71a582bbe0..46b0efd2634 100644 --- a/src/Functions/array/arrayDistinct.cpp +++ b/src/Functions/array/arrayDistinct.cpp @@ -289,7 +289,7 @@ void FunctionArrayDistinct::executeHashed( } -void registerFunctionArrayDistinct(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayDistinct) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayElement.cpp b/src/Functions/array/arrayElement.cpp index d8e44dc297d..d3255d6412e 100644 --- a/src/Functions/array/arrayElement.cpp +++ b/src/Functions/array/arrayElement.cpp @@ -1203,7 +1203,7 @@ ColumnPtr FunctionArrayElement::perform(const ColumnsWithTypeAndName & arguments } -void registerFunctionArrayElement(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayElement) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayEnumerate.cpp b/src/Functions/array/arrayEnumerate.cpp index f7b6a7e9547..b20f91fe2dd 100644 --- a/src/Functions/array/arrayEnumerate.cpp +++ b/src/Functions/array/arrayEnumerate.cpp @@ -76,7 +76,7 @@ public: }; -void registerFunctionArrayEnumerate(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayEnumerate) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayEnumerateDense.cpp b/src/Functions/array/arrayEnumerateDense.cpp index 4539aed18ab..6f82bbed448 100644 --- a/src/Functions/array/arrayEnumerateDense.cpp +++ b/src/Functions/array/arrayEnumerateDense.cpp @@ -14,7 +14,7 @@ public: using Base::create; }; -void registerFunctionArrayEnumerateDense(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayEnumerateDense) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayEnumerateDenseRanked.cpp b/src/Functions/array/arrayEnumerateDenseRanked.cpp index 735211fb3df..8799612de00 100644 --- a/src/Functions/array/arrayEnumerateDenseRanked.cpp +++ b/src/Functions/array/arrayEnumerateDenseRanked.cpp @@ -14,7 +14,7 @@ public: using Base::create; }; -void registerFunctionArrayEnumerateDenseRanked(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayEnumerateDenseRanked) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayEnumerateUniq.cpp b/src/Functions/array/arrayEnumerateUniq.cpp index 848b29064c4..e25a07450f0 100644 --- a/src/Functions/array/arrayEnumerateUniq.cpp +++ b/src/Functions/array/arrayEnumerateUniq.cpp @@ -20,7 +20,7 @@ public: using Base::create; }; -void registerFunctionArrayEnumerateUniq(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayEnumerateUniq) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayEnumerateUniqRanked.cpp b/src/Functions/array/arrayEnumerateUniqRanked.cpp index 2cd1fe40c2e..42e3d989f62 100644 --- a/src/Functions/array/arrayEnumerateUniqRanked.cpp +++ b/src/Functions/array/arrayEnumerateUniqRanked.cpp @@ -14,7 +14,7 @@ public: using Base::create; }; -void registerFunctionArrayEnumerateUniqRanked(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayEnumerateUniqRanked) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayExists.cpp b/src/Functions/array/arrayExists.cpp index ea39cc0dc0b..c1731b05229 100644 --- a/src/Functions/array/arrayExists.cpp +++ b/src/Functions/array/arrayExists.cpp @@ -87,7 +87,7 @@ struct ArrayExistsImpl struct NameArrayExists { static constexpr auto name = "arrayExists"; }; using FunctionArrayExists = FunctionArrayMapped; -void registerFunctionArrayExists(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayExists) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayFill.cpp b/src/Functions/array/arrayFill.cpp index 22b9e9a657b..bd424c3f474 100644 --- a/src/Functions/array/arrayFill.cpp +++ b/src/Functions/array/arrayFill.cpp @@ -129,7 +129,7 @@ struct NameArrayReverseFill { static constexpr auto name = "arrayReverseFill"; } using FunctionArrayFill = FunctionArrayMapped, NameArrayFill>; using FunctionArrayReverseFill = FunctionArrayMapped, NameArrayReverseFill>; -void registerFunctionsArrayFill(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayFill) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/array/arrayFilter.cpp b/src/Functions/array/arrayFilter.cpp index 89a9de44532..b66c8570f11 100644 --- a/src/Functions/array/arrayFilter.cpp +++ b/src/Functions/array/arrayFilter.cpp @@ -74,7 +74,7 @@ struct ArrayFilterImpl struct NameArrayFilter { static constexpr auto name = "arrayFilter"; }; using FunctionArrayFilter = FunctionArrayMapped; -void registerFunctionArrayFilter(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayFilter) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayFirstLast.cpp b/src/Functions/array/arrayFirstLast.cpp index c9a8d2ba497..8160234a6b0 100644 --- a/src/Functions/array/arrayFirstLast.cpp +++ b/src/Functions/array/arrayFirstLast.cpp @@ -194,7 +194,7 @@ struct NameArrayLastOrNull { static constexpr auto name = "arrayLastOrNull"; }; using ArrayLastOrNullImpl = ArrayFirstLastImpl; using FunctionArrayLastOrNull = FunctionArrayMapped; -void registerFunctionArrayFirst(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayFirst) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/array/arrayFirstLastIndex.cpp b/src/Functions/array/arrayFirstLastIndex.cpp index 9392cbdc840..f7355eb2b38 100644 --- a/src/Functions/array/arrayFirstLastIndex.cpp +++ b/src/Functions/array/arrayFirstLastIndex.cpp @@ -128,7 +128,7 @@ struct NameArrayLastIndex { static constexpr auto name = "arrayLastIndex"; }; using ArrayLastIndexImpl = ArrayFirstLastIndexImpl; using FunctionArrayLastIndex = FunctionArrayMapped; -void registerFunctionArrayFirstIndex(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayFirstIndex) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/array/arrayFlatten.cpp b/src/Functions/array/arrayFlatten.cpp index 929ad041b9a..b1f9399776e 100644 --- a/src/Functions/array/arrayFlatten.cpp +++ b/src/Functions/array/arrayFlatten.cpp @@ -121,7 +121,7 @@ private: }; -void registerFunctionArrayFlatten(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayFlatten) { factory.registerFunction(); factory.registerAlias("flatten", "arrayFlatten", FunctionFactory::CaseInsensitive); diff --git a/src/Functions/array/arrayIntersect.cpp b/src/Functions/array/arrayIntersect.cpp index f1b849b64f0..4ed946d8bb3 100644 --- a/src/Functions/array/arrayIntersect.cpp +++ b/src/Functions/array/arrayIntersect.cpp @@ -585,7 +585,7 @@ ColumnPtr FunctionArrayIntersect::execute(const UnpackedArrays & arrays, Mutable } -void registerFunctionArrayIntersect(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayIntersect) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayJoin.cpp b/src/Functions/array/arrayJoin.cpp index 6d9950d9110..3230886c731 100644 --- a/src/Functions/array/arrayJoin.cpp +++ b/src/Functions/array/arrayJoin.cpp @@ -69,7 +69,7 @@ public: }; -void registerFunctionArrayJoin(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayJoin) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayMap.cpp b/src/Functions/array/arrayMap.cpp index ec1973d573b..216d488a360 100644 --- a/src/Functions/array/arrayMap.cpp +++ b/src/Functions/array/arrayMap.cpp @@ -34,7 +34,7 @@ struct ArrayMapImpl struct NameArrayMap { static constexpr auto name = "arrayMap"; }; using FunctionArrayMap = FunctionArrayMapped; -void registerFunctionArrayMap(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayMap) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayPopBack.cpp b/src/Functions/array/arrayPopBack.cpp index 0b532c22661..b2e80302f9a 100644 --- a/src/Functions/array/arrayPopBack.cpp +++ b/src/Functions/array/arrayPopBack.cpp @@ -13,7 +13,7 @@ public: FunctionArrayPopBack() : FunctionArrayPop(false, name) {} }; -void registerFunctionArrayPopBack(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayPopBack) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayPopFront.cpp b/src/Functions/array/arrayPopFront.cpp index 57eedf9b44a..4e7798c3b8a 100644 --- a/src/Functions/array/arrayPopFront.cpp +++ b/src/Functions/array/arrayPopFront.cpp @@ -13,7 +13,7 @@ public: FunctionArrayPopFront() : FunctionArrayPop(true, name) {} }; -void registerFunctionArrayPopFront(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayPopFront) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayPushBack.cpp b/src/Functions/array/arrayPushBack.cpp index 3171bca9d2c..e9651ed187b 100644 --- a/src/Functions/array/arrayPushBack.cpp +++ b/src/Functions/array/arrayPushBack.cpp @@ -13,7 +13,7 @@ public: FunctionArrayPushBack() : FunctionArrayPush(false, name) {} }; -void registerFunctionArrayPushBack(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayPushBack) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayPushFront.cpp b/src/Functions/array/arrayPushFront.cpp index d12dfe02af1..993883d93b6 100644 --- a/src/Functions/array/arrayPushFront.cpp +++ b/src/Functions/array/arrayPushFront.cpp @@ -15,7 +15,7 @@ public: }; -void registerFunctionArrayPushFront(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayPushFront) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayReduce.cpp b/src/Functions/array/arrayReduce.cpp index b706cfa42b3..fd16f1fc986 100644 --- a/src/Functions/array/arrayReduce.cpp +++ b/src/Functions/array/arrayReduce.cpp @@ -198,7 +198,7 @@ ColumnPtr FunctionArrayReduce::executeImpl(const ColumnsWithTypeAndName & argume } -void registerFunctionArrayReduce(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayReduce) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayReduceInRanges.cpp b/src/Functions/array/arrayReduceInRanges.cpp index b92bfb04fe5..d2a382e86ba 100644 --- a/src/Functions/array/arrayReduceInRanges.cpp +++ b/src/Functions/array/arrayReduceInRanges.cpp @@ -391,7 +391,7 @@ ColumnPtr FunctionArrayReduceInRanges::executeImpl( } -void registerFunctionArrayReduceInRanges(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayReduceInRanges) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayResize.cpp b/src/Functions/array/arrayResize.cpp index 1e6dcfbf069..9aa06918659 100644 --- a/src/Functions/array/arrayResize.cpp +++ b/src/Functions/array/arrayResize.cpp @@ -132,7 +132,7 @@ public: }; -void registerFunctionArrayResize(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayResize) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayReverse.cpp b/src/Functions/array/arrayReverse.cpp index a517080ff2c..88703cbb032 100644 --- a/src/Functions/array/arrayReverse.cpp +++ b/src/Functions/array/arrayReverse.cpp @@ -247,7 +247,7 @@ bool FunctionArrayReverse::executeString(const IColumn & src_data, const ColumnA } -void registerFunctionArrayReverse(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayReverse) { factory.registerFunction(); } diff --git a/src/Functions/array/arraySlice.cpp b/src/Functions/array/arraySlice.cpp index 7a2e97de78a..9d7efd8fccf 100644 --- a/src/Functions/array/arraySlice.cpp +++ b/src/Functions/array/arraySlice.cpp @@ -150,7 +150,7 @@ public: }; -void registerFunctionArraySlice(FunctionFactory & factory) +REGISTER_FUNCTION(ArraySlice) { factory.registerFunction(); } diff --git a/src/Functions/array/arraySort.cpp b/src/Functions/array/arraySort.cpp index 5421185211e..34c2881a413 100644 --- a/src/Functions/array/arraySort.cpp +++ b/src/Functions/array/arraySort.cpp @@ -67,7 +67,7 @@ struct NameArrayReverseSort { static constexpr auto name = "arrayReverseSort"; } using FunctionArraySort = FunctionArrayMapped, NameArraySort>; using FunctionArrayReverseSort = FunctionArrayMapped, NameArrayReverseSort>; -void registerFunctionsArraySort(FunctionFactory & factory) +REGISTER_FUNCTION(ArraySort) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/array/arraySplit.cpp b/src/Functions/array/arraySplit.cpp index c818be97f60..caa6438adb1 100644 --- a/src/Functions/array/arraySplit.cpp +++ b/src/Functions/array/arraySplit.cpp @@ -116,7 +116,7 @@ struct NameArrayReverseSplit { static constexpr auto name = "arrayReverseSplit"; using FunctionArraySplit = FunctionArrayMapped, NameArraySplit>; using FunctionArrayReverseSplit = FunctionArrayMapped, NameArrayReverseSplit>; -void registerFunctionsArraySplit(FunctionFactory & factory) +REGISTER_FUNCTION(ArraySplit) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/array/arrayUniq.cpp b/src/Functions/array/arrayUniq.cpp index 8d3393c43c4..ff75efaae71 100644 --- a/src/Functions/array/arrayUniq.cpp +++ b/src/Functions/array/arrayUniq.cpp @@ -315,7 +315,7 @@ void FunctionArrayUniq::executeHashed( } -void registerFunctionArrayUniq(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayUniq) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayWithConstant.cpp b/src/Functions/array/arrayWithConstant.cpp index a178147837c..2feac209b22 100644 --- a/src/Functions/array/arrayWithConstant.cpp +++ b/src/Functions/array/arrayWithConstant.cpp @@ -78,7 +78,7 @@ public: } }; -void registerFunctionArrayWithConstant(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayWithConstant) { factory.registerFunction(); } diff --git a/src/Functions/array/arrayZip.cpp b/src/Functions/array/arrayZip.cpp index 6e77d9a5442..a56a8843e9b 100644 --- a/src/Functions/array/arrayZip.cpp +++ b/src/Functions/array/arrayZip.cpp @@ -93,7 +93,7 @@ public: } }; -void registerFunctionArrayZip(FunctionFactory & factory) +REGISTER_FUNCTION(ArrayZip) { factory.registerFunction(); } diff --git a/src/Functions/array/countEqual.cpp b/src/Functions/array/countEqual.cpp index 4c0fc495f38..a68042f9dea 100644 --- a/src/Functions/array/countEqual.cpp +++ b/src/Functions/array/countEqual.cpp @@ -7,7 +7,7 @@ struct NameCountEqual { static constexpr auto name = "countEqual"; }; using FunctionCountEqual = FunctionArrayIndex; -void registerFunctionCountEqual(FunctionFactory & factory) +REGISTER_FUNCTION(CountEqual) { factory.registerFunction(); } diff --git a/src/Functions/array/emptyArray.cpp b/src/Functions/array/emptyArray.cpp index 28e9b5ca2e2..aeb7f83c6c5 100644 --- a/src/Functions/array/emptyArray.cpp +++ b/src/Functions/array/emptyArray.cpp @@ -55,7 +55,7 @@ void registerFunction(FunctionFactory & factory) } -void registerFunctionsEmptyArray(FunctionFactory & factory) +REGISTER_FUNCTION(EmptyArray) { registerFunction>(factory); registerFunction>(factory); diff --git a/src/Functions/array/emptyArrayToSingle.cpp b/src/Functions/array/emptyArrayToSingle.cpp index d6501216bd8..3fc8f6caa54 100644 --- a/src/Functions/array/emptyArrayToSingle.cpp +++ b/src/Functions/array/emptyArrayToSingle.cpp @@ -417,7 +417,7 @@ ColumnPtr FunctionEmptyArrayToSingle::executeImpl(const ColumnsWithTypeAndName & } -void registerFunctionEmptyArrayToSingle(FunctionFactory & factory) +REGISTER_FUNCTION(EmptyArrayToSingle) { factory.registerFunction(); } diff --git a/src/Functions/array/has.cpp b/src/Functions/array/has.cpp index 1a00c44671e..f08a4f29d2d 100644 --- a/src/Functions/array/has.cpp +++ b/src/Functions/array/has.cpp @@ -8,5 +8,5 @@ struct NameHas { static constexpr auto name = "has"; }; /// has(arr, x) - whether there is an element x in the array. using FunctionHas = FunctionArrayIndex; -void registerFunctionHas(FunctionFactory & factory) { factory.registerFunction(); } +REGISTER_FUNCTION(Has) { factory.registerFunction(); } } diff --git a/src/Functions/array/hasAll.cpp b/src/Functions/array/hasAll.cpp index c1959515e9f..7136db67ebc 100644 --- a/src/Functions/array/hasAll.cpp +++ b/src/Functions/array/hasAll.cpp @@ -14,7 +14,7 @@ public: FunctionArrayHasAll() : FunctionArrayHasAllAny(GatherUtils::ArraySearchType::All, name) {} }; -void registerFunctionHasAll(FunctionFactory & factory) +REGISTER_FUNCTION(HasAll) { factory.registerFunction(); } diff --git a/src/Functions/array/hasAny.cpp b/src/Functions/array/hasAny.cpp index fac810c7ba1..6ec87a6ff34 100644 --- a/src/Functions/array/hasAny.cpp +++ b/src/Functions/array/hasAny.cpp @@ -14,7 +14,7 @@ public: FunctionArrayHasAny() : FunctionArrayHasAllAny(GatherUtils::ArraySearchType::Any, name) {} }; -void registerFunctionHasAny(FunctionFactory & factory) +REGISTER_FUNCTION(HasAny) { factory.registerFunction(); } diff --git a/src/Functions/array/hasSubstr.cpp b/src/Functions/array/hasSubstr.cpp index 886f82f577b..4553026f2fa 100644 --- a/src/Functions/array/hasSubstr.cpp +++ b/src/Functions/array/hasSubstr.cpp @@ -14,7 +14,7 @@ public: FunctionArrayHasSubstr() : FunctionArrayHasAllAny(GatherUtils::ArraySearchType::Substr, name) {} }; -void registerFunctionHasSubstr(FunctionFactory & factory) +REGISTER_FUNCTION(HasSubstr) { factory.registerFunction(); } diff --git a/src/Functions/array/indexOf.cpp b/src/Functions/array/indexOf.cpp index 6ab87e6182c..9e61d22bc68 100644 --- a/src/Functions/array/indexOf.cpp +++ b/src/Functions/array/indexOf.cpp @@ -9,7 +9,7 @@ struct NameIndexOf { static constexpr auto name = "indexOf"; }; /// doesn't. using FunctionIndexOf = FunctionArrayIndex; -void registerFunctionIndexOf(FunctionFactory & factory) +REGISTER_FUNCTION(IndexOf) { factory.registerFunction(); } diff --git a/src/Functions/array/length.cpp b/src/Functions/array/length.cpp index 3e41f9e3a38..dca38474ab0 100644 --- a/src/Functions/array/length.cpp +++ b/src/Functions/array/length.cpp @@ -53,7 +53,7 @@ struct NameLength using FunctionLength = FunctionStringOrArrayToT; -void registerFunctionLength(FunctionFactory & factory) +REGISTER_FUNCTION(Length) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/array/mapOp.cpp b/src/Functions/array/mapOp.cpp index f743cfb5b5d..dccb8001c1c 100644 --- a/src/Functions/array/mapOp.cpp +++ b/src/Functions/array/mapOp.cpp @@ -448,7 +448,7 @@ private: } -void registerFunctionMapOp(FunctionFactory & factory) +REGISTER_FUNCTION(MapOp) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/array/mapPopulateSeries.cpp b/src/Functions/array/mapPopulateSeries.cpp index 8b4a1dda197..8814ce26189 100644 --- a/src/Functions/array/mapPopulateSeries.cpp +++ b/src/Functions/array/mapPopulateSeries.cpp @@ -496,7 +496,7 @@ private: } }; -void registerFunctionMapPopulateSeries(FunctionFactory & factory) +REGISTER_FUNCTION(MapPopulateSeries) { factory.registerFunction(); } diff --git a/src/Functions/array/range.cpp b/src/Functions/array/range.cpp index 47e28da536a..6b3d8ad1139 100644 --- a/src/Functions/array/range.cpp +++ b/src/Functions/array/range.cpp @@ -455,7 +455,7 @@ private: }; -void registerFunctionRange(FunctionFactory & factory) +REGISTER_FUNCTION(Range) { factory.registerFunction(); } diff --git a/src/Functions/array/registerFunctionsArray.cpp b/src/Functions/array/registerFunctionsArray.cpp deleted file mode 100644 index 3bb27cbadf9..00000000000 --- a/src/Functions/array/registerFunctionsArray.cpp +++ /dev/null @@ -1,80 +0,0 @@ -namespace DB -{ -class FunctionFactory; - -void registerFunctionArray(FunctionFactory &); -void registerFunctionArrayElement(FunctionFactory &); -void registerFunctionArrayResize(FunctionFactory &); -void registerFunctionHas(FunctionFactory &); -void registerFunctionHasAll(FunctionFactory &); -void registerFunctionHasAny(FunctionFactory &); -void registerFunctionHasSubstr(FunctionFactory &); -void registerFunctionIndexOf(FunctionFactory &); -void registerFunctionCountEqual(FunctionFactory &); -void registerFunctionArrayIntersect(FunctionFactory &); -void registerFunctionArrayPushFront(FunctionFactory &); -void registerFunctionArrayPushBack(FunctionFactory &); -void registerFunctionArrayPopFront(FunctionFactory &); -void registerFunctionArrayPopBack(FunctionFactory &); -void registerFunctionArrayConcat(FunctionFactory &); -void registerFunctionArraySlice(FunctionFactory &); -void registerFunctionArrayReverse(FunctionFactory &); -void registerFunctionArrayReduce(FunctionFactory &); -void registerFunctionRange(FunctionFactory &); -void registerFunctionsEmptyArray(FunctionFactory &); -void registerFunctionEmptyArrayToSingle(FunctionFactory &); -void registerFunctionArrayEnumerate(FunctionFactory &); -void registerFunctionArrayEnumerateUniq(FunctionFactory &); -void registerFunctionArrayEnumerateDense(FunctionFactory &); -void registerFunctionArrayEnumerateUniqRanked(FunctionFactory &); -void registerFunctionArrayEnumerateDenseRanked(FunctionFactory &); -void registerFunctionArrayUniq(FunctionFactory &); -void registerFunctionArrayDistinct(FunctionFactory &); -void registerFunctionArrayFlatten(FunctionFactory &); -void registerFunctionArrayWithConstant(FunctionFactory &); -void registerFunctionArrayZip(FunctionFactory &); -void registerFunctionArrayAUC(FunctionFactory &); -void registerFunctionArrayReduceInRanges(FunctionFactory &); -void registerFunctionMapOp(FunctionFactory &); -void registerFunctionMapPopulateSeries(FunctionFactory &); - -void registerFunctionsArray(FunctionFactory & factory) -{ - registerFunctionArray(factory); - registerFunctionArrayElement(factory); - registerFunctionArrayResize(factory); - registerFunctionHas(factory); - registerFunctionHasAll(factory); - registerFunctionHasAny(factory); - registerFunctionHasSubstr(factory); - registerFunctionIndexOf(factory); - registerFunctionCountEqual(factory); - registerFunctionArrayIntersect(factory); - registerFunctionArrayPushFront(factory); - registerFunctionArrayPushBack(factory); - registerFunctionArrayPopFront(factory); - registerFunctionArrayPopBack(factory); - registerFunctionArrayConcat(factory); - registerFunctionArraySlice(factory); - registerFunctionArrayReverse(factory); - registerFunctionArrayReduce(factory); - registerFunctionArrayReduceInRanges(factory); - registerFunctionRange(factory); - registerFunctionsEmptyArray(factory); - registerFunctionEmptyArrayToSingle(factory); - registerFunctionArrayEnumerate(factory); - registerFunctionArrayEnumerateUniq(factory); - registerFunctionArrayEnumerateDense(factory); - registerFunctionArrayEnumerateUniqRanked(factory); - registerFunctionArrayEnumerateDenseRanked(factory); - registerFunctionArrayUniq(factory); - registerFunctionArrayDistinct(factory); - registerFunctionArrayFlatten(factory); - registerFunctionArrayWithConstant(factory); - registerFunctionArrayZip(factory); - registerFunctionArrayAUC(factory); - registerFunctionMapOp(factory); - registerFunctionMapPopulateSeries(factory); -} - -} diff --git a/src/Functions/asin.cpp b/src/Functions/asin.cpp index 92391fdef70..a02175367b0 100644 --- a/src/Functions/asin.cpp +++ b/src/Functions/asin.cpp @@ -12,7 +12,7 @@ using FunctionAsin = FunctionMathUnary>; } -void registerFunctionAsin(FunctionFactory & factory) +REGISTER_FUNCTION(Asin) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/asinh.cpp b/src/Functions/asinh.cpp index fd2ac6c7a9f..6af832ae07c 100644 --- a/src/Functions/asinh.cpp +++ b/src/Functions/asinh.cpp @@ -13,7 +13,7 @@ namespace } -void registerFunctionAsinh(FunctionFactory & factory) +REGISTER_FUNCTION(Asinh) { factory.registerFunction(); } diff --git a/src/Functions/assumeNotNull.cpp b/src/Functions/assumeNotNull.cpp index 8f999af9ef0..4dd88163ecb 100644 --- a/src/Functions/assumeNotNull.cpp +++ b/src/Functions/assumeNotNull.cpp @@ -63,7 +63,7 @@ public: } -void registerFunctionAssumeNotNull(FunctionFactory & factory) +REGISTER_FUNCTION(AssumeNotNull) { factory.registerFunction(); } diff --git a/src/Functions/atan.cpp b/src/Functions/atan.cpp index be0af8a9108..b735846fea7 100644 --- a/src/Functions/atan.cpp +++ b/src/Functions/atan.cpp @@ -12,7 +12,7 @@ using FunctionAtan = FunctionMathUnary>; } -void registerFunctionAtan(FunctionFactory & factory) +REGISTER_FUNCTION(Atan) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/atan2.cpp b/src/Functions/atan2.cpp index 0e363440d09..c6a9f70286c 100644 --- a/src/Functions/atan2.cpp +++ b/src/Functions/atan2.cpp @@ -13,7 +13,7 @@ namespace } -void registerFunctionAtan2(FunctionFactory & factory) +REGISTER_FUNCTION(Atan2) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/atanh.cpp b/src/Functions/atanh.cpp index a8154c719fc..fab25414725 100644 --- a/src/Functions/atanh.cpp +++ b/src/Functions/atanh.cpp @@ -13,7 +13,7 @@ namespace } -void registerFunctionAtanh(FunctionFactory & factory) +REGISTER_FUNCTION(Atanh) { factory.registerFunction(); } diff --git a/src/Functions/bar.cpp b/src/Functions/bar.cpp index 0b5e48de067..982e1ff3a25 100644 --- a/src/Functions/bar.cpp +++ b/src/Functions/bar.cpp @@ -129,7 +129,7 @@ public: } -void registerFunctionBar(FunctionFactory & factory) +REGISTER_FUNCTION(Bar) { factory.registerFunction(); } diff --git a/src/Functions/base64Decode.cpp b/src/Functions/base64Decode.cpp index 027fef73911..8922e1e0095 100644 --- a/src/Functions/base64Decode.cpp +++ b/src/Functions/base64Decode.cpp @@ -6,7 +6,7 @@ namespace DB { -void registerFunctionBase64Decode(FunctionFactory & factory) +REGISTER_FUNCTION(Base64Decode) { tb64ini(0, 0); factory.registerFunction>(); diff --git a/src/Functions/base64Encode.cpp b/src/Functions/base64Encode.cpp index 3e456e50379..14523f8b0f3 100644 --- a/src/Functions/base64Encode.cpp +++ b/src/Functions/base64Encode.cpp @@ -8,7 +8,7 @@ namespace DB { -void registerFunctionBase64Encode(FunctionFactory & factory) +REGISTER_FUNCTION(Base64Encode) { tb64ini(0, 0); factory.registerFunction>(); diff --git a/src/Functions/bitAnd.cpp b/src/Functions/bitAnd.cpp index 63f92839638..2e3b79c6710 100644 --- a/src/Functions/bitAnd.cpp +++ b/src/Functions/bitAnd.cpp @@ -42,7 +42,7 @@ using FunctionBitAnd = BinaryArithmeticOverloadResolver(); } diff --git a/src/Functions/bitBoolMaskAnd.cpp b/src/Functions/bitBoolMaskAnd.cpp index ec208615363..2a0735e5ac8 100644 --- a/src/Functions/bitBoolMaskAnd.cpp +++ b/src/Functions/bitBoolMaskAnd.cpp @@ -47,7 +47,7 @@ using FunctionBitBoolMaskAnd = BinaryArithmeticOverloadResolver(); } diff --git a/src/Functions/bitBoolMaskOr.cpp b/src/Functions/bitBoolMaskOr.cpp index 595417d04b4..e0acde17a62 100644 --- a/src/Functions/bitBoolMaskOr.cpp +++ b/src/Functions/bitBoolMaskOr.cpp @@ -47,7 +47,7 @@ using FunctionBitBoolMaskOr = BinaryArithmeticOverloadResolver(); } diff --git a/src/Functions/bitCount.cpp b/src/Functions/bitCount.cpp index 84743ae24ed..d425dd1dca2 100644 --- a/src/Functions/bitCount.cpp +++ b/src/Functions/bitCount.cpp @@ -53,7 +53,7 @@ template <> struct FunctionUnaryArithmeticMonotonicity } }; -void registerFunctionBitCount(FunctionFactory & factory) +REGISTER_FUNCTION(BitCount) { factory.registerFunction(); } diff --git a/src/Functions/bitHammingDistance.cpp b/src/Functions/bitHammingDistance.cpp index afc281e7e76..2090d17432c 100644 --- a/src/Functions/bitHammingDistance.cpp +++ b/src/Functions/bitHammingDistance.cpp @@ -28,7 +28,7 @@ struct NameBitHammingDistance }; using FunctionBitHammingDistance = BinaryArithmeticOverloadResolver; -void registerFunctionBitHammingDistance(FunctionFactory & factory) +REGISTER_FUNCTION(BitHammingDistance) { factory.registerFunction(); } diff --git a/src/Functions/bitNot.cpp b/src/Functions/bitNot.cpp index 3e1e69fccc2..b13becedc31 100644 --- a/src/Functions/bitNot.cpp +++ b/src/Functions/bitNot.cpp @@ -51,7 +51,7 @@ template <> struct FunctionUnaryArithmeticMonotonicity } }; -void registerFunctionBitNot(FunctionFactory & factory) +REGISTER_FUNCTION(BitNot) { factory.registerFunction(); } diff --git a/src/Functions/bitOr.cpp b/src/Functions/bitOr.cpp index 65847e93b8b..40d5f41884e 100644 --- a/src/Functions/bitOr.cpp +++ b/src/Functions/bitOr.cpp @@ -41,7 +41,7 @@ using FunctionBitOr = BinaryArithmeticOverloadResolver(); } diff --git a/src/Functions/bitRotateLeft.cpp b/src/Functions/bitRotateLeft.cpp index 2cddb93e333..8b99d45d9f0 100644 --- a/src/Functions/bitRotateLeft.cpp +++ b/src/Functions/bitRotateLeft.cpp @@ -48,7 +48,7 @@ using FunctionBitRotateLeft = BinaryArithmeticOverloadResolver(); } diff --git a/src/Functions/bitRotateRight.cpp b/src/Functions/bitRotateRight.cpp index 66664c4ab8d..0d84fbd9f64 100644 --- a/src/Functions/bitRotateRight.cpp +++ b/src/Functions/bitRotateRight.cpp @@ -47,7 +47,7 @@ using FunctionBitRotateRight = BinaryArithmeticOverloadResolver(); } diff --git a/src/Functions/bitShiftLeft.cpp b/src/Functions/bitShiftLeft.cpp index 83a31ddc122..b53c2b05da0 100644 --- a/src/Functions/bitShiftLeft.cpp +++ b/src/Functions/bitShiftLeft.cpp @@ -159,7 +159,7 @@ using FunctionBitShiftLeft = BinaryArithmeticOverloadResolver(); } diff --git a/src/Functions/bitShiftRight.cpp b/src/Functions/bitShiftRight.cpp index d12847c5df8..8134a64ac53 100644 --- a/src/Functions/bitShiftRight.cpp +++ b/src/Functions/bitShiftRight.cpp @@ -138,7 +138,7 @@ using FunctionBitShiftRight = BinaryArithmeticOverloadResolver(); } diff --git a/src/Functions/bitSlice.cpp b/src/Functions/bitSlice.cpp index 7d9b48acc89..c2392760194 100644 --- a/src/Functions/bitSlice.cpp +++ b/src/Functions/bitSlice.cpp @@ -419,7 +419,7 @@ public: }; -void registerFunctionBitSlice(FunctionFactory & factory) +REGISTER_FUNCTION(BitSlice) { factory.registerFunction(); } diff --git a/src/Functions/bitSwapLastTwo.cpp b/src/Functions/bitSwapLastTwo.cpp index 566e6a47352..e7f07160693 100644 --- a/src/Functions/bitSwapLastTwo.cpp +++ b/src/Functions/bitSwapLastTwo.cpp @@ -62,7 +62,7 @@ template <> struct FunctionUnaryArithmeticMonotonicity } }; -void registerFunctionBitSwapLastTwo(FunctionFactory & factory) +REGISTER_FUNCTION(BitSwapLastTwo) { factory.registerFunction(); } diff --git a/src/Functions/bitTest.cpp b/src/Functions/bitTest.cpp index 9c12268d2ef..ac21423ced6 100644 --- a/src/Functions/bitTest.cpp +++ b/src/Functions/bitTest.cpp @@ -39,7 +39,7 @@ using FunctionBitTest = BinaryArithmeticOverloadResolver(); } diff --git a/src/Functions/bitTestAll.cpp b/src/Functions/bitTestAll.cpp index 901fd600106..a2dcef3eb96 100644 --- a/src/Functions/bitTestAll.cpp +++ b/src/Functions/bitTestAll.cpp @@ -17,7 +17,7 @@ using FunctionBitTestAll = FunctionBitTestMany; } -void registerFunctionBitTestAll(FunctionFactory & factory) +REGISTER_FUNCTION(BitTestAll) { factory.registerFunction(); } diff --git a/src/Functions/bitTestAny.cpp b/src/Functions/bitTestAny.cpp index 6874e74e2be..6b20d6c184c 100644 --- a/src/Functions/bitTestAny.cpp +++ b/src/Functions/bitTestAny.cpp @@ -17,7 +17,7 @@ using FunctionBitTestAny = FunctionBitTestMany; } -void registerFunctionBitTestAny(FunctionFactory & factory) +REGISTER_FUNCTION(BitTestAny) { factory.registerFunction(); } diff --git a/src/Functions/bitWrapperFunc.cpp b/src/Functions/bitWrapperFunc.cpp index f47ffbfe883..83c89c753fc 100644 --- a/src/Functions/bitWrapperFunc.cpp +++ b/src/Functions/bitWrapperFunc.cpp @@ -50,7 +50,7 @@ template <> struct FunctionUnaryArithmeticMonotonicity } }; -void registerFunctionBitWrapperFunc(FunctionFactory & factory) +REGISTER_FUNCTION(BitWrapperFunc) { factory.registerFunction(); } diff --git a/src/Functions/bitXor.cpp b/src/Functions/bitXor.cpp index 8f3b6a1f9ca..89aaf5eadd1 100644 --- a/src/Functions/bitXor.cpp +++ b/src/Functions/bitXor.cpp @@ -41,7 +41,7 @@ using FunctionBitXor = BinaryArithmeticOverloadResolver(); } diff --git a/src/Functions/blockNumber.cpp b/src/Functions/blockNumber.cpp index a62a0ac873f..0a8e51206a6 100644 --- a/src/Functions/blockNumber.cpp +++ b/src/Functions/blockNumber.cpp @@ -68,7 +68,7 @@ public: } -void registerFunctionBlockNumber(FunctionFactory & factory) +REGISTER_FUNCTION(BlockNumber) { factory.registerFunction(); } diff --git a/src/Functions/blockSerializedSize.cpp b/src/Functions/blockSerializedSize.cpp index dd4fd1a4588..d406984c51c 100644 --- a/src/Functions/blockSerializedSize.cpp +++ b/src/Functions/blockSerializedSize.cpp @@ -66,7 +66,7 @@ public: } -void registerFunctionBlockSerializedSize(FunctionFactory & factory) +REGISTER_FUNCTION(BlockSerializedSize) { factory.registerFunction(); } diff --git a/src/Functions/blockSize.cpp b/src/Functions/blockSize.cpp index a5e87f58d98..af3c4ed27b4 100644 --- a/src/Functions/blockSize.cpp +++ b/src/Functions/blockSize.cpp @@ -56,7 +56,7 @@ public: } -void registerFunctionBlockSize(FunctionFactory & factory) +REGISTER_FUNCTION(BlockSize) { factory.registerFunction(); } diff --git a/src/Functions/byteSize.cpp b/src/Functions/byteSize.cpp index 03ac85b7cfc..93a3a86641a 100644 --- a/src/Functions/byteSize.cpp +++ b/src/Functions/byteSize.cpp @@ -80,7 +80,7 @@ public: } -void registerFunctionByteSize(FunctionFactory & factory) +REGISTER_FUNCTION(ByteSize) { factory.registerFunction(); } diff --git a/src/Functions/caseWithExpression.cpp b/src/Functions/caseWithExpression.cpp index e06a01431da..b198d60eb7d 100644 --- a/src/Functions/caseWithExpression.cpp +++ b/src/Functions/caseWithExpression.cpp @@ -107,7 +107,7 @@ private: } -void registerFunctionCaseWithExpression(FunctionFactory & factory) +REGISTER_FUNCTION(CaseWithExpression) { factory.registerFunction(); diff --git a/src/Functions/castOrDefault.cpp b/src/Functions/castOrDefault.cpp index 2e9f13545fd..1f527b7ec23 100644 --- a/src/Functions/castOrDefault.cpp +++ b/src/Functions/castOrDefault.cpp @@ -363,7 +363,7 @@ using FunctionToDecimal256OrDefault = FunctionCastOrDefaultTyped; -void registerFunctionCastOrDefault(FunctionFactory & factory) +REGISTER_FUNCTION(CastOrDefault) { factory.registerFunction(); diff --git a/src/Functions/cbrt.cpp b/src/Functions/cbrt.cpp index 94c8627c320..0b9f0d5a0b6 100644 --- a/src/Functions/cbrt.cpp +++ b/src/Functions/cbrt.cpp @@ -11,7 +11,7 @@ using FunctionCbrt = FunctionMathUnary>; } -void registerFunctionCbrt(FunctionFactory & factory) +REGISTER_FUNCTION(Cbrt) { factory.registerFunction(); } diff --git a/src/Functions/coalesce.cpp b/src/Functions/coalesce.cpp index b2b234df515..aafbcd7d714 100644 --- a/src/Functions/coalesce.cpp +++ b/src/Functions/coalesce.cpp @@ -174,7 +174,7 @@ private: } -void registerFunctionCoalesce(FunctionFactory & factory) +REGISTER_FUNCTION(Coalesce) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp index e11071265ce..3b02f2c0b78 100644 --- a/src/Functions/concat.cpp +++ b/src/Functions/concat.cpp @@ -228,7 +228,7 @@ private: } -void registerFunctionsConcat(FunctionFactory & factory) +REGISTER_FUNCTION(Concat) { factory.registerFunction(FunctionFactory::CaseInsensitive); factory.registerFunction(); diff --git a/src/Functions/connectionId.cpp b/src/Functions/connectionId.cpp index 69c8b9e86ea..b9d772e3871 100644 --- a/src/Functions/connectionId.cpp +++ b/src/Functions/connectionId.cpp @@ -31,7 +31,7 @@ public: } }; -void registerFunctionConnectionId(FunctionFactory & factory) +REGISTER_FUNCTION(ConnectionId) { factory.registerFunction(FunctionFactory::CaseInsensitive); factory.registerAlias("connection_id", "connectionID", FunctionFactory::CaseInsensitive); diff --git a/src/Functions/convertCharset.cpp b/src/Functions/convertCharset.cpp index ccfb28ad7ef..49faae521f7 100644 --- a/src/Functions/convertCharset.cpp +++ b/src/Functions/convertCharset.cpp @@ -214,7 +214,7 @@ public: } -void registerFunctionConvertCharset(FunctionFactory & factory) +REGISTER_FUNCTION(ConvertCharset) { factory.registerFunction(); } diff --git a/src/Functions/cos.cpp b/src/Functions/cos.cpp index e18524dd56b..e7c9d7759ed 100644 --- a/src/Functions/cos.cpp +++ b/src/Functions/cos.cpp @@ -11,7 +11,7 @@ using FunctionCos = FunctionMathUnary>; } -void registerFunctionCos(FunctionFactory & factory) +REGISTER_FUNCTION(Cos) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/cosh.cpp b/src/Functions/cosh.cpp index 88753fcb95b..54b52051aab 100644 --- a/src/Functions/cosh.cpp +++ b/src/Functions/cosh.cpp @@ -13,7 +13,7 @@ namespace } -void registerFunctionCosh(FunctionFactory & factory) +REGISTER_FUNCTION(Cosh) { factory.registerFunction(); } diff --git a/src/Functions/countDigits.cpp b/src/Functions/countDigits.cpp index 47877f042b9..e5142a89689 100644 --- a/src/Functions/countDigits.cpp +++ b/src/Functions/countDigits.cpp @@ -142,7 +142,7 @@ private: } -void registerFunctionCountDigits(FunctionFactory & factory) +REGISTER_FUNCTION(CountDigits) { factory.registerFunction(); } diff --git a/src/Functions/countMatches.cpp b/src/Functions/countMatches.cpp index e1ad445befb..d8948f85d44 100644 --- a/src/Functions/countMatches.cpp +++ b/src/Functions/countMatches.cpp @@ -20,7 +20,7 @@ struct FunctionCountMatchesCaseInsensitive namespace DB { -void registerFunctionCountMatches(FunctionFactory & factory) +REGISTER_FUNCTION(CountMatches) { factory.registerFunction>(FunctionFactory::CaseSensitive); factory.registerFunction>(FunctionFactory::CaseSensitive); diff --git a/src/Functions/countSubstrings.cpp b/src/Functions/countSubstrings.cpp index 1bf95f9526f..ba8d150fb41 100644 --- a/src/Functions/countSubstrings.cpp +++ b/src/Functions/countSubstrings.cpp @@ -17,7 +17,7 @@ using FunctionCountSubstrings = FunctionsStringSearch(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/countSubstringsCaseInsensitive.cpp b/src/Functions/countSubstringsCaseInsensitive.cpp index fa234953cc3..ada8cf77bc9 100644 --- a/src/Functions/countSubstringsCaseInsensitive.cpp +++ b/src/Functions/countSubstringsCaseInsensitive.cpp @@ -17,7 +17,7 @@ using FunctionCountSubstringsCaseInsensitive = FunctionsStringSearch(); } diff --git a/src/Functions/countSubstringsCaseInsensitiveUTF8.cpp b/src/Functions/countSubstringsCaseInsensitiveUTF8.cpp index 93f77fddd7f..3f71bca63d2 100644 --- a/src/Functions/countSubstringsCaseInsensitiveUTF8.cpp +++ b/src/Functions/countSubstringsCaseInsensitiveUTF8.cpp @@ -18,7 +18,7 @@ using FunctionCountSubstringsCaseInsensitiveUTF8 = FunctionsStringSearch< } -void registerFunctionCountSubstringsCaseInsensitiveUTF8(FunctionFactory & factory) +REGISTER_FUNCTION(CountSubstringsCaseInsensitiveUTF8) { factory.registerFunction(); } diff --git a/src/Functions/currentDatabase.cpp b/src/Functions/currentDatabase.cpp index c5f559d0838..981b324fb51 100644 --- a/src/Functions/currentDatabase.cpp +++ b/src/Functions/currentDatabase.cpp @@ -51,7 +51,7 @@ public: } -void registerFunctionCurrentDatabase(FunctionFactory & factory) +REGISTER_FUNCTION(CurrentDatabase) { factory.registerFunction(); factory.registerFunction("DATABASE", FunctionFactory::CaseInsensitive); diff --git a/src/Functions/currentProfiles.cpp b/src/Functions/currentProfiles.cpp index 22d6af2f16e..eee458f4f63 100644 --- a/src/Functions/currentProfiles.cpp +++ b/src/Functions/currentProfiles.cpp @@ -79,7 +79,7 @@ namespace }; } -void registerFunctionCurrentProfiles(FunctionFactory & factory) +REGISTER_FUNCTION(CurrentProfiles) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/currentRoles.cpp b/src/Functions/currentRoles.cpp index 69f3061064a..45d2000d088 100644 --- a/src/Functions/currentRoles.cpp +++ b/src/Functions/currentRoles.cpp @@ -81,7 +81,7 @@ namespace }; } -void registerFunctionCurrentRoles(FunctionFactory & factory) +REGISTER_FUNCTION(CurrentRoles) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/currentUser.cpp b/src/Functions/currentUser.cpp index efa6e2cad2a..67b5d7626bf 100644 --- a/src/Functions/currentUser.cpp +++ b/src/Functions/currentUser.cpp @@ -51,7 +51,7 @@ public: } -void registerFunctionCurrentUser(FunctionFactory & factory) +REGISTER_FUNCTION(CurrentUser) { factory.registerFunction(); factory.registerAlias("user", FunctionCurrentUser::name, FunctionFactory::CaseInsensitive); diff --git a/src/Functions/dateDiff.cpp b/src/Functions/dateDiff.cpp index e778e4dc6a9..dd6e9e70381 100644 --- a/src/Functions/dateDiff.cpp +++ b/src/Functions/dateDiff.cpp @@ -261,7 +261,7 @@ private: } -void registerFunctionDateDiff(FunctionFactory & factory) +REGISTER_FUNCTION(DateDiff) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp index 91ea8995777..8f551dfd136 100644 --- a/src/Functions/dateName.cpp +++ b/src/Functions/dateName.cpp @@ -343,7 +343,7 @@ private: } -void registerFunctionDateName(FunctionFactory & factory) +REGISTER_FUNCTION(DateName) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/date_trunc.cpp b/src/Functions/date_trunc.cpp index da6a4839ca0..bb891ac702f 100644 --- a/src/Functions/date_trunc.cpp +++ b/src/Functions/date_trunc.cpp @@ -153,7 +153,7 @@ private: } -void registerFunctionDateTrunc(FunctionFactory & factory) +REGISTER_FUNCTION(DateTrunc) { factory.registerFunction(FunctionFactory::CaseInsensitive); diff --git a/src/Functions/decodeXMLComponent.cpp b/src/Functions/decodeXMLComponent.cpp index 79af3024c09..80d1b5c3d79 100644 --- a/src/Functions/decodeXMLComponent.cpp +++ b/src/Functions/decodeXMLComponent.cpp @@ -231,7 +231,7 @@ namespace } -void registerFunctionDecodeXMLComponent(FunctionFactory & factory) +REGISTER_FUNCTION(DecodeXMLComponent) { factory.registerFunction(); } diff --git a/src/Functions/decrypt.cpp b/src/Functions/decrypt.cpp index 1cbda0dba99..da794116a41 100644 --- a/src/Functions/decrypt.cpp +++ b/src/Functions/decrypt.cpp @@ -19,7 +19,7 @@ struct DecryptImpl namespace DB { -void registerFunctionDecrypt(FunctionFactory & factory) +REGISTER_FUNCTION(Decrypt) { factory.registerFunction>(); } diff --git a/src/Functions/defaultValueOfArgumentType.cpp b/src/Functions/defaultValueOfArgumentType.cpp index f4031692b4b..2ad75a9add4 100644 --- a/src/Functions/defaultValueOfArgumentType.cpp +++ b/src/Functions/defaultValueOfArgumentType.cpp @@ -56,7 +56,7 @@ public: } -void registerFunctionDefaultValueOfArgumentType(FunctionFactory & factory) +REGISTER_FUNCTION(DefaultValueOfArgumentType) { factory.registerFunction(); } diff --git a/src/Functions/defaultValueOfTypeName.cpp b/src/Functions/defaultValueOfTypeName.cpp index 2efbaef46c5..b20048bbde7 100644 --- a/src/Functions/defaultValueOfTypeName.cpp +++ b/src/Functions/defaultValueOfTypeName.cpp @@ -63,7 +63,7 @@ public: } -void registerFunctionDefaultValueOfTypeName(FunctionFactory & factory) +REGISTER_FUNCTION(DefaultValueOfTypeName) { factory.registerFunction(); } diff --git a/src/Functions/degrees.cpp b/src/Functions/degrees.cpp index 543b7ac7315..2881f8d2df6 100644 --- a/src/Functions/degrees.cpp +++ b/src/Functions/degrees.cpp @@ -21,7 +21,7 @@ namespace using FunctionDegrees = FunctionMathUnary>; } -void registerFunctionDegrees(FunctionFactory & factory) +REGISTER_FUNCTION(Degrees) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/demange.cpp b/src/Functions/demange.cpp index ecf6661d20d..9026790f740 100644 --- a/src/Functions/demange.cpp +++ b/src/Functions/demange.cpp @@ -96,7 +96,7 @@ public: } -void registerFunctionDemangle(FunctionFactory & factory) +REGISTER_FUNCTION(Demangle) { factory.registerFunction(); } diff --git a/src/Functions/divide.cpp b/src/Functions/divide.cpp index 7b8fcf94313..1d042e19b9f 100644 --- a/src/Functions/divide.cpp +++ b/src/Functions/divide.cpp @@ -36,7 +36,7 @@ struct DivideFloatingImpl struct NameDivide { static constexpr auto name = "divide"; }; using FunctionDivide = BinaryArithmeticOverloadResolver; -void registerFunctionDivide(FunctionFactory & factory) +REGISTER_FUNCTION(Divide) { factory.registerFunction(); } diff --git a/src/Functions/dumpColumnStructure.cpp b/src/Functions/dumpColumnStructure.cpp index 8435aa4b1aa..11c0cb81922 100644 --- a/src/Functions/dumpColumnStructure.cpp +++ b/src/Functions/dumpColumnStructure.cpp @@ -54,7 +54,7 @@ public: } -void registerFunctionDumpColumnStructure(FunctionFactory & factory) +REGISTER_FUNCTION(DumpColumnStructure) { factory.registerFunction(); } diff --git a/src/Functions/empty.cpp b/src/Functions/empty.cpp index d53f3585158..51811d21a0c 100644 --- a/src/Functions/empty.cpp +++ b/src/Functions/empty.cpp @@ -17,7 +17,7 @@ using FunctionEmpty = FunctionStringOrArrayToT, NameEmpty, UInt } -void registerFunctionEmpty(FunctionFactory & factory) +REGISTER_FUNCTION(Empty) { factory.registerFunction(); } diff --git a/src/Functions/encodeXMLComponent.cpp b/src/Functions/encodeXMLComponent.cpp index 380d0da69ac..5d31d7f463d 100644 --- a/src/Functions/encodeXMLComponent.cpp +++ b/src/Functions/encodeXMLComponent.cpp @@ -137,7 +137,7 @@ namespace } -void registerFunctionEncodeXMLComponent(FunctionFactory & factory) +REGISTER_FUNCTION(EncodeXMLComponent) { factory.registerFunction(); } diff --git a/src/Functions/encrypt.cpp b/src/Functions/encrypt.cpp index 807443a2fb8..dc68b650d74 100644 --- a/src/Functions/encrypt.cpp +++ b/src/Functions/encrypt.cpp @@ -19,7 +19,7 @@ struct EncryptImpl namespace DB { -void registerFunctionEncrypt(FunctionFactory & factory) +REGISTER_FUNCTION(Encrypt) { factory.registerFunction>(); } diff --git a/src/Functions/endsWith.cpp b/src/Functions/endsWith.cpp index 5f32fbbe200..308eb67e4b8 100644 --- a/src/Functions/endsWith.cpp +++ b/src/Functions/endsWith.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionEndsWith = FunctionStartsEndsWith; -void registerFunctionEndsWith(FunctionFactory & factory) +REGISTER_FUNCTION(EndsWith) { factory.registerFunction(); } diff --git a/src/Functions/equals.cpp b/src/Functions/equals.cpp index 868a491050d..de1cf623ea6 100644 --- a/src/Functions/equals.cpp +++ b/src/Functions/equals.cpp @@ -7,7 +7,7 @@ namespace DB using FunctionEquals = FunctionComparison; -void registerFunctionEquals(FunctionFactory & factory) +REGISTER_FUNCTION(Equals) { factory.registerFunction(); } diff --git a/src/Functions/erf.cpp b/src/Functions/erf.cpp index cb9428a3dbd..3e5ee9fa724 100644 --- a/src/Functions/erf.cpp +++ b/src/Functions/erf.cpp @@ -11,7 +11,7 @@ using FunctionErf = FunctionMathUnary } -void registerFunctionErf(FunctionFactory & factory) +REGISTER_FUNCTION(Erf) { factory.registerFunction(); } diff --git a/src/Functions/erfc.cpp b/src/Functions/erfc.cpp index 48707ff26d6..7596d18ebde 100644 --- a/src/Functions/erfc.cpp +++ b/src/Functions/erfc.cpp @@ -11,7 +11,7 @@ using FunctionErfc = FunctionMathUnary(); } diff --git a/src/Functions/errorCodeToName.cpp b/src/Functions/errorCodeToName.cpp index f447621c23c..1736311c6cc 100644 --- a/src/Functions/errorCodeToName.cpp +++ b/src/Functions/errorCodeToName.cpp @@ -54,7 +54,7 @@ public: }; -void registerFunctionErrorCodeToName(FunctionFactory & factory) +REGISTER_FUNCTION(ErrorCodeToName) { factory.registerFunction(); } diff --git a/src/Functions/evalMLMethod.cpp b/src/Functions/evalMLMethod.cpp index 876fec39c7b..003aaa2d312 100644 --- a/src/Functions/evalMLMethod.cpp +++ b/src/Functions/evalMLMethod.cpp @@ -92,7 +92,7 @@ public: } -void registerFunctionEvalMLMethod(FunctionFactory & factory) +REGISTER_FUNCTION(EvalMLMethod) { factory.registerFunction(); } diff --git a/src/Functions/exp.cpp b/src/Functions/exp.cpp index c6eb3335f25..71037692f15 100644 --- a/src/Functions/exp.cpp +++ b/src/Functions/exp.cpp @@ -34,7 +34,7 @@ using FunctionExp = FunctionMathUnary>; } -void registerFunctionExp(FunctionFactory & factory) +REGISTER_FUNCTION(Exp) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/exp10.cpp b/src/Functions/exp10.cpp index abd0755c245..a21350c532c 100644 --- a/src/Functions/exp10.cpp +++ b/src/Functions/exp10.cpp @@ -12,7 +12,7 @@ using FunctionExp10 = FunctionMathUnary(); } diff --git a/src/Functions/exp2.cpp b/src/Functions/exp2.cpp index 8825b849346..aff84f65e23 100644 --- a/src/Functions/exp2.cpp +++ b/src/Functions/exp2.cpp @@ -11,7 +11,7 @@ using FunctionExp2 = FunctionMathUnary>; } -void registerFunctionExp2(FunctionFactory & factory) +REGISTER_FUNCTION(Exp2) { factory.registerFunction(); } diff --git a/src/Functions/extract.cpp b/src/Functions/extract.cpp index f5917015e27..5d539e03dae 100644 --- a/src/Functions/extract.cpp +++ b/src/Functions/extract.cpp @@ -65,7 +65,7 @@ using FunctionExtract = FunctionsStringSearchToString; } -void registerFunctionExtract(FunctionFactory & factory) +REGISTER_FUNCTION(Extract) { factory.registerFunction(); } diff --git a/src/Functions/extractAllGroupsHorizontal.cpp b/src/Functions/extractAllGroupsHorizontal.cpp index fba7483ba03..d0ed7757384 100644 --- a/src/Functions/extractAllGroupsHorizontal.cpp +++ b/src/Functions/extractAllGroupsHorizontal.cpp @@ -15,7 +15,7 @@ struct HorizontalImpl namespace DB { -void registerFunctionExtractAllGroupsHorizontal(FunctionFactory & factory) +REGISTER_FUNCTION(ExtractAllGroupsHorizontal) { factory.registerFunction>(); } diff --git a/src/Functions/extractAllGroupsVertical.cpp b/src/Functions/extractAllGroupsVertical.cpp index bf33eef70f3..87a0b4cf7bc 100644 --- a/src/Functions/extractAllGroupsVertical.cpp +++ b/src/Functions/extractAllGroupsVertical.cpp @@ -15,7 +15,7 @@ struct VerticalImpl namespace DB { -void registerFunctionExtractAllGroupsVertical(FunctionFactory & factory) +REGISTER_FUNCTION(ExtractAllGroupsVertical) { factory.registerFunction>(); factory.registerAlias("extractAllGroups", VerticalImpl::Name, FunctionFactory::CaseSensitive); diff --git a/src/Functions/extractGroups.cpp b/src/Functions/extractGroups.cpp index c6633732aaa..eb6e609a4be 100644 --- a/src/Functions/extractGroups.cpp +++ b/src/Functions/extractGroups.cpp @@ -108,7 +108,7 @@ public: } -void registerFunctionExtractGroups(FunctionFactory & factory) +REGISTER_FUNCTION(ExtractGroups) { factory.registerFunction(); } diff --git a/src/Functions/extractTextFromHTML.cpp b/src/Functions/extractTextFromHTML.cpp index f321a59f734..3c95431452f 100644 --- a/src/Functions/extractTextFromHTML.cpp +++ b/src/Functions/extractTextFromHTML.cpp @@ -351,7 +351,7 @@ public: } }; -void registerFunctionExtractTextFromHTML(FunctionFactory & factory) +REGISTER_FUNCTION(ExtractTextFromHTML) { factory.registerFunction(); } diff --git a/src/Functions/filesystem.cpp b/src/Functions/filesystem.cpp index 36db68617e9..12813c3d852 100644 --- a/src/Functions/filesystem.cpp +++ b/src/Functions/filesystem.cpp @@ -66,7 +66,7 @@ private: } -void registerFunctionFilesystem(FunctionFactory & factory) +REGISTER_FUNCTION(Filesystem) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/finalizeAggregation.cpp b/src/Functions/finalizeAggregation.cpp index 237d9fa3ef8..61604a3abc0 100644 --- a/src/Functions/finalizeAggregation.cpp +++ b/src/Functions/finalizeAggregation.cpp @@ -73,7 +73,7 @@ public: } -void registerFunctionFinalizeAggregation(FunctionFactory & factory) +REGISTER_FUNCTION(FinalizeAggregation) { factory.registerFunction(); } diff --git a/src/Functions/flattenTuple.cpp b/src/Functions/flattenTuple.cpp index f5d5b4cb773..bde081b40a0 100644 --- a/src/Functions/flattenTuple.cpp +++ b/src/Functions/flattenTuple.cpp @@ -60,7 +60,7 @@ public: } -void registerFunctionFlattenTuple(FunctionFactory & factory) +REGISTER_FUNCTION(FlattenTuple) { factory.registerFunction(); } diff --git a/src/Functions/formatDateTime.cpp b/src/Functions/formatDateTime.cpp index e2ec90f4e61..37f1f7e83f8 100644 --- a/src/Functions/formatDateTime.cpp +++ b/src/Functions/formatDateTime.cpp @@ -730,7 +730,7 @@ using FunctionFROM_UNIXTIME = FunctionFormatDateTimeImpl } -void registerFunctionFormatDateTime(FunctionFactory & factory) +REGISTER_FUNCTION(FormatDateTime) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/formatReadableQuantity.cpp b/src/Functions/formatReadableQuantity.cpp index 2ccfbd4a857..682fac88969 100644 --- a/src/Functions/formatReadableQuantity.cpp +++ b/src/Functions/formatReadableQuantity.cpp @@ -18,7 +18,7 @@ namespace }; } -void registerFunctionFormatReadableQuantity(FunctionFactory & factory) +REGISTER_FUNCTION(FormatReadableQuantity) { factory.registerFunction>(); } diff --git a/src/Functions/formatReadableSize.cpp b/src/Functions/formatReadableSize.cpp index b0b79834eef..95441d43b2f 100644 --- a/src/Functions/formatReadableSize.cpp +++ b/src/Functions/formatReadableSize.cpp @@ -18,7 +18,7 @@ namespace }; } -void registerFunctionFormatReadableSize(FunctionFactory & factory) +REGISTER_FUNCTION(FormatReadableSize) { factory.registerFunction>(); } diff --git a/src/Functions/formatReadableTimeDelta.cpp b/src/Functions/formatReadableTimeDelta.cpp index 219c2d95353..058d4c0548c 100644 --- a/src/Functions/formatReadableTimeDelta.cpp +++ b/src/Functions/formatReadableTimeDelta.cpp @@ -222,7 +222,7 @@ public: } -void registerFunctionFormatReadableTimeDelta(FunctionFactory & factory) +REGISTER_FUNCTION(FormatReadableTimeDelta) { factory.registerFunction(); } diff --git a/src/Functions/formatRow.cpp b/src/Functions/formatRow.cpp index 77fc2118791..0c2df48105e 100644 --- a/src/Functions/formatRow.cpp +++ b/src/Functions/formatRow.cpp @@ -125,7 +125,7 @@ private: } -void registerFunctionFormatRow(FunctionFactory & factory) +REGISTER_FUNCTION(FormatRow) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/formatString.cpp b/src/Functions/formatString.cpp index ee90a082e5b..3d04d6e94a1 100644 --- a/src/Functions/formatString.cpp +++ b/src/Functions/formatString.cpp @@ -135,7 +135,7 @@ using FunctionFormat = FormatFunction; } -void registerFunctionFormat(FunctionFactory & factory) +REGISTER_FUNCTION(Format) { factory.registerFunction(); } diff --git a/src/Functions/fromModifiedJulianDay.cpp b/src/Functions/fromModifiedJulianDay.cpp index e699547e346..ff434715338 100644 --- a/src/Functions/fromModifiedJulianDay.cpp +++ b/src/Functions/fromModifiedJulianDay.cpp @@ -232,7 +232,7 @@ namespace DB static constexpr auto name = "fromModifiedJulianDayOrNull"; }; - void registerFunctionFromModifiedJulianDay(FunctionFactory & factory) + REGISTER_FUNCTION(FromModifiedJulianDay) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/fromUnixTimestamp64Micro.cpp b/src/Functions/fromUnixTimestamp64Micro.cpp index 1fd98e6e673..70dcbcd1d4b 100644 --- a/src/Functions/fromUnixTimestamp64Micro.cpp +++ b/src/Functions/fromUnixTimestamp64Micro.cpp @@ -4,7 +4,7 @@ namespace DB { -void registerFromUnixTimestamp64Micro(FunctionFactory & factory) +REGISTER_FUNCTION(FromUnixTimestamp64Micro) { factory.registerFunction("fromUnixTimestamp64Micro", [](ContextPtr){ return std::make_unique( diff --git a/src/Functions/fromUnixTimestamp64Milli.cpp b/src/Functions/fromUnixTimestamp64Milli.cpp index 90f18699e5d..532013dfe5f 100644 --- a/src/Functions/fromUnixTimestamp64Milli.cpp +++ b/src/Functions/fromUnixTimestamp64Milli.cpp @@ -4,7 +4,7 @@ namespace DB { -void registerFromUnixTimestamp64Milli(FunctionFactory & factory) +REGISTER_FUNCTION(FromUnixTimestamp64Milli) { factory.registerFunction("fromUnixTimestamp64Milli", [](ContextPtr){ return std::make_unique( diff --git a/src/Functions/fromUnixTimestamp64Nano.cpp b/src/Functions/fromUnixTimestamp64Nano.cpp index c489b48fb6e..96afdda0fa8 100644 --- a/src/Functions/fromUnixTimestamp64Nano.cpp +++ b/src/Functions/fromUnixTimestamp64Nano.cpp @@ -4,7 +4,7 @@ namespace DB { -void registerFromUnixTimestamp64Nano(FunctionFactory & factory) +REGISTER_FUNCTION(FromUnixTimestamp64Nano) { factory.registerFunction("fromUnixTimestamp64Nano", [](ContextPtr){ return std::make_unique( diff --git a/src/Functions/fuzzBits.cpp b/src/Functions/fuzzBits.cpp index 8b54026724d..d97e8aff7ff 100644 --- a/src/Functions/fuzzBits.cpp +++ b/src/Functions/fuzzBits.cpp @@ -179,7 +179,7 @@ public: } -void registerFunctionFuzzBits(FunctionFactory & factory) +REGISTER_FUNCTION(FuzzBits) { factory.registerFunction(); } diff --git a/src/Functions/gcd.cpp b/src/Functions/gcd.cpp index c8e70dc87a2..0cd017bb0b4 100644 --- a/src/Functions/gcd.cpp +++ b/src/Functions/gcd.cpp @@ -29,7 +29,7 @@ using FunctionGCD = BinaryArithmeticOverloadResolver(); } diff --git a/src/Functions/generateUUIDv4.cpp b/src/Functions/generateUUIDv4.cpp index e4ecf5358f9..da4e324182a 100644 --- a/src/Functions/generateUUIDv4.cpp +++ b/src/Functions/generateUUIDv4.cpp @@ -100,7 +100,7 @@ private: ImplementationSelector selector; }; -void registerFunctionGenerateUUIDv4(FunctionFactory & factory) +REGISTER_FUNCTION(GenerateUUIDv4) { factory.registerFunction(); } diff --git a/src/Functions/geoToH3.cpp b/src/Functions/geoToH3.cpp index edbed6335ef..284598ee4d5 100644 --- a/src/Functions/geoToH3.cpp +++ b/src/Functions/geoToH3.cpp @@ -141,7 +141,7 @@ public: } -void registerFunctionGeoToH3(FunctionFactory & factory) +REGISTER_FUNCTION(GeoToH3) { factory.registerFunction(); } diff --git a/src/Functions/geoToS2.cpp b/src/Functions/geoToS2.cpp index 8d90552652a..6cfa892b193 100644 --- a/src/Functions/geoToS2.cpp +++ b/src/Functions/geoToS2.cpp @@ -119,7 +119,7 @@ public: } -void registerFunctionGeoToS2(FunctionFactory & factory) +REGISTER_FUNCTION(GeoToS2) { factory.registerFunction(); } diff --git a/src/Functions/geohashDecode.cpp b/src/Functions/geohashDecode.cpp index b4e5d8e46e9..b8a38c46dbf 100644 --- a/src/Functions/geohashDecode.cpp +++ b/src/Functions/geohashDecode.cpp @@ -93,7 +93,7 @@ public: } -void registerFunctionGeohashDecode(FunctionFactory & factory) +REGISTER_FUNCTION(GeohashDecode) { factory.registerFunction(); } diff --git a/src/Functions/geohashEncode.cpp b/src/Functions/geohashEncode.cpp index ebfa8167998..2f05a0b902a 100644 --- a/src/Functions/geohashEncode.cpp +++ b/src/Functions/geohashEncode.cpp @@ -133,7 +133,7 @@ public: } -void registerFunctionGeohashEncode(FunctionFactory & factory) +REGISTER_FUNCTION(GeohashEncode) { factory.registerFunction(); } diff --git a/src/Functions/geohashesInBox.cpp b/src/Functions/geohashesInBox.cpp index f6a8526ba9f..9ba4a62a2a0 100644 --- a/src/Functions/geohashesInBox.cpp +++ b/src/Functions/geohashesInBox.cpp @@ -181,7 +181,7 @@ public: } -void registerFunctionGeohashesInBox(FunctionFactory & factory) +REGISTER_FUNCTION(GeohashesInBox) { factory.registerFunction(); } diff --git a/src/Functions/getFuzzerData.cpp b/src/Functions/getFuzzerData.cpp index f516c871950..6d748619926 100644 --- a/src/Functions/getFuzzerData.cpp +++ b/src/Functions/getFuzzerData.cpp @@ -1,11 +1,13 @@ +#ifdef FUZZING_MODE #include namespace DB { -void registerFunctionGetFuzzerData(FunctionFactory & factory) +REGISTER_FUNCTION(GetFuzzerData) { factory.registerFunction(); } } +#endif diff --git a/src/Functions/getMacro.cpp b/src/Functions/getMacro.cpp index 29e706ce2a0..d5c9f8439dd 100644 --- a/src/Functions/getMacro.cpp +++ b/src/Functions/getMacro.cpp @@ -84,7 +84,7 @@ public: } -void registerFunctionGetMacro(FunctionFactory & factory) +REGISTER_FUNCTION(GetMacro) { factory.registerFunction(); } diff --git a/src/Functions/getScalar.cpp b/src/Functions/getScalar.cpp index c165ef26ffa..b30dbc4299b 100644 --- a/src/Functions/getScalar.cpp +++ b/src/Functions/getScalar.cpp @@ -143,7 +143,7 @@ struct GetShardCount } -void registerFunctionGetScalar(FunctionFactory & factory) +REGISTER_FUNCTION(GetScalar) { factory.registerFunction(); factory.registerFunction>(); diff --git a/src/Functions/getServerPort.cpp b/src/Functions/getServerPort.cpp index 8596bcd6a07..054f7b538f7 100644 --- a/src/Functions/getServerPort.cpp +++ b/src/Functions/getServerPort.cpp @@ -128,7 +128,7 @@ public: } -void registerFunctionGetServerPort(FunctionFactory & factory) +REGISTER_FUNCTION(GetServerPort) { factory.registerFunction(); } diff --git a/src/Functions/getSetting.cpp b/src/Functions/getSetting.cpp index a27f698d54c..e7ea44aab28 100644 --- a/src/Functions/getSetting.cpp +++ b/src/Functions/getSetting.cpp @@ -63,7 +63,7 @@ private: } -void registerFunctionGetSetting(FunctionFactory & factory) +REGISTER_FUNCTION(GetSetting) { factory.registerFunction(); } diff --git a/src/Functions/getSizeOfEnumType.cpp b/src/Functions/getSizeOfEnumType.cpp index 17436de02f7..6128f5a44cb 100644 --- a/src/Functions/getSizeOfEnumType.cpp +++ b/src/Functions/getSizeOfEnumType.cpp @@ -76,7 +76,7 @@ private: } -void registerFunctionGetSizeOfEnumType(FunctionFactory & factory) +REGISTER_FUNCTION(GetSizeOfEnumType) { factory.registerFunction(); } diff --git a/src/Functions/getTypeSerializationStreams.cpp b/src/Functions/getTypeSerializationStreams.cpp index 3306c91dd0a..2b13f0f140d 100644 --- a/src/Functions/getTypeSerializationStreams.cpp +++ b/src/Functions/getTypeSerializationStreams.cpp @@ -79,7 +79,7 @@ private: } -void registerFunctionGetTypeSerializationStreams(FunctionFactory & factory) +REGISTER_FUNCTION(GetTypeSerializationStreams) { factory.registerFunction(); } diff --git a/src/Functions/globalVariable.cpp b/src/Functions/globalVariable.cpp index 581ddbea371..50e94fad614 100644 --- a/src/Functions/globalVariable.cpp +++ b/src/Functions/globalVariable.cpp @@ -88,7 +88,7 @@ private: } -void registerFunctionGlobalVariable(FunctionFactory & factory) +REGISTER_FUNCTION(GlobalVariable) { factory.registerFunction(); } diff --git a/src/Functions/greatCircleDistance.cpp b/src/Functions/greatCircleDistance.cpp index ab8b37af960..5c1923da908 100644 --- a/src/Functions/greatCircleDistance.cpp +++ b/src/Functions/greatCircleDistance.cpp @@ -344,7 +344,7 @@ private: ImplementationSelector selector; }; -void registerFunctionGeoDistance(FunctionFactory & factory) +REGISTER_FUNCTION(GeoDistance) { geodistInit(); factory.registerFunction>(); diff --git a/src/Functions/greater.cpp b/src/Functions/greater.cpp index 1a0e999717d..c36f8d7acca 100644 --- a/src/Functions/greater.cpp +++ b/src/Functions/greater.cpp @@ -7,7 +7,7 @@ namespace DB using FunctionGreater = FunctionComparison; -void registerFunctionGreater(FunctionFactory & factory) +REGISTER_FUNCTION(Greater) { factory.registerFunction(); } diff --git a/src/Functions/greaterOrEquals.cpp b/src/Functions/greaterOrEquals.cpp index 4412faa6412..089ac4d5466 100644 --- a/src/Functions/greaterOrEquals.cpp +++ b/src/Functions/greaterOrEquals.cpp @@ -7,7 +7,7 @@ namespace DB using FunctionGreaterOrEquals = FunctionComparison; -void registerFunctionGreaterOrEquals(FunctionFactory & factory) +REGISTER_FUNCTION(GreaterOrEquals) { factory.registerFunction(); } diff --git a/src/Functions/greatest.cpp b/src/Functions/greatest.cpp index 75837a409e9..cac02eea1be 100644 --- a/src/Functions/greatest.cpp +++ b/src/Functions/greatest.cpp @@ -63,7 +63,7 @@ using GreatestImpl = std::conditional_t; -void registerFunctionGreatest(FunctionFactory & factory) +REGISTER_FUNCTION(Greatest) { factory.registerFunction>(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/h3CellAreaM2.cpp b/src/Functions/h3CellAreaM2.cpp index e5fb5aa2ed7..10fac6c9441 100644 --- a/src/Functions/h3CellAreaM2.cpp +++ b/src/Functions/h3CellAreaM2.cpp @@ -91,7 +91,7 @@ public: } -void registerFunctionH3CellAreaM2(FunctionFactory & factory) +REGISTER_FUNCTION(H3CellAreaM2) { factory.registerFunction(); } diff --git a/src/Functions/h3CellAreaRads2.cpp b/src/Functions/h3CellAreaRads2.cpp index 15d18613b72..c74944e4e2f 100644 --- a/src/Functions/h3CellAreaRads2.cpp +++ b/src/Functions/h3CellAreaRads2.cpp @@ -90,7 +90,7 @@ public: } -void registerFunctionH3CellAreaRads2(FunctionFactory & factory) +REGISTER_FUNCTION(H3CellAreaRads2) { factory.registerFunction(); } diff --git a/src/Functions/h3Distance.cpp b/src/Functions/h3Distance.cpp index b6206a1922f..d4291c30424 100644 --- a/src/Functions/h3Distance.cpp +++ b/src/Functions/h3Distance.cpp @@ -107,7 +107,7 @@ public: } -void registerFunctionH3Distance(FunctionFactory & factory) +REGISTER_FUNCTION(H3Distance) { factory.registerFunction(); } diff --git a/src/Functions/h3EdgeAngle.cpp b/src/Functions/h3EdgeAngle.cpp index 8240f092ee8..f80bfd1cdf9 100644 --- a/src/Functions/h3EdgeAngle.cpp +++ b/src/Functions/h3EdgeAngle.cpp @@ -94,7 +94,7 @@ public: } -void registerFunctionH3EdgeAngle(FunctionFactory & factory) +REGISTER_FUNCTION(H3EdgeAngle) { factory.registerFunction(); } diff --git a/src/Functions/h3EdgeLengthKm.cpp b/src/Functions/h3EdgeLengthKm.cpp index b7072a2f309..ca2b050b485 100644 --- a/src/Functions/h3EdgeLengthKm.cpp +++ b/src/Functions/h3EdgeLengthKm.cpp @@ -92,7 +92,7 @@ public: } -void registerFunctionH3EdgeLengthKm(FunctionFactory & factory) +REGISTER_FUNCTION(H3EdgeLengthKm) { factory.registerFunction(); } diff --git a/src/Functions/h3EdgeLengthM.cpp b/src/Functions/h3EdgeLengthM.cpp index 2c934bc6c05..a2786da51f1 100644 --- a/src/Functions/h3EdgeLengthM.cpp +++ b/src/Functions/h3EdgeLengthM.cpp @@ -97,7 +97,7 @@ public: } -void registerFunctionH3EdgeLengthM(FunctionFactory & factory) +REGISTER_FUNCTION(H3EdgeLengthM) { factory.registerFunction(); } diff --git a/src/Functions/h3ExactEdgeLengthKm.cpp b/src/Functions/h3ExactEdgeLengthKm.cpp index f37d93fd715..1cd43c98576 100644 --- a/src/Functions/h3ExactEdgeLengthKm.cpp +++ b/src/Functions/h3ExactEdgeLengthKm.cpp @@ -84,7 +84,7 @@ public: } -void registerFunctionH3ExactEdgeLengthKm(FunctionFactory & factory) +REGISTER_FUNCTION(H3ExactEdgeLengthKm) { factory.registerFunction(); } diff --git a/src/Functions/h3ExactEdgeLengthM.cpp b/src/Functions/h3ExactEdgeLengthM.cpp index 99acbb757c3..1b930a592db 100644 --- a/src/Functions/h3ExactEdgeLengthM.cpp +++ b/src/Functions/h3ExactEdgeLengthM.cpp @@ -84,7 +84,7 @@ public: } -void registerFunctionH3ExactEdgeLengthM(FunctionFactory & factory) +REGISTER_FUNCTION(H3ExactEdgeLengthM) { factory.registerFunction(); } diff --git a/src/Functions/h3ExactEdgeLengthRads.cpp b/src/Functions/h3ExactEdgeLengthRads.cpp index a2937e85c65..62b9d916cdf 100644 --- a/src/Functions/h3ExactEdgeLengthRads.cpp +++ b/src/Functions/h3ExactEdgeLengthRads.cpp @@ -84,7 +84,7 @@ public: } -void registerFunctionH3ExactEdgeLengthRads(FunctionFactory & factory) +REGISTER_FUNCTION(H3ExactEdgeLengthRads) { factory.registerFunction(); } diff --git a/src/Functions/h3GetBaseCell.cpp b/src/Functions/h3GetBaseCell.cpp index a3023561824..a0cd75c86a9 100644 --- a/src/Functions/h3GetBaseCell.cpp +++ b/src/Functions/h3GetBaseCell.cpp @@ -84,7 +84,7 @@ public: } -void registerFunctionH3GetBaseCell(FunctionFactory & factory) +REGISTER_FUNCTION(H3GetBaseCell) { factory.registerFunction(); } diff --git a/src/Functions/h3GetDestinationIndexFromUnidirectionalEdge.cpp b/src/Functions/h3GetDestinationIndexFromUnidirectionalEdge.cpp index 4a4a36076b2..38a97ac8be0 100644 --- a/src/Functions/h3GetDestinationIndexFromUnidirectionalEdge.cpp +++ b/src/Functions/h3GetDestinationIndexFromUnidirectionalEdge.cpp @@ -85,7 +85,7 @@ public: } -void registerFunctionH3GetDestinationIndexFromUnidirectionalEdge(FunctionFactory & factory) +REGISTER_FUNCTION(H3GetDestinationIndexFromUnidirectionalEdge) { factory.registerFunction(); } diff --git a/src/Functions/h3GetFaces.cpp b/src/Functions/h3GetFaces.cpp index 0344ccc7944..42f430fb2ab 100644 --- a/src/Functions/h3GetFaces.cpp +++ b/src/Functions/h3GetFaces.cpp @@ -106,7 +106,7 @@ public: } -void registerFunctionH3GetFaces(FunctionFactory & factory) +REGISTER_FUNCTION(H3GetFaces) { factory.registerFunction(); } diff --git a/src/Functions/h3GetIndexesFromUnidirectionalEdge.cpp b/src/Functions/h3GetIndexesFromUnidirectionalEdge.cpp index 91ab22aed29..3d98a6374c6 100644 --- a/src/Functions/h3GetIndexesFromUnidirectionalEdge.cpp +++ b/src/Functions/h3GetIndexesFromUnidirectionalEdge.cpp @@ -100,7 +100,7 @@ public: } -void registerFunctionH3GetIndexesFromUnidirectionalEdge(FunctionFactory & factory) +REGISTER_FUNCTION(H3GetIndexesFromUnidirectionalEdge) { factory.registerFunction(); } diff --git a/src/Functions/h3GetOriginIndexFromUnidirectionalEdge.cpp b/src/Functions/h3GetOriginIndexFromUnidirectionalEdge.cpp index acf94fcf95f..cbe69a4e887 100644 --- a/src/Functions/h3GetOriginIndexFromUnidirectionalEdge.cpp +++ b/src/Functions/h3GetOriginIndexFromUnidirectionalEdge.cpp @@ -85,7 +85,7 @@ public: } -void registerFunctionH3GetOriginIndexFromUnidirectionalEdge(FunctionFactory & factory) +REGISTER_FUNCTION(H3GetOriginIndexFromUnidirectionalEdge) { factory.registerFunction(); } diff --git a/src/Functions/h3GetPentagonIndexes.cpp b/src/Functions/h3GetPentagonIndexes.cpp index bc90187bb3a..b24b58a2568 100644 --- a/src/Functions/h3GetPentagonIndexes.cpp +++ b/src/Functions/h3GetPentagonIndexes.cpp @@ -108,7 +108,7 @@ public: } -void registerFunctionH3GetPentagonIndexes(FunctionFactory & factory) +REGISTER_FUNCTION(H3GetPentagonIndexes) { factory.registerFunction(); } diff --git a/src/Functions/h3GetRes0Indexes.cpp b/src/Functions/h3GetRes0Indexes.cpp index 9be55c0bb62..0db89752fa0 100644 --- a/src/Functions/h3GetRes0Indexes.cpp +++ b/src/Functions/h3GetRes0Indexes.cpp @@ -59,7 +59,7 @@ public: } -void registerFunctionH3GetRes0Indexes(FunctionFactory & factory) +REGISTER_FUNCTION(H3GetRes0Indexes) { factory.registerFunction(); } diff --git a/src/Functions/h3GetResolution.cpp b/src/Functions/h3GetResolution.cpp index 4d0a92a1414..153cf883b19 100644 --- a/src/Functions/h3GetResolution.cpp +++ b/src/Functions/h3GetResolution.cpp @@ -84,7 +84,7 @@ public: } -void registerFunctionH3GetResolution(FunctionFactory & factory) +REGISTER_FUNCTION(H3GetResolution) { factory.registerFunction(); } diff --git a/src/Functions/h3GetUnidirectionalEdge.cpp b/src/Functions/h3GetUnidirectionalEdge.cpp index 7d4122150e6..eddc35bd45a 100644 --- a/src/Functions/h3GetUnidirectionalEdge.cpp +++ b/src/Functions/h3GetUnidirectionalEdge.cpp @@ -117,7 +117,7 @@ public: } -void registerFunctionH3GetUnidirectionalEdge(FunctionFactory & factory) +REGISTER_FUNCTION(H3GetUnidirectionalEdge) { factory.registerFunction(); } diff --git a/src/Functions/h3GetUnidirectionalEdgeBoundary.cpp b/src/Functions/h3GetUnidirectionalEdgeBoundary.cpp index 4cbdda2cb70..bf2e904c473 100644 --- a/src/Functions/h3GetUnidirectionalEdgeBoundary.cpp +++ b/src/Functions/h3GetUnidirectionalEdgeBoundary.cpp @@ -101,7 +101,7 @@ public: } -void registerFunctionH3GetUnidirectionalEdgeBoundary(FunctionFactory & factory) +REGISTER_FUNCTION(H3GetUnidirectionalEdgeBoundary) { factory.registerFunction(); } diff --git a/src/Functions/h3GetUnidirectionalEdgesFromHexagon.cpp b/src/Functions/h3GetUnidirectionalEdgesFromHexagon.cpp index 10e0a5d8cb2..e31359c297d 100644 --- a/src/Functions/h3GetUnidirectionalEdgesFromHexagon.cpp +++ b/src/Functions/h3GetUnidirectionalEdgesFromHexagon.cpp @@ -102,7 +102,7 @@ public: } -void registerFunctionH3GetUnidirectionalEdgesFromHexagon(FunctionFactory & factory) +REGISTER_FUNCTION(H3GetUnidirectionalEdgesFromHexagon) { factory.registerFunction(); } diff --git a/src/Functions/h3HexAreaKm2.cpp b/src/Functions/h3HexAreaKm2.cpp index c4a3c1258ab..e29b66f7538 100644 --- a/src/Functions/h3HexAreaKm2.cpp +++ b/src/Functions/h3HexAreaKm2.cpp @@ -93,7 +93,7 @@ public: } -void registerFunctionH3HexAreaKm2(FunctionFactory & factory) +REGISTER_FUNCTION(H3HexAreaKm2) { factory.registerFunction(); } diff --git a/src/Functions/h3HexAreaM2.cpp b/src/Functions/h3HexAreaM2.cpp index f8599651b0f..eb90e5daa2e 100644 --- a/src/Functions/h3HexAreaM2.cpp +++ b/src/Functions/h3HexAreaM2.cpp @@ -94,7 +94,7 @@ public: } -void registerFunctionH3HexAreaM2(FunctionFactory & factory) +REGISTER_FUNCTION(H3HexAreaM2) { factory.registerFunction(); } diff --git a/src/Functions/h3HexRing.cpp b/src/Functions/h3HexRing.cpp index cc3acf9d7a5..25cde81e061 100644 --- a/src/Functions/h3HexRing.cpp +++ b/src/Functions/h3HexRing.cpp @@ -143,7 +143,7 @@ public: } -void registerFunctionH3HexRing(FunctionFactory & factory) +REGISTER_FUNCTION(H3HexRing) { factory.registerFunction(); } diff --git a/src/Functions/h3IndexesAreNeighbors.cpp b/src/Functions/h3IndexesAreNeighbors.cpp index ee603f7de49..82a05a02f0d 100644 --- a/src/Functions/h3IndexesAreNeighbors.cpp +++ b/src/Functions/h3IndexesAreNeighbors.cpp @@ -103,7 +103,7 @@ public: } -void registerFunctionH3IndexesAreNeighbors(FunctionFactory & factory) +REGISTER_FUNCTION(H3IndexesAreNeighbors) { factory.registerFunction(); } diff --git a/src/Functions/h3IsPentagon.cpp b/src/Functions/h3IsPentagon.cpp index b7374bc1d8d..048a5ca50ce 100644 --- a/src/Functions/h3IsPentagon.cpp +++ b/src/Functions/h3IsPentagon.cpp @@ -80,7 +80,7 @@ public: } -void registerFunctionH3IsPentagon(FunctionFactory & factory) +REGISTER_FUNCTION(H3IsPentagon) { factory.registerFunction(); } diff --git a/src/Functions/h3IsResClassIII.cpp b/src/Functions/h3IsResClassIII.cpp index 08025c966cd..23a11f1a544 100644 --- a/src/Functions/h3IsResClassIII.cpp +++ b/src/Functions/h3IsResClassIII.cpp @@ -80,7 +80,7 @@ public: } -void registerFunctionH3IsResClassIII(FunctionFactory & factory) +REGISTER_FUNCTION(H3IsResClassIII) { factory.registerFunction(); } diff --git a/src/Functions/h3IsValid.cpp b/src/Functions/h3IsValid.cpp index b6701d89de6..6b7b47bc8d4 100644 --- a/src/Functions/h3IsValid.cpp +++ b/src/Functions/h3IsValid.cpp @@ -83,7 +83,7 @@ public: } -void registerFunctionH3IsValid(FunctionFactory & factory) +REGISTER_FUNCTION(H3IsValid) { factory.registerFunction(); } diff --git a/src/Functions/h3Line.cpp b/src/Functions/h3Line.cpp index 2e92d65b0b9..d596c6ec956 100644 --- a/src/Functions/h3Line.cpp +++ b/src/Functions/h3Line.cpp @@ -133,7 +133,7 @@ public: } -void registerFunctionH3Line(FunctionFactory & factory) +REGISTER_FUNCTION(H3Line) { factory.registerFunction(); } diff --git a/src/Functions/h3NumHexagons.cpp b/src/Functions/h3NumHexagons.cpp index 009ff182940..3a13071d6cf 100644 --- a/src/Functions/h3NumHexagons.cpp +++ b/src/Functions/h3NumHexagons.cpp @@ -89,7 +89,7 @@ public: } -void registerFunctionH3NumHexagons(FunctionFactory & factory) +REGISTER_FUNCTION(H3NumHexagons) { factory.registerFunction(); } diff --git a/src/Functions/h3PointDist.cpp b/src/Functions/h3PointDist.cpp index 463050c9b81..2d3512f0192 100644 --- a/src/Functions/h3PointDist.cpp +++ b/src/Functions/h3PointDist.cpp @@ -144,9 +144,9 @@ struct H3PointDistRads }; -void registerFunctionH3PointDistM(FunctionFactory & factory) { factory.registerFunction>(); } -void registerFunctionH3PointDistKm(FunctionFactory & factory) { factory.registerFunction>(); } -void registerFunctionH3PointDistRads(FunctionFactory & factory) { factory.registerFunction>(); } +REGISTER_FUNCTION(H3PointDistM) { factory.registerFunction>(); } +REGISTER_FUNCTION(H3PointDistKm) { factory.registerFunction>(); } +REGISTER_FUNCTION(H3PointDistRads) { factory.registerFunction>(); } } diff --git a/src/Functions/h3ToCenterChild.cpp b/src/Functions/h3ToCenterChild.cpp index 6b9724c297c..6104d179d30 100644 --- a/src/Functions/h3ToCenterChild.cpp +++ b/src/Functions/h3ToCenterChild.cpp @@ -109,7 +109,7 @@ namespace } -void registerFunctionH3ToCenterChild(FunctionFactory & factory) +REGISTER_FUNCTION(H3ToCenterChild) { factory.registerFunction(); } diff --git a/src/Functions/h3ToChildren.cpp b/src/Functions/h3ToChildren.cpp index 26d3e3dc204..f18d96c6a90 100644 --- a/src/Functions/h3ToChildren.cpp +++ b/src/Functions/h3ToChildren.cpp @@ -139,7 +139,7 @@ public: } -void registerFunctionH3ToChildren(FunctionFactory & factory) +REGISTER_FUNCTION(H3ToChildren) { factory.registerFunction(); } diff --git a/src/Functions/h3ToGeoBoundary.cpp b/src/Functions/h3ToGeoBoundary.cpp index acf1ec64020..dedb195f79c 100644 --- a/src/Functions/h3ToGeoBoundary.cpp +++ b/src/Functions/h3ToGeoBoundary.cpp @@ -98,7 +98,7 @@ public: } }; -void registerFunctionH3ToGeoBoundary(FunctionFactory & factory) +REGISTER_FUNCTION(H3ToGeoBoundary) { factory.registerFunction(); } diff --git a/src/Functions/h3ToParent.cpp b/src/Functions/h3ToParent.cpp index 8fd9931b81b..d7678004125 100644 --- a/src/Functions/h3ToParent.cpp +++ b/src/Functions/h3ToParent.cpp @@ -114,7 +114,7 @@ public: } -void registerFunctionH3ToParent(FunctionFactory & factory) +REGISTER_FUNCTION(H3ToParent) { factory.registerFunction(); } diff --git a/src/Functions/h3ToString.cpp b/src/Functions/h3ToString.cpp index 7ca46c3b064..9a6b1504af0 100644 --- a/src/Functions/h3ToString.cpp +++ b/src/Functions/h3ToString.cpp @@ -101,7 +101,7 @@ public: } -void registerFunctionH3ToString(FunctionFactory & factory) +REGISTER_FUNCTION(H3ToString) { factory.registerFunction(); } diff --git a/src/Functions/h3UnidirectionalEdgeIsValid.cpp b/src/Functions/h3UnidirectionalEdgeIsValid.cpp index 6b00fba3c5a..012a14823c9 100644 --- a/src/Functions/h3UnidirectionalEdgeIsValid.cpp +++ b/src/Functions/h3UnidirectionalEdgeIsValid.cpp @@ -87,7 +87,7 @@ public: } -void registerFunctionH3UnidirectionalEdgeIsValid(FunctionFactory & factory) +REGISTER_FUNCTION(H3UnidirectionalEdgeIsValid) { factory.registerFunction(); } diff --git a/src/Functions/h3kRing.cpp b/src/Functions/h3kRing.cpp index a68f2a5e23d..f2d50532e61 100644 --- a/src/Functions/h3kRing.cpp +++ b/src/Functions/h3kRing.cpp @@ -137,7 +137,7 @@ public: } -void registerFunctionH3KRing(FunctionFactory & factory) +REGISTER_FUNCTION(H3KRing) { factory.registerFunction(); } diff --git a/src/Functions/h3toGeo.cpp b/src/Functions/h3toGeo.cpp index 52c9b1217ca..aff55324e48 100644 --- a/src/Functions/h3toGeo.cpp +++ b/src/Functions/h3toGeo.cpp @@ -100,7 +100,7 @@ public: } -void registerFunctionH3ToGeo(FunctionFactory & factory) +REGISTER_FUNCTION(H3ToGeo) { factory.registerFunction(); } diff --git a/src/Functions/hasColumnInTable.cpp b/src/Functions/hasColumnInTable.cpp index a70bf57f7c2..4c3008dd516 100644 --- a/src/Functions/hasColumnInTable.cpp +++ b/src/Functions/hasColumnInTable.cpp @@ -155,7 +155,7 @@ ColumnPtr FunctionHasColumnInTable::executeImpl(const ColumnsWithTypeAndName & a } -void registerFunctionHasColumnInTable(FunctionFactory & factory) +REGISTER_FUNCTION(HasColumnInTable) { factory.registerFunction(); } diff --git a/src/Functions/hasThreadFuzzer.cpp b/src/Functions/hasThreadFuzzer.cpp index 7ee0b3ee687..a0b1b13b580 100644 --- a/src/Functions/hasThreadFuzzer.cpp +++ b/src/Functions/hasThreadFuzzer.cpp @@ -46,7 +46,7 @@ public: } -void registerFunctionHasThreadFuzzer(FunctionFactory & factory) +REGISTER_FUNCTION(HasThreadFuzzer) { factory.registerFunction(); } diff --git a/src/Functions/hasToken.cpp b/src/Functions/hasToken.cpp index 72d6c11a5fa..646ff0b54f7 100644 --- a/src/Functions/hasToken.cpp +++ b/src/Functions/hasToken.cpp @@ -18,7 +18,7 @@ using FunctionHasToken = FunctionsStringSearch(); } diff --git a/src/Functions/hasTokenCaseInsensitive.cpp b/src/Functions/hasTokenCaseInsensitive.cpp index a0280bc12a5..0012ea3e148 100644 --- a/src/Functions/hasTokenCaseInsensitive.cpp +++ b/src/Functions/hasTokenCaseInsensitive.cpp @@ -19,7 +19,7 @@ using FunctionHasTokenCaseInsensitive } -void registerFunctionHasTokenCaseInsensitive(FunctionFactory & factory) +REGISTER_FUNCTION(HasTokenCaseInsensitive) { factory.registerFunction(); } diff --git a/src/Functions/hypot.cpp b/src/Functions/hypot.cpp index 00297713d11..4963e0262e4 100644 --- a/src/Functions/hypot.cpp +++ b/src/Functions/hypot.cpp @@ -13,7 +13,7 @@ namespace } -void registerFunctionHypot(FunctionFactory & factory) +REGISTER_FUNCTION(Hypot) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/identity.cpp b/src/Functions/identity.cpp index b3d05226bda..8a29eafca52 100644 --- a/src/Functions/identity.cpp +++ b/src/Functions/identity.cpp @@ -34,7 +34,7 @@ public: } -void registerFunctionIdentity(FunctionFactory & factory) +REGISTER_FUNCTION(Identity) { factory.registerFunction(); } diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp index d4c2fcdd779..d7fefb1ad0e 100644 --- a/src/Functions/if.cpp +++ b/src/Functions/if.cpp @@ -1120,7 +1120,7 @@ public: } -void registerFunctionIf(FunctionFactory & factory) +REGISTER_FUNCTION(If) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/ifNotFinite.cpp b/src/Functions/ifNotFinite.cpp index 49f74c2f5d0..5ce5d0ede70 100644 --- a/src/Functions/ifNotFinite.cpp +++ b/src/Functions/ifNotFinite.cpp @@ -65,7 +65,7 @@ private: } -void registerFunctionIfNotFinite(FunctionFactory & factory) +REGISTER_FUNCTION(IfNotFinite) { factory.registerFunction(); } diff --git a/src/Functions/ifNull.cpp b/src/Functions/ifNull.cpp index ab8e2677d28..a586a695752 100644 --- a/src/Functions/ifNull.cpp +++ b/src/Functions/ifNull.cpp @@ -89,7 +89,7 @@ private: } -void registerFunctionIfNull(FunctionFactory & factory) +REGISTER_FUNCTION(IfNull) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/ignore.cpp b/src/Functions/ignore.cpp index 77c16cf7819..87dd0ab1cbc 100644 --- a/src/Functions/ignore.cpp +++ b/src/Functions/ignore.cpp @@ -56,7 +56,7 @@ public: } -void registerFunctionIgnore(FunctionFactory & factory) +REGISTER_FUNCTION(Ignore) { factory.registerFunction(); } diff --git a/src/Functions/ilike.cpp b/src/Functions/ilike.cpp index b88d01986d5..c9f1fb37c14 100644 --- a/src/Functions/ilike.cpp +++ b/src/Functions/ilike.cpp @@ -17,7 +17,7 @@ using FunctionILike = FunctionsStringSearch; } -void registerFunctionILike(FunctionFactory & factory) +REGISTER_FUNCTION(ILike) { factory.registerFunction(); } diff --git a/src/Functions/in.cpp b/src/Functions/in.cpp index 469b98ed00d..5773e823a80 100644 --- a/src/Functions/in.cpp +++ b/src/Functions/in.cpp @@ -174,7 +174,7 @@ void registerFunctionsInImpl(FunctionFactory & factory) } -void registerFunctionsIn(FunctionFactory & factory) +REGISTER_FUNCTION(In) { registerFunctionsInImpl(factory); registerFunctionsInImpl(factory); diff --git a/src/Functions/indexHint.cpp b/src/Functions/indexHint.cpp index bb38a56cf27..fc402fa3be6 100644 --- a/src/Functions/indexHint.cpp +++ b/src/Functions/indexHint.cpp @@ -60,7 +60,7 @@ public: }; -void registerFunctionIndexHint(FunctionFactory & factory) +REGISTER_FUNCTION(IndexHint) { factory.registerFunction(); } diff --git a/src/Functions/initialQueryID.cpp b/src/Functions/initialQueryID.cpp index d032bde24e3..469f37cf614 100644 --- a/src/Functions/initialQueryID.cpp +++ b/src/Functions/initialQueryID.cpp @@ -38,7 +38,7 @@ public: } }; -void registerFunctionInitialQueryID(FunctionFactory & factory) +REGISTER_FUNCTION(InitialQueryID) { factory.registerFunction(); factory.registerAlias("initial_query_id", FunctionInitialQueryID::name, FunctionFactory::CaseInsensitive); diff --git a/src/Functions/initializeAggregation.cpp b/src/Functions/initializeAggregation.cpp index 8843b1edc8d..b7dcce9c188 100644 --- a/src/Functions/initializeAggregation.cpp +++ b/src/Functions/initializeAggregation.cpp @@ -160,7 +160,7 @@ ColumnPtr FunctionInitializeAggregation::executeImpl(const ColumnsWithTypeAndNam } -void registerFunctionInitializeAggregation(FunctionFactory & factory) +REGISTER_FUNCTION(InitializeAggregation) { factory.registerFunction(); } diff --git a/src/Functions/intDiv.cpp b/src/Functions/intDiv.cpp index f2713bb1adb..89a5fe2fbf2 100644 --- a/src/Functions/intDiv.cpp +++ b/src/Functions/intDiv.cpp @@ -126,7 +126,7 @@ template <> struct BinaryOperationImpl; -void registerFunctionIntDiv(FunctionFactory & factory) +REGISTER_FUNCTION(IntDiv) { factory.registerFunction(); } diff --git a/src/Functions/intDivOrZero.cpp b/src/Functions/intDivOrZero.cpp index d1b3429abfb..96ff6ea80fc 100644 --- a/src/Functions/intDivOrZero.cpp +++ b/src/Functions/intDivOrZero.cpp @@ -29,7 +29,7 @@ struct DivideIntegralOrZeroImpl struct NameIntDivOrZero { static constexpr auto name = "intDivOrZero"; }; using FunctionIntDivOrZero = BinaryArithmeticOverloadResolver; -void registerFunctionIntDivOrZero(FunctionFactory & factory) +REGISTER_FUNCTION(IntDivOrZero) { factory.registerFunction(); } diff --git a/src/Functions/intExp10.cpp b/src/Functions/intExp10.cpp index abaa17c862b..3b5a2c2ac88 100644 --- a/src/Functions/intExp10.cpp +++ b/src/Functions/intExp10.cpp @@ -59,7 +59,7 @@ template <> struct FunctionUnaryArithmeticMonotonicity } }; -void registerFunctionIntExp10(FunctionFactory & factory) +REGISTER_FUNCTION(IntExp10) { factory.registerFunction(); } diff --git a/src/Functions/intExp2.cpp b/src/Functions/intExp2.cpp index b2c7e80ae4b..ae2a66dda78 100644 --- a/src/Functions/intExp2.cpp +++ b/src/Functions/intExp2.cpp @@ -62,7 +62,7 @@ template <> struct FunctionUnaryArithmeticMonotonicity } }; -void registerFunctionIntExp2(FunctionFactory & factory) +REGISTER_FUNCTION(IntExp2) { factory.registerFunction(); } diff --git a/src/Functions/isConstant.cpp b/src/Functions/isConstant.cpp index 09b29aaf260..c203674e7f9 100644 --- a/src/Functions/isConstant.cpp +++ b/src/Functions/isConstant.cpp @@ -52,7 +52,7 @@ public: } -void registerFunctionIsConstant(FunctionFactory & factory) +REGISTER_FUNCTION(IsConstant) { factory.registerFunction(); } diff --git a/src/Functions/isDecimalOverflow.cpp b/src/Functions/isDecimalOverflow.cpp index 9962d4b7314..fcba8c1f64b 100644 --- a/src/Functions/isDecimalOverflow.cpp +++ b/src/Functions/isDecimalOverflow.cpp @@ -146,7 +146,7 @@ private: } -void registerFunctionIsDecimalOverflow(FunctionFactory & factory) +REGISTER_FUNCTION(IsDecimalOverflow) { factory.registerFunction(); } diff --git a/src/Functions/isFinite.cpp b/src/Functions/isFinite.cpp index b1ff5f326e5..90185b64fff 100644 --- a/src/Functions/isFinite.cpp +++ b/src/Functions/isFinite.cpp @@ -39,7 +39,7 @@ using FunctionIsFinite = FunctionNumericPredicate; } -void registerFunctionIsFinite(FunctionFactory & factory) +REGISTER_FUNCTION(IsFinite) { factory.registerFunction(); } diff --git a/src/Functions/isIPAddressContainedIn.cpp b/src/Functions/isIPAddressContainedIn.cpp index a6f94c77ad1..15dcdc584d8 100644 --- a/src/Functions/isIPAddressContainedIn.cpp +++ b/src/Functions/isIPAddressContainedIn.cpp @@ -252,7 +252,7 @@ namespace DB } }; - void registerFunctionIsIPAddressContainedIn(FunctionFactory & factory) + REGISTER_FUNCTION(IsIPAddressContainedIn) { factory.registerFunction(); } diff --git a/src/Functions/isInfinite.cpp b/src/Functions/isInfinite.cpp index 1fbc134d0c9..e923e1461bc 100644 --- a/src/Functions/isInfinite.cpp +++ b/src/Functions/isInfinite.cpp @@ -35,7 +35,7 @@ using FunctionIsInfinite = FunctionNumericPredicate; } -void registerFunctionIsInfinite(FunctionFactory & factory) +REGISTER_FUNCTION(IsInfinite) { factory.registerFunction(); } diff --git a/src/Functions/isNaN.cpp b/src/Functions/isNaN.cpp index 2e35e8cfbb5..e6ab425a4d3 100644 --- a/src/Functions/isNaN.cpp +++ b/src/Functions/isNaN.cpp @@ -22,7 +22,7 @@ using FunctionIsNaN = FunctionNumericPredicate; } -void registerFunctionIsNaN(FunctionFactory & factory) +REGISTER_FUNCTION(IsNaN) { factory.registerFunction(); } diff --git a/src/Functions/isNotNull.cpp b/src/Functions/isNotNull.cpp index 44ea8aeaeb0..cbdc08c2fab 100644 --- a/src/Functions/isNotNull.cpp +++ b/src/Functions/isNotNull.cpp @@ -79,7 +79,7 @@ public: } -void registerFunctionIsNotNull(FunctionFactory & factory) +REGISTER_FUNCTION(IsNotNull) { factory.registerFunction(); } diff --git a/src/Functions/isNull.cpp b/src/Functions/isNull.cpp index e22b1cf469c..1e1d4edd6ed 100644 --- a/src/Functions/isNull.cpp +++ b/src/Functions/isNull.cpp @@ -72,7 +72,7 @@ public: } -void registerFunctionIsNull(FunctionFactory & factory) +REGISTER_FUNCTION(IsNull) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/isNullable.cpp b/src/Functions/isNullable.cpp index 3680ac7ccb0..14874487f40 100644 --- a/src/Functions/isNullable.cpp +++ b/src/Functions/isNullable.cpp @@ -54,7 +54,7 @@ public: } -void registerFunctionIsNullable(FunctionFactory & factory) +REGISTER_FUNCTION(IsNullable) { factory.registerFunction(); } diff --git a/src/Functions/isValidUTF8.cpp b/src/Functions/isValidUTF8.cpp index 0168a36b5b5..3ecf69cc364 100644 --- a/src/Functions/isValidUTF8.cpp +++ b/src/Functions/isValidUTF8.cpp @@ -256,7 +256,7 @@ struct NameIsValidUTF8 }; using FunctionValidUTF8 = FunctionStringOrArrayToT; -void registerFunctionIsValidUTF8(FunctionFactory & factory) +REGISTER_FUNCTION(IsValidUTF8) { factory.registerFunction(); } diff --git a/src/Functions/isZeroOrNull.cpp b/src/Functions/isZeroOrNull.cpp index 1c3b6889f16..021e4a842a6 100644 --- a/src/Functions/isZeroOrNull.cpp +++ b/src/Functions/isZeroOrNull.cpp @@ -115,7 +115,7 @@ private: } -void registerFunctionIsZeroOrNull(FunctionFactory & factory) +REGISTER_FUNCTION(IsZeroOrNull) { factory.registerFunction(); } diff --git a/src/Functions/jumpConsistentHash.cpp b/src/Functions/jumpConsistentHash.cpp index bc41913f7e7..ffc21eb5cea 100644 --- a/src/Functions/jumpConsistentHash.cpp +++ b/src/Functions/jumpConsistentHash.cpp @@ -39,7 +39,7 @@ using FunctionJumpConsistentHash = FunctionConsistentHashImpl(); } diff --git a/src/Functions/kostikConsistentHash.cpp b/src/Functions/kostikConsistentHash.cpp index a38c3c965d8..47a9a928976 100644 --- a/src/Functions/kostikConsistentHash.cpp +++ b/src/Functions/kostikConsistentHash.cpp @@ -25,7 +25,7 @@ struct KostikConsistentHashImpl using FunctionKostikConsistentHash = FunctionConsistentHashImpl; -void registerFunctionKostikConsistentHash(FunctionFactory & factory) +REGISTER_FUNCTION(KostikConsistentHash) { factory.registerFunction(); factory.registerAlias("yandexConsistentHash", "kostikConsistentHash"); diff --git a/src/Functions/lcm.cpp b/src/Functions/lcm.cpp index 51a1bf59d1b..495199fa964 100644 --- a/src/Functions/lcm.cpp +++ b/src/Functions/lcm.cpp @@ -61,7 +61,7 @@ using FunctionLCM = BinaryArithmeticOverloadResolver(); } diff --git a/src/Functions/least.cpp b/src/Functions/least.cpp index b7fe2b302ff..53676f0c00d 100644 --- a/src/Functions/least.cpp +++ b/src/Functions/least.cpp @@ -63,7 +63,7 @@ using LeastImpl = std::conditional_t; -void registerFunctionLeast(FunctionFactory & factory) +REGISTER_FUNCTION(Least) { factory.registerFunction>(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/left.cpp b/src/Functions/left.cpp index aa7a2cdd5a8..93983d698ce 100644 --- a/src/Functions/left.cpp +++ b/src/Functions/left.cpp @@ -4,7 +4,7 @@ namespace DB { -void registerFunctionLeft(FunctionFactory & factory) +REGISTER_FUNCTION(Left) { factory.registerFunction>(FunctionFactory::CaseInsensitive); factory.registerFunction>(FunctionFactory::CaseSensitive); diff --git a/src/Functions/lemmatize.cpp b/src/Functions/lemmatize.cpp index 78f9ee75ed8..72d4fe98a86 100644 --- a/src/Functions/lemmatize.cpp +++ b/src/Functions/lemmatize.cpp @@ -120,7 +120,7 @@ public: } -void registerFunctionLemmatize(FunctionFactory & factory) +REGISTER_FUNCTION(Lemmatize) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/lengthUTF8.cpp b/src/Functions/lengthUTF8.cpp index 349635160a6..b139f87bc64 100644 --- a/src/Functions/lengthUTF8.cpp +++ b/src/Functions/lengthUTF8.cpp @@ -68,7 +68,7 @@ using FunctionLengthUTF8 = FunctionStringOrArrayToT(); diff --git a/src/Functions/less.cpp b/src/Functions/less.cpp index 5e9107c920d..63bfcfc9f40 100644 --- a/src/Functions/less.cpp +++ b/src/Functions/less.cpp @@ -7,7 +7,7 @@ namespace DB using FunctionLess = FunctionComparison; -void registerFunctionLess(FunctionFactory & factory) +REGISTER_FUNCTION(Less) { factory.registerFunction(); } diff --git a/src/Functions/lessOrEquals.cpp b/src/Functions/lessOrEquals.cpp index 8fb4702acd1..a91afabe226 100644 --- a/src/Functions/lessOrEquals.cpp +++ b/src/Functions/lessOrEquals.cpp @@ -7,7 +7,7 @@ namespace DB using FunctionLessOrEquals = FunctionComparison; -void registerFunctionLessOrEquals(FunctionFactory & factory) +REGISTER_FUNCTION(LessOrEquals) { factory.registerFunction(); } diff --git a/src/Functions/lgamma.cpp b/src/Functions/lgamma.cpp index 57ab74f8f87..93b013e8da6 100644 --- a/src/Functions/lgamma.cpp +++ b/src/Functions/lgamma.cpp @@ -25,7 +25,7 @@ using FunctionLGamma = FunctionMathUnary(); } diff --git a/src/Functions/like.cpp b/src/Functions/like.cpp index bd73aa4d471..3a3345051d4 100644 --- a/src/Functions/like.cpp +++ b/src/Functions/like.cpp @@ -6,7 +6,7 @@ namespace DB { -void registerFunctionLike(FunctionFactory & factory) +REGISTER_FUNCTION(Like) { factory.registerFunction(); } diff --git a/src/Functions/log.cpp b/src/Functions/log.cpp index 791c73d13aa..cacb6dec1d2 100644 --- a/src/Functions/log.cpp +++ b/src/Functions/log.cpp @@ -32,7 +32,7 @@ using FunctionLog = FunctionMathUnary>; } -void registerFunctionLog(FunctionFactory & factory) +REGISTER_FUNCTION(Log) { factory.registerFunction(FunctionFactory::CaseInsensitive); factory.registerAlias("ln", "log", FunctionFactory::CaseInsensitive); diff --git a/src/Functions/log10.cpp b/src/Functions/log10.cpp index 2e0bd484ed3..87b1e84f0fd 100644 --- a/src/Functions/log10.cpp +++ b/src/Functions/log10.cpp @@ -11,7 +11,7 @@ using FunctionLog10 = FunctionMathUnary(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/log1p.cpp b/src/Functions/log1p.cpp index dc3a16224be..3e44cdf1cc8 100644 --- a/src/Functions/log1p.cpp +++ b/src/Functions/log1p.cpp @@ -13,7 +13,7 @@ namespace } -void registerFunctionLog1p(FunctionFactory & factory) +REGISTER_FUNCTION(Log1p) { factory.registerFunction(); } diff --git a/src/Functions/log2.cpp b/src/Functions/log2.cpp index 6ca770eafee..487a9850958 100644 --- a/src/Functions/log2.cpp +++ b/src/Functions/log2.cpp @@ -11,7 +11,7 @@ using FunctionLog2 = FunctionMathUnary>; } -void registerFunctionLog2(FunctionFactory & factory) +REGISTER_FUNCTION(Log2) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/logTrace.cpp b/src/Functions/logTrace.cpp index deb49f66123..e1e4c5eb2b0 100644 --- a/src/Functions/logTrace.cpp +++ b/src/Functions/logTrace.cpp @@ -56,7 +56,7 @@ namespace } -void registerFunctionLogTrace(FunctionFactory & factory) +REGISTER_FUNCTION(LogTrace) { factory.registerFunction(); } diff --git a/src/Functions/lowCardinalityIndices.cpp b/src/Functions/lowCardinalityIndices.cpp index de96053c1e7..3c9d618b9ba 100644 --- a/src/Functions/lowCardinalityIndices.cpp +++ b/src/Functions/lowCardinalityIndices.cpp @@ -57,7 +57,7 @@ public: } -void registerFunctionLowCardinalityIndices(FunctionFactory & factory) +REGISTER_FUNCTION(LowCardinalityIndices) { factory.registerFunction(); } diff --git a/src/Functions/lowCardinalityKeys.cpp b/src/Functions/lowCardinalityKeys.cpp index aa5c87d5b6d..963034f0830 100644 --- a/src/Functions/lowCardinalityKeys.cpp +++ b/src/Functions/lowCardinalityKeys.cpp @@ -50,7 +50,7 @@ public: } -void registerFunctionLowCardinalityKeys(FunctionFactory & factory) +REGISTER_FUNCTION(LowCardinalityKeys) { factory.registerFunction(); } diff --git a/src/Functions/lower.cpp b/src/Functions/lower.cpp index 0b19ae03f86..a1b777db112 100644 --- a/src/Functions/lower.cpp +++ b/src/Functions/lower.cpp @@ -17,7 +17,7 @@ using FunctionLower = FunctionStringToString, NameLower } -void registerFunctionLower(FunctionFactory & factory) +REGISTER_FUNCTION(Lower) { factory.registerFunction(FunctionFactory::CaseInsensitive); factory.registerAlias("lcase", NameLower::name, FunctionFactory::CaseInsensitive); diff --git a/src/Functions/lowerUTF8.cpp b/src/Functions/lowerUTF8.cpp index c8ff9636209..7adb0069121 100644 --- a/src/Functions/lowerUTF8.cpp +++ b/src/Functions/lowerUTF8.cpp @@ -19,7 +19,7 @@ using FunctionLowerUTF8 = FunctionStringToString(); } diff --git a/src/Functions/makeDate.cpp b/src/Functions/makeDate.cpp index dbf29322787..3e53dcff600 100644 --- a/src/Functions/makeDate.cpp +++ b/src/Functions/makeDate.cpp @@ -433,7 +433,7 @@ private: } -void registerFunctionsMakeDate(FunctionFactory & factory) +REGISTER_FUNCTION(MakeDate) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp index 964dc92ef3c..cdb68a142a0 100644 --- a/src/Functions/map.cpp +++ b/src/Functions/map.cpp @@ -647,7 +647,7 @@ public: } -void registerFunctionsMap(FunctionFactory & factory) +REGISTER_FUNCTION(Map) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/mapFilter.cpp b/src/Functions/mapFilter.cpp index f38f8f8b4d1..639cee7ce0f 100644 --- a/src/Functions/mapFilter.cpp +++ b/src/Functions/mapFilter.cpp @@ -133,7 +133,7 @@ struct MapApplyImpl } }; -void registerFunctionMapApply(FunctionFactory & factory) +REGISTER_FUNCTION(MapApply) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/match.cpp b/src/Functions/match.cpp index a0789f229fd..c719cc6dd82 100644 --- a/src/Functions/match.cpp +++ b/src/Functions/match.cpp @@ -17,7 +17,7 @@ using FunctionMatch = FunctionsStringSearch(); factory.registerAlias("REGEXP_MATCHES", NameMatch::name, FunctionFactory::CaseInsensitive); diff --git a/src/Functions/materialize.cpp b/src/Functions/materialize.cpp index c9fdd019be7..5cef610b60a 100644 --- a/src/Functions/materialize.cpp +++ b/src/Functions/materialize.cpp @@ -5,7 +5,7 @@ namespace DB { -void registerFunctionMaterialize(FunctionFactory & factory) +REGISTER_FUNCTION(Materialize) { factory.registerFunction(); } diff --git a/src/Functions/mathConstants.cpp b/src/Functions/mathConstants.cpp index ecc2f8c48b5..c65b55cf7cf 100644 --- a/src/Functions/mathConstants.cpp +++ b/src/Functions/mathConstants.cpp @@ -34,12 +34,12 @@ namespace using FunctionPi = FunctionMathConstFloat64; } -void registerFunctionE(FunctionFactory & factory) +REGISTER_FUNCTION(E) { factory.registerFunction(); } -void registerFunctionPi(FunctionFactory & factory) +REGISTER_FUNCTION(Pi) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/max2.cpp b/src/Functions/max2.cpp index 0bfba238b1a..3a693f1f5bb 100644 --- a/src/Functions/max2.cpp +++ b/src/Functions/max2.cpp @@ -19,7 +19,7 @@ namespace using FunctionMax2 = FunctionMathBinaryFloat64>; } -void registerFunctionMax2(FunctionFactory & factory) +REGISTER_FUNCTION(Max2) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/meiliMatch.cpp b/src/Functions/meiliMatch.cpp index d4886bd0a14..55002354ba7 100644 --- a/src/Functions/meiliMatch.cpp +++ b/src/Functions/meiliMatch.cpp @@ -41,7 +41,7 @@ namespace } -void registerFunctionMeiliMatch(FunctionFactory & factory) +REGISTER_FUNCTION(MeiliMatch) { factory.registerFunction(); } diff --git a/src/Functions/min2.cpp b/src/Functions/min2.cpp index d02db598139..10233ab4011 100644 --- a/src/Functions/min2.cpp +++ b/src/Functions/min2.cpp @@ -20,7 +20,7 @@ namespace using FunctionMin2 = FunctionMathBinaryFloat64>; } -void registerFunctionMin2(FunctionFactory & factory) +REGISTER_FUNCTION(Min2) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/minSampleSize.cpp b/src/Functions/minSampleSize.cpp index 02a94c743e8..66af9f6c2e9 100644 --- a/src/Functions/minSampleSize.cpp +++ b/src/Functions/minSampleSize.cpp @@ -282,7 +282,7 @@ struct ConversionImpl }; -void registerFunctionMinSampleSize(FunctionFactory & factory) +REGISTER_FUNCTION(MinSampleSize) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/minus.cpp b/src/Functions/minus.cpp index 76459545790..3668e4afc18 100644 --- a/src/Functions/minus.cpp +++ b/src/Functions/minus.cpp @@ -46,7 +46,7 @@ struct MinusImpl struct NameMinus { static constexpr auto name = "minus"; }; using FunctionMinus = BinaryArithmeticOverloadResolver; -void registerFunctionMinus(FunctionFactory & factory) +REGISTER_FUNCTION(Minus) { factory.registerFunction(); } diff --git a/src/Functions/modelEvaluate.cpp b/src/Functions/modelEvaluate.cpp index 53bb109728a..3ee2ae3fae4 100644 --- a/src/Functions/modelEvaluate.cpp +++ b/src/Functions/modelEvaluate.cpp @@ -168,7 +168,7 @@ private: }; -void registerFunctionsExternalModels(FunctionFactory & factory) +REGISTER_FUNCTION(ExternalModels) { factory.registerFunction(); } diff --git a/src/Functions/modulo.cpp b/src/Functions/modulo.cpp index 175f74bab8f..9a3aa12037f 100644 --- a/src/Functions/modulo.cpp +++ b/src/Functions/modulo.cpp @@ -162,7 +162,7 @@ template <> struct BinaryOperationImpl> : struct NameModulo { static constexpr auto name = "modulo"; }; using FunctionModulo = BinaryArithmeticOverloadResolver; -void registerFunctionModulo(FunctionFactory & factory) +REGISTER_FUNCTION(Modulo) { factory.registerFunction(); factory.registerAlias("mod", "modulo", FunctionFactory::CaseInsensitive); @@ -171,7 +171,7 @@ void registerFunctionModulo(FunctionFactory & factory) struct NameModuloLegacy { static constexpr auto name = "moduloLegacy"; }; using FunctionModuloLegacy = BinaryArithmeticOverloadResolver; -void registerFunctionModuloLegacy(FunctionFactory & factory) +REGISTER_FUNCTION(ModuloLegacy) { factory.registerFunction(); } diff --git a/src/Functions/moduloOrZero.cpp b/src/Functions/moduloOrZero.cpp index ca55ccc79fd..3551ae74c5f 100644 --- a/src/Functions/moduloOrZero.cpp +++ b/src/Functions/moduloOrZero.cpp @@ -41,7 +41,7 @@ using FunctionModuloOrZero = BinaryArithmeticOverloadResolver(); } diff --git a/src/Functions/monthName.cpp b/src/Functions/monthName.cpp index c397fdffaa5..e841f68b326 100644 --- a/src/Functions/monthName.cpp +++ b/src/Functions/monthName.cpp @@ -72,7 +72,7 @@ private: FunctionOverloadResolverPtr function_resolver; }; -void registerFunctionMonthName(FunctionFactory & factory) +REGISTER_FUNCTION(MonthName) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/multiFuzzyMatchAllIndices.cpp b/src/Functions/multiFuzzyMatchAllIndices.cpp index 93ffb936dc1..6a78164e111 100644 --- a/src/Functions/multiFuzzyMatchAllIndices.cpp +++ b/src/Functions/multiFuzzyMatchAllIndices.cpp @@ -17,7 +17,7 @@ using FunctionMultiFuzzyMatchAllIndices = FunctionsMultiStringFuzzySearch(); } diff --git a/src/Functions/multiFuzzyMatchAny.cpp b/src/Functions/multiFuzzyMatchAny.cpp index a627030d7af..628f9a656ae 100644 --- a/src/Functions/multiFuzzyMatchAny.cpp +++ b/src/Functions/multiFuzzyMatchAny.cpp @@ -17,7 +17,7 @@ using FunctionMultiFuzzyMatchAny = FunctionsMultiStringFuzzySearch(); } diff --git a/src/Functions/multiFuzzyMatchAnyIndex.cpp b/src/Functions/multiFuzzyMatchAnyIndex.cpp index 4b24a06e171..a2c0d3370dd 100644 --- a/src/Functions/multiFuzzyMatchAnyIndex.cpp +++ b/src/Functions/multiFuzzyMatchAnyIndex.cpp @@ -17,7 +17,7 @@ using FunctionMultiFuzzyMatchAnyIndex = FunctionsMultiStringFuzzySearch(); } diff --git a/src/Functions/multiIf.cpp b/src/Functions/multiIf.cpp index 9e0ca4142e5..6fc722e32f4 100644 --- a/src/Functions/multiIf.cpp +++ b/src/Functions/multiIf.cpp @@ -332,7 +332,7 @@ private: } -void registerFunctionMultiIf(FunctionFactory & factory) +REGISTER_FUNCTION(MultiIf) { factory.registerFunction(); diff --git a/src/Functions/multiMatchAllIndices.cpp b/src/Functions/multiMatchAllIndices.cpp index 47bd57029e2..6a907654511 100644 --- a/src/Functions/multiMatchAllIndices.cpp +++ b/src/Functions/multiMatchAllIndices.cpp @@ -17,7 +17,7 @@ using FunctionMultiMatchAllIndices = FunctionsMultiStringSearch(); } diff --git a/src/Functions/multiMatchAny.cpp b/src/Functions/multiMatchAny.cpp index 324e435de26..6e6abe61898 100644 --- a/src/Functions/multiMatchAny.cpp +++ b/src/Functions/multiMatchAny.cpp @@ -17,7 +17,7 @@ using FunctionMultiMatchAny = FunctionsMultiStringSearch(); } diff --git a/src/Functions/multiMatchAnyIndex.cpp b/src/Functions/multiMatchAnyIndex.cpp index 6a11fa4eb35..9e96f89c8de 100644 --- a/src/Functions/multiMatchAnyIndex.cpp +++ b/src/Functions/multiMatchAnyIndex.cpp @@ -17,7 +17,7 @@ using FunctionMultiMatchAnyIndex = FunctionsMultiStringSearch(); } diff --git a/src/Functions/multiSearchAllPositions.cpp b/src/Functions/multiSearchAllPositions.cpp index 53f3da9cde6..6d75e814192 100644 --- a/src/Functions/multiSearchAllPositions.cpp +++ b/src/Functions/multiSearchAllPositions.cpp @@ -19,7 +19,7 @@ using FunctionMultiSearchAllPositions } -void registerFunctionMultiSearchAllPositions(FunctionFactory & factory) +REGISTER_FUNCTION(MultiSearchAllPositions) { factory.registerFunction(); } diff --git a/src/Functions/multiSearchAllPositionsCaseInsensitive.cpp b/src/Functions/multiSearchAllPositionsCaseInsensitive.cpp index 55c112eb093..03f22ff7417 100644 --- a/src/Functions/multiSearchAllPositionsCaseInsensitive.cpp +++ b/src/Functions/multiSearchAllPositionsCaseInsensitive.cpp @@ -19,7 +19,7 @@ using FunctionMultiSearchAllPositionsCaseInsensitive } -void registerFunctionMultiSearchAllPositionsCaseInsensitive(FunctionFactory & factory) +REGISTER_FUNCTION(MultiSearchAllPositionsCaseInsensitive) { factory.registerFunction(); } diff --git a/src/Functions/multiSearchAllPositionsCaseInsensitiveUTF8.cpp b/src/Functions/multiSearchAllPositionsCaseInsensitiveUTF8.cpp index df9de8a17ec..9259a20cda4 100644 --- a/src/Functions/multiSearchAllPositionsCaseInsensitiveUTF8.cpp +++ b/src/Functions/multiSearchAllPositionsCaseInsensitiveUTF8.cpp @@ -19,7 +19,7 @@ using FunctionMultiSearchAllPositionsCaseInsensitiveUTF8 } -void registerFunctionMultiSearchAllPositionsCaseInsensitiveUTF8(FunctionFactory & factory) +REGISTER_FUNCTION(MultiSearchAllPositionsCaseInsensitiveUTF8) { factory.registerFunction(); } diff --git a/src/Functions/multiSearchAllPositionsUTF8.cpp b/src/Functions/multiSearchAllPositionsUTF8.cpp index e5f9a02afcc..9924be18e35 100644 --- a/src/Functions/multiSearchAllPositionsUTF8.cpp +++ b/src/Functions/multiSearchAllPositionsUTF8.cpp @@ -19,7 +19,7 @@ using FunctionMultiSearchAllPositionsUTF8 } -void registerFunctionMultiSearchAllPositionsUTF8(FunctionFactory & factory) +REGISTER_FUNCTION(MultiSearchAllPositionsUTF8) { factory.registerFunction(); } diff --git a/src/Functions/multiSearchAny.cpp b/src/Functions/multiSearchAny.cpp index 113289b83ed..166bba94913 100644 --- a/src/Functions/multiSearchAny.cpp +++ b/src/Functions/multiSearchAny.cpp @@ -18,7 +18,7 @@ using FunctionMultiSearch = FunctionsMultiStringSearch(); } diff --git a/src/Functions/multiSearchAnyCaseInsensitive.cpp b/src/Functions/multiSearchAnyCaseInsensitive.cpp index af463805ea5..209a586b4aa 100644 --- a/src/Functions/multiSearchAnyCaseInsensitive.cpp +++ b/src/Functions/multiSearchAnyCaseInsensitive.cpp @@ -17,7 +17,7 @@ using FunctionMultiSearchCaseInsensitive = FunctionsMultiStringSearch(); } diff --git a/src/Functions/multiSearchAnyCaseInsensitiveUTF8.cpp b/src/Functions/multiSearchAnyCaseInsensitiveUTF8.cpp index c83dc843f78..ac1b36c66a7 100644 --- a/src/Functions/multiSearchAnyCaseInsensitiveUTF8.cpp +++ b/src/Functions/multiSearchAnyCaseInsensitiveUTF8.cpp @@ -19,7 +19,7 @@ using FunctionMultiSearchCaseInsensitiveUTF8 } -void registerFunctionMultiSearchAnyCaseInsensitiveUTF8(FunctionFactory & factory) +REGISTER_FUNCTION(MultiSearchAnyCaseInsensitiveUTF8) { factory.registerFunction(); } diff --git a/src/Functions/multiSearchAnyUTF8.cpp b/src/Functions/multiSearchAnyUTF8.cpp index 3f34f70ab51..b53197b10f2 100644 --- a/src/Functions/multiSearchAnyUTF8.cpp +++ b/src/Functions/multiSearchAnyUTF8.cpp @@ -17,7 +17,7 @@ using FunctionMultiSearchUTF8 = FunctionsMultiStringSearch(); } diff --git a/src/Functions/multiSearchFirstIndex.cpp b/src/Functions/multiSearchFirstIndex.cpp index a5fbec2dc36..a489d55e233 100644 --- a/src/Functions/multiSearchFirstIndex.cpp +++ b/src/Functions/multiSearchFirstIndex.cpp @@ -18,7 +18,7 @@ using FunctionMultiSearchFirstIndex = FunctionsMultiStringSearch(); } diff --git a/src/Functions/multiSearchFirstIndexCaseInsensitive.cpp b/src/Functions/multiSearchFirstIndexCaseInsensitive.cpp index cc4869d1200..896247b9572 100644 --- a/src/Functions/multiSearchFirstIndexCaseInsensitive.cpp +++ b/src/Functions/multiSearchFirstIndexCaseInsensitive.cpp @@ -19,7 +19,7 @@ using FunctionMultiSearchFirstIndexCaseInsensitive } -void registerFunctionMultiSearchFirstIndexCaseInsensitive(FunctionFactory & factory) +REGISTER_FUNCTION(MultiSearchFirstIndexCaseInsensitive) { factory.registerFunction(); } diff --git a/src/Functions/multiSearchFirstIndexCaseInsensitiveUTF8.cpp b/src/Functions/multiSearchFirstIndexCaseInsensitiveUTF8.cpp index fd95947bc67..e5048120721 100644 --- a/src/Functions/multiSearchFirstIndexCaseInsensitiveUTF8.cpp +++ b/src/Functions/multiSearchFirstIndexCaseInsensitiveUTF8.cpp @@ -19,7 +19,7 @@ using FunctionMultiSearchFirstIndexCaseInsensitiveUTF8 } -void registerFunctionMultiSearchFirstIndexCaseInsensitiveUTF8(FunctionFactory & factory) +REGISTER_FUNCTION(MultiSearchFirstIndexCaseInsensitiveUTF8) { factory.registerFunction(); } diff --git a/src/Functions/multiSearchFirstIndexUTF8.cpp b/src/Functions/multiSearchFirstIndexUTF8.cpp index 6854201d14d..929399bb75f 100644 --- a/src/Functions/multiSearchFirstIndexUTF8.cpp +++ b/src/Functions/multiSearchFirstIndexUTF8.cpp @@ -19,7 +19,7 @@ using FunctionMultiSearchFirstIndexUTF8 } -void registerFunctionMultiSearchFirstIndexUTF8(FunctionFactory & factory) +REGISTER_FUNCTION(MultiSearchFirstIndexUTF8) { factory.registerFunction(); } diff --git a/src/Functions/multiSearchFirstPosition.cpp b/src/Functions/multiSearchFirstPosition.cpp index 4ca1ac35a4d..e1eaa151107 100644 --- a/src/Functions/multiSearchFirstPosition.cpp +++ b/src/Functions/multiSearchFirstPosition.cpp @@ -19,7 +19,7 @@ using FunctionMultiSearchFirstPosition } -void registerFunctionMultiSearchFirstPosition(FunctionFactory & factory) +REGISTER_FUNCTION(MultiSearchFirstPosition) { factory.registerFunction(); } diff --git a/src/Functions/multiSearchFirstPositionCaseInsensitive.cpp b/src/Functions/multiSearchFirstPositionCaseInsensitive.cpp index 4e356335e98..eadfa1822e2 100644 --- a/src/Functions/multiSearchFirstPositionCaseInsensitive.cpp +++ b/src/Functions/multiSearchFirstPositionCaseInsensitive.cpp @@ -19,7 +19,7 @@ using FunctionMultiSearchFirstPositionCaseInsensitive } -void registerFunctionMultiSearchFirstPositionCaseInsensitive(FunctionFactory & factory) +REGISTER_FUNCTION(MultiSearchFirstPositionCaseInsensitive) { factory.registerFunction(); } diff --git a/src/Functions/multiSearchFirstPositionCaseInsensitiveUTF8.cpp b/src/Functions/multiSearchFirstPositionCaseInsensitiveUTF8.cpp index 647fc3a2cc8..da254b42e21 100644 --- a/src/Functions/multiSearchFirstPositionCaseInsensitiveUTF8.cpp +++ b/src/Functions/multiSearchFirstPositionCaseInsensitiveUTF8.cpp @@ -19,7 +19,7 @@ using FunctionMultiSearchFirstPositionCaseInsensitiveUTF8 = FunctionsMultiString } -void registerFunctionMultiSearchFirstPositionCaseInsensitiveUTF8(FunctionFactory & factory) +REGISTER_FUNCTION(MultiSearchFirstPositionCaseInsensitiveUTF8) { factory.registerFunction(); } diff --git a/src/Functions/multiSearchFirstPositionUTF8.cpp b/src/Functions/multiSearchFirstPositionUTF8.cpp index fbb1099ec35..b56a0b1c403 100644 --- a/src/Functions/multiSearchFirstPositionUTF8.cpp +++ b/src/Functions/multiSearchFirstPositionUTF8.cpp @@ -19,7 +19,7 @@ using FunctionMultiSearchFirstPositionUTF8 } -void registerFunctionMultiSearchFirstPositionUTF8(FunctionFactory & factory) +REGISTER_FUNCTION(MultiSearchFirstPositionUTF8) { factory.registerFunction(); } diff --git a/src/Functions/multiply.cpp b/src/Functions/multiply.cpp index 7cbf806faa0..4dc8cd10f31 100644 --- a/src/Functions/multiply.cpp +++ b/src/Functions/multiply.cpp @@ -53,7 +53,7 @@ struct MultiplyImpl struct NameMultiply { static constexpr auto name = "multiply"; }; using FunctionMultiply = BinaryArithmeticOverloadResolver; -void registerFunctionMultiply(FunctionFactory & factory) +REGISTER_FUNCTION(Multiply) { factory.registerFunction(); } diff --git a/src/Functions/negate.cpp b/src/Functions/negate.cpp index a378589fe30..779cc0fff78 100644 --- a/src/Functions/negate.cpp +++ b/src/Functions/negate.cpp @@ -46,7 +46,7 @@ template <> struct FunctionUnaryArithmeticMonotonicity } }; -void registerFunctionNegate(FunctionFactory & factory) +REGISTER_FUNCTION(Negate) { factory.registerFunction(); } diff --git a/src/Functions/neighbor.cpp b/src/Functions/neighbor.cpp index 296eaf67d9c..11f6edd1786 100644 --- a/src/Functions/neighbor.cpp +++ b/src/Functions/neighbor.cpp @@ -203,7 +203,7 @@ public: } -void registerFunctionNeighbor(FunctionFactory & factory) +REGISTER_FUNCTION(Neighbor) { factory.registerFunction(); } diff --git a/src/Functions/normalizeQuery.cpp b/src/Functions/normalizeQuery.cpp index dd36e931bc4..ee1688fe197 100644 --- a/src/Functions/normalizeQuery.cpp +++ b/src/Functions/normalizeQuery.cpp @@ -51,7 +51,7 @@ struct Impl } -void registerFunctionNormalizeQuery(FunctionFactory & factory) +REGISTER_FUNCTION(NormalizeQuery) { factory.registerFunction, Impl>>(); factory.registerFunction, Impl>>(); diff --git a/src/Functions/normalizeString.cpp b/src/Functions/normalizeString.cpp index 625247ca432..a6bec0878f7 100644 --- a/src/Functions/normalizeString.cpp +++ b/src/Functions/normalizeString.cpp @@ -169,7 +169,7 @@ using FunctionNormalizeUTF8NFKC = FunctionStringToString, NormalizeNFKDImpl>; } -void registerFunctionNormalizeUTF8(FunctionFactory & factory) +REGISTER_FUNCTION(NormalizeUTF8) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/normalizedQueryHash.cpp b/src/Functions/normalizedQueryHash.cpp index 1e2875ae3e3..b6efa4e18cd 100644 --- a/src/Functions/normalizedQueryHash.cpp +++ b/src/Functions/normalizedQueryHash.cpp @@ -97,7 +97,7 @@ public: } -void registerFunctionNormalizedQueryHash(FunctionFactory & factory) +REGISTER_FUNCTION(NormalizedQueryHash) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/notEmpty.cpp b/src/Functions/notEmpty.cpp index cc40be88ab1..f7d58e15143 100644 --- a/src/Functions/notEmpty.cpp +++ b/src/Functions/notEmpty.cpp @@ -17,7 +17,7 @@ using FunctionNotEmpty = FunctionStringOrArrayToT, NameNotEmpty, } -void registerFunctionNotEmpty(FunctionFactory & factory) +REGISTER_FUNCTION(NotEmpty) { factory.registerFunction(); } diff --git a/src/Functions/notEquals.cpp b/src/Functions/notEquals.cpp index 78e4b19fe21..08bedff399e 100644 --- a/src/Functions/notEquals.cpp +++ b/src/Functions/notEquals.cpp @@ -7,7 +7,7 @@ namespace DB using FunctionNotEquals = FunctionComparison; -void registerFunctionNotEquals(FunctionFactory & factory) +REGISTER_FUNCTION(NotEquals) { factory.registerFunction(); } diff --git a/src/Functions/notILike.cpp b/src/Functions/notILike.cpp index 5e78db1c518..f3b92300d7f 100644 --- a/src/Functions/notILike.cpp +++ b/src/Functions/notILike.cpp @@ -17,7 +17,7 @@ using FunctionNotILike = FunctionsStringSearch; } -void registerFunctionNotILike(FunctionFactory & factory) +REGISTER_FUNCTION(NotILike) { factory.registerFunction(); } diff --git a/src/Functions/notLike.cpp b/src/Functions/notLike.cpp index 33a36748bb1..a546b511a0b 100644 --- a/src/Functions/notLike.cpp +++ b/src/Functions/notLike.cpp @@ -16,7 +16,7 @@ using FunctionNotLike = FunctionsStringSearch(); } diff --git a/src/Functions/now.cpp b/src/Functions/now.cpp index b2cfa7aa2b1..9ecaca55e52 100644 --- a/src/Functions/now.cpp +++ b/src/Functions/now.cpp @@ -126,7 +126,7 @@ public: } -void registerFunctionNow(FunctionFactory & factory) +REGISTER_FUNCTION(Now) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/now64.cpp b/src/Functions/now64.cpp index 071140f4bf6..0308fa95b39 100644 --- a/src/Functions/now64.cpp +++ b/src/Functions/now64.cpp @@ -158,7 +158,7 @@ public: } -void registerFunctionNow64(FunctionFactory & factory) +REGISTER_FUNCTION(Now64) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/nowInBlock.cpp b/src/Functions/nowInBlock.cpp index b657bc92085..db72e791587 100644 --- a/src/Functions/nowInBlock.cpp +++ b/src/Functions/nowInBlock.cpp @@ -80,7 +80,7 @@ public: } -void registerFunctionNowInBlock(FunctionFactory & factory) +REGISTER_FUNCTION(NowInBlock) { factory.registerFunction(); } diff --git a/src/Functions/nullIf.cpp b/src/Functions/nullIf.cpp index 0b4d024c91c..e85747834b1 100644 --- a/src/Functions/nullIf.cpp +++ b/src/Functions/nullIf.cpp @@ -67,7 +67,7 @@ public: } -void registerFunctionNullIf(FunctionFactory & factory) +REGISTER_FUNCTION(NullIf) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/padString.cpp b/src/Functions/padString.cpp index 544911919fe..c8ed920755c 100644 --- a/src/Functions/padString.cpp +++ b/src/Functions/padString.cpp @@ -307,7 +307,7 @@ namespace }; } -void registerFunctionPadString(FunctionFactory & factory) +REGISTER_FUNCTION(PadString) { factory.registerFunction>(); /// leftPad factory.registerFunction>(); /// leftPadUTF8 diff --git a/src/Functions/parseTimeDelta.cpp b/src/Functions/parseTimeDelta.cpp index 8cb7c229ae8..6842997a71c 100644 --- a/src/Functions/parseTimeDelta.cpp +++ b/src/Functions/parseTimeDelta.cpp @@ -304,7 +304,7 @@ namespace } -void registerFunctionParseTimeDelta(FunctionFactory & factory) +REGISTER_FUNCTION(ParseTimeDelta) { factory.registerFunction(); } diff --git a/src/Functions/partitionId.cpp b/src/Functions/partitionId.cpp index c5dced68e88..22c3910feaf 100644 --- a/src/Functions/partitionId.cpp +++ b/src/Functions/partitionId.cpp @@ -63,7 +63,7 @@ public: } }; -void registerFunctionPartitionId(FunctionFactory & factory) +REGISTER_FUNCTION(PartitionId) { factory.registerFunction(); } diff --git a/src/Functions/plus.cpp b/src/Functions/plus.cpp index 997cae0dbed..4b81c23584c 100644 --- a/src/Functions/plus.cpp +++ b/src/Functions/plus.cpp @@ -48,7 +48,7 @@ struct PlusImpl struct NamePlus { static constexpr auto name = "plus"; }; using FunctionPlus = BinaryArithmeticOverloadResolver; -void registerFunctionPlus(FunctionFactory & factory) +REGISTER_FUNCTION(Plus) { factory.registerFunction(); } diff --git a/src/Functions/pointInEllipses.cpp b/src/Functions/pointInEllipses.cpp index 1de6e1d5ce6..f69886ad71f 100644 --- a/src/Functions/pointInEllipses.cpp +++ b/src/Functions/pointInEllipses.cpp @@ -194,7 +194,7 @@ private: } -void registerFunctionPointInEllipses(FunctionFactory & factory) +REGISTER_FUNCTION(PointInEllipses) { factory.registerFunction(); } diff --git a/src/Functions/pointInPolygon.cpp b/src/Functions/pointInPolygon.cpp index 7d2369fd5e7..ee2f566e619 100644 --- a/src/Functions/pointInPolygon.cpp +++ b/src/Functions/pointInPolygon.cpp @@ -574,7 +574,7 @@ private: } -void registerFunctionPointInPolygon(FunctionFactory & factory) +REGISTER_FUNCTION(PointInPolygon) { factory.registerFunction>>(); } diff --git a/src/Functions/polygonArea.cpp b/src/Functions/polygonArea.cpp index f7a5c96da55..a83e960dc0d 100644 --- a/src/Functions/polygonArea.cpp +++ b/src/Functions/polygonArea.cpp @@ -100,7 +100,7 @@ template <> const char * FunctionPolygonArea::name = "polygonAreaSpherical"; -void registerFunctionPolygonArea(FunctionFactory & factory) +REGISTER_FUNCTION(PolygonArea) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/polygonConvexHull.cpp b/src/Functions/polygonConvexHull.cpp index a027b9536c5..c0b079d3014 100644 --- a/src/Functions/polygonConvexHull.cpp +++ b/src/Functions/polygonConvexHull.cpp @@ -99,7 +99,7 @@ template <> const char * FunctionPolygonConvexHull::name = "polygonConvexHullCartesian"; -void registerFunctionPolygonConvexHull(FunctionFactory & factory) +REGISTER_FUNCTION(PolygonConvexHull) { factory.registerFunction>(); } diff --git a/src/Functions/polygonPerimeter.cpp b/src/Functions/polygonPerimeter.cpp index 1930454a8d4..5d5a3e91e77 100644 --- a/src/Functions/polygonPerimeter.cpp +++ b/src/Functions/polygonPerimeter.cpp @@ -99,7 +99,7 @@ template <> const char * FunctionPolygonPerimeter::name = "polygonPerimeterSpherical"; -void registerFunctionPolygonPerimeter(FunctionFactory & factory) +REGISTER_FUNCTION(PolygonPerimeter) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/polygonsDistance.cpp b/src/Functions/polygonsDistance.cpp index db120da61c9..3b01d01f311 100644 --- a/src/Functions/polygonsDistance.cpp +++ b/src/Functions/polygonsDistance.cpp @@ -109,7 +109,7 @@ template <> const char * FunctionPolygonsDistance::name = "polygonsDistanceSpherical"; -void registerFunctionPolygonsDistance(FunctionFactory & factory) +REGISTER_FUNCTION(PolygonsDistance) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/polygonsEquals.cpp b/src/Functions/polygonsEquals.cpp index 7112c9f1434..2793df7efac 100644 --- a/src/Functions/polygonsEquals.cpp +++ b/src/Functions/polygonsEquals.cpp @@ -108,7 +108,7 @@ template <> const char * FunctionPolygonsEquals::name = "polygonsEqualsCartesian"; -void registerFunctionPolygonsEquals(FunctionFactory & factory) +REGISTER_FUNCTION(PolygonsEquals) { factory.registerFunction>(); } diff --git a/src/Functions/polygonsIntersection.cpp b/src/Functions/polygonsIntersection.cpp index 01c3dbf4e11..dd17665edba 100644 --- a/src/Functions/polygonsIntersection.cpp +++ b/src/Functions/polygonsIntersection.cpp @@ -115,7 +115,7 @@ template <> const char * FunctionPolygonsIntersection::name = "polygonsIntersectionSpherical"; -void registerFunctionPolygonsIntersection(FunctionFactory & factory) +REGISTER_FUNCTION(PolygonsIntersection) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/polygonsSymDifference.cpp b/src/Functions/polygonsSymDifference.cpp index ed357fb796b..b8f4443f1ed 100644 --- a/src/Functions/polygonsSymDifference.cpp +++ b/src/Functions/polygonsSymDifference.cpp @@ -109,7 +109,7 @@ const char * FunctionPolygonsSymDifference::name = "polygonsSymD template <> const char * FunctionPolygonsSymDifference::name = "polygonsSymDifferenceSpherical"; -void registerFunctionPolygonsSymDifference(FunctionFactory & factory) +REGISTER_FUNCTION(PolygonsSymDifference) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/polygonsUnion.cpp b/src/Functions/polygonsUnion.cpp index 01ef4dda8d6..ea770f461f6 100644 --- a/src/Functions/polygonsUnion.cpp +++ b/src/Functions/polygonsUnion.cpp @@ -113,7 +113,7 @@ template <> const char * FunctionPolygonsUnion::name = "polygonsUnionSpherical"; -void registerFunctionPolygonsUnion(FunctionFactory & factory) +REGISTER_FUNCTION(PolygonsUnion) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/polygonsWithin.cpp b/src/Functions/polygonsWithin.cpp index 3d86c16fce7..6cc6a8cb2ef 100644 --- a/src/Functions/polygonsWithin.cpp +++ b/src/Functions/polygonsWithin.cpp @@ -112,7 +112,7 @@ template <> const char * FunctionPolygonsWithin::name = "polygonsWithinSpherical"; -void registerFunctionPolygonsWithin(FunctionFactory & factory) +REGISTER_FUNCTION(PolygonsWithin) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/position.cpp b/src/Functions/position.cpp index e38dc52b9af..c25beec5ed9 100644 --- a/src/Functions/position.cpp +++ b/src/Functions/position.cpp @@ -17,7 +17,7 @@ using FunctionPosition = FunctionsStringSearch(FunctionFactory::CaseInsensitive); factory.registerAlias("locate", NamePosition::name, FunctionFactory::CaseInsensitive); diff --git a/src/Functions/positionCaseInsensitive.cpp b/src/Functions/positionCaseInsensitive.cpp index ed9d86c033c..4e3b670fe54 100644 --- a/src/Functions/positionCaseInsensitive.cpp +++ b/src/Functions/positionCaseInsensitive.cpp @@ -17,7 +17,7 @@ using FunctionPositionCaseInsensitive = FunctionsStringSearch(); } diff --git a/src/Functions/positionCaseInsensitiveUTF8.cpp b/src/Functions/positionCaseInsensitiveUTF8.cpp index f6e344b119e..e7ebf02d156 100644 --- a/src/Functions/positionCaseInsensitiveUTF8.cpp +++ b/src/Functions/positionCaseInsensitiveUTF8.cpp @@ -18,7 +18,7 @@ using FunctionPositionCaseInsensitiveUTF8 } -void registerFunctionPositionCaseInsensitiveUTF8(FunctionFactory & factory) +REGISTER_FUNCTION(PositionCaseInsensitiveUTF8) { factory.registerFunction(); } diff --git a/src/Functions/positionUTF8.cpp b/src/Functions/positionUTF8.cpp index ecb2a1e9e97..fd9def8109d 100644 --- a/src/Functions/positionUTF8.cpp +++ b/src/Functions/positionUTF8.cpp @@ -17,7 +17,7 @@ using FunctionPositionUTF8 = FunctionsStringSearch(); } diff --git a/src/Functions/pow.cpp b/src/Functions/pow.cpp index 7e60e0e878e..afbf9d10f16 100644 --- a/src/Functions/pow.cpp +++ b/src/Functions/pow.cpp @@ -11,7 +11,7 @@ using FunctionPow = FunctionMathBinaryFloat64(FunctionFactory::CaseInsensitive); factory.registerAlias("power", "pow", FunctionFactory::CaseInsensitive); diff --git a/src/Functions/queryID.cpp b/src/Functions/queryID.cpp index b1945bb1686..704206e1de5 100644 --- a/src/Functions/queryID.cpp +++ b/src/Functions/queryID.cpp @@ -38,7 +38,7 @@ public: } }; -void registerFunctionQueryID(FunctionFactory & factory) +REGISTER_FUNCTION(QueryID) { factory.registerFunction(); factory.registerAlias("query_id", FunctionQueryID::name, FunctionFactory::CaseInsensitive); diff --git a/src/Functions/radians.cpp b/src/Functions/radians.cpp index 4dffdc08547..5e46ccca5bd 100644 --- a/src/Functions/radians.cpp +++ b/src/Functions/radians.cpp @@ -21,7 +21,7 @@ namespace using FunctionRadians = FunctionMathUnary>; } -void registerFunctionRadians(FunctionFactory & factory) +REGISTER_FUNCTION(Radians) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/rand.cpp b/src/Functions/rand.cpp index 660119a64cb..ba511382651 100644 --- a/src/Functions/rand.cpp +++ b/src/Functions/rand.cpp @@ -11,7 +11,7 @@ using FunctionRand = FunctionRandom; } -void registerFunctionRand(FunctionFactory & factory) +REGISTER_FUNCTION(Rand) { factory.registerFunction(FunctionFactory::CaseInsensitive); factory.registerAlias("rand32", NameRand::name); diff --git a/src/Functions/rand64.cpp b/src/Functions/rand64.cpp index 9377d3d40d0..b8b9d4d2b02 100644 --- a/src/Functions/rand64.cpp +++ b/src/Functions/rand64.cpp @@ -11,7 +11,7 @@ using FunctionRand64 = FunctionRandom; } -void registerFunctionRand64(FunctionFactory & factory) +REGISTER_FUNCTION(Rand64) { factory.registerFunction(); } diff --git a/src/Functions/randConstant.cpp b/src/Functions/randConstant.cpp index e273eb4992e..7bf8630f92a 100644 --- a/src/Functions/randConstant.cpp +++ b/src/Functions/randConstant.cpp @@ -117,7 +117,7 @@ using FunctionBuilderRandConstant = RandomConstantOverloadResolver(); } diff --git a/src/Functions/randomFixedString.cpp b/src/Functions/randomFixedString.cpp index eb8214383a7..a0b7a999d36 100644 --- a/src/Functions/randomFixedString.cpp +++ b/src/Functions/randomFixedString.cpp @@ -106,7 +106,7 @@ private: } -void registerFunctionRandomFixedString(FunctionFactory & factory) +REGISTER_FUNCTION(RandomFixedString) { factory.registerFunction(); } diff --git a/src/Functions/randomPrintableASCII.cpp b/src/Functions/randomPrintableASCII.cpp index 3bfa1fbda6d..cd46f175977 100644 --- a/src/Functions/randomPrintableASCII.cpp +++ b/src/Functions/randomPrintableASCII.cpp @@ -113,7 +113,7 @@ public: } -void registerFunctionRandomPrintableASCII(FunctionFactory & factory) +REGISTER_FUNCTION(RandomPrintableASCII) { factory.registerFunction(); } diff --git a/src/Functions/randomString.cpp b/src/Functions/randomString.cpp index 0e64892bbf4..16b0254c536 100644 --- a/src/Functions/randomString.cpp +++ b/src/Functions/randomString.cpp @@ -126,7 +126,7 @@ private: } -void registerFunctionRandomString(FunctionFactory & factory) +REGISTER_FUNCTION(RandomString) { factory.registerFunction(); } diff --git a/src/Functions/randomStringUTF8.cpp b/src/Functions/randomStringUTF8.cpp index 59fc45a574d..043db179d71 100644 --- a/src/Functions/randomStringUTF8.cpp +++ b/src/Functions/randomStringUTF8.cpp @@ -148,7 +148,7 @@ public: } -void registerFunctionRandomStringUTF8(FunctionFactory & factory) +REGISTER_FUNCTION(RandomStringUTF8) { factory.registerFunction(); } diff --git a/src/Functions/readWkt.cpp b/src/Functions/readWkt.cpp index b8d0d20acb3..28ed9a64ea8 100644 --- a/src/Functions/readWkt.cpp +++ b/src/Functions/readWkt.cpp @@ -96,7 +96,7 @@ struct ReadWKTMultiPolygonNameHolder static constexpr const char * name = "readWKTMultiPolygon"; }; -void registerFunctionReadWKT(FunctionFactory & factory) +REGISTER_FUNCTION(ReadWKT) { factory.registerFunction, ReadWKTPointNameHolder>>(); factory.registerFunction, ReadWKTRingNameHolder>>(); diff --git a/src/Functions/regexpQuoteMeta.cpp b/src/Functions/regexpQuoteMeta.cpp index 0db0405323a..d0ec572ac59 100644 --- a/src/Functions/regexpQuoteMeta.cpp +++ b/src/Functions/regexpQuoteMeta.cpp @@ -112,7 +112,7 @@ public: } -void registerFunctionRegexpQuoteMeta(FunctionFactory & factory) +REGISTER_FUNCTION(RegexpQuoteMeta) { factory.registerFunction(); } diff --git a/src/Functions/registerFunctions.cpp b/src/Functions/registerFunctions.cpp index f578bfc9d68..202ad1e3971 100644 --- a/src/Functions/registerFunctions.cpp +++ b/src/Functions/registerFunctions.cpp @@ -5,141 +5,13 @@ namespace DB { -void registerFunctionsArithmetic(FunctionFactory &); -void registerFunctionsArray(FunctionFactory &); -void registerFunctionsTuple(FunctionFactory &); -void registerFunctionsMakeDate(FunctionFactory &); -void registerFunctionsMap(FunctionFactory &); -void registerFunctionsBitmap(FunctionFactory &); -void registerFunctionsBinaryRepr(FunctionFactory &); -void registerFunctionsCoding(FunctionFactory &); -void registerFunctionsCodingUUID(FunctionFactory &); -void registerFunctionChar(FunctionFactory &); -void registerFunctionsComparison(FunctionFactory &); -void registerFunctionsConditional(FunctionFactory &); -void registerFunctionsConversion(FunctionFactory &); -void registerFunctionCastOrDefault(FunctionFactory &); -void registerFunctionsDateTime(FunctionFactory &); -void registerFunctionsEmbeddedDictionaries(FunctionFactory &); -void registerFunctionsExternalDictionaries(FunctionFactory &); -void registerFunctionsExternalModels(FunctionFactory &); -void registerFunctionsFormatting(FunctionFactory &); -void registerFunctionHashID(FunctionFactory &); -void registerFunctionsHashing(FunctionFactory &); -void registerFunctionsHigherOrder(FunctionFactory &); -void registerFunctionsLogical(FunctionFactory &); -void registerFunctionsMiscellaneous(FunctionFactory &); -void registerFunctionsRandom(FunctionFactory &); -void registerFunctionsReinterpret(FunctionFactory &); -void registerFunctionsRound(FunctionFactory &); -void registerFunctionsString(FunctionFactory &); -void registerFunctionsStringArray(FunctionFactory &); -void registerFunctionsStringSearch(FunctionFactory &); -void registerFunctionsStringRegexp(FunctionFactory &); -void registerFunctionsStringSimilarity(FunctionFactory &); -void registerFunctionsStringTokenExtractor(FunctionFactory &); -void registerFunctionsURL(FunctionFactory &); -void registerFunctionsVisitParam(FunctionFactory &); -void registerFunctionsMath(FunctionFactory &); -void registerFunctionsGeo(FunctionFactory &); -void registerFunctionsIntrospection(FunctionFactory &); -void registerFunctionsNull(FunctionFactory &); -void registerFunctionsJSON(FunctionFactory &); -void registerFunctionsSQLJSON(FunctionFactory &); -void registerFunctionToJSONString(FunctionFactory &); -void registerFunctionsConsistentHashing(FunctionFactory & factory); -void registerFunctionsUnixTimestamp64(FunctionFactory & factory); -void registerFunctionBitHammingDistance(FunctionFactory & factory); -void registerFunctionTupleHammingDistance(FunctionFactory & factory); -void registerFunctionsStringHash(FunctionFactory & factory); -void registerFunctionValidateNestedArraySizes(FunctionFactory & factory); -void registerFunctionsSnowflake(FunctionFactory & factory); -void registerFunctionTid(FunctionFactory & factory); -void registerFunctionLogTrace(FunctionFactory & factory); -void registerFunctionsTimeWindow(FunctionFactory &); -void registerFunctionToBool(FunctionFactory &); -void registerFunctionMinSampleSize(FunctionFactory &); - -/// For Meilisearch -void registerFunctionMeiliMatch(FunctionFactory & factory); - -#if USE_SSL -void registerFunctionEncrypt(FunctionFactory & factory); -void registerFunctionDecrypt(FunctionFactory & factory); -void registerFunctionAESEncryptMysql(FunctionFactory & factory); -void registerFunctionAESDecryptMysql(FunctionFactory & factory); -void registerFunctionShowCertificate(FunctionFactory &); -#endif void registerFunctions() { auto & factory = FunctionFactory::instance(); - registerFunctionsArithmetic(factory); - registerFunctionsArray(factory); - registerFunctionsTuple(factory); - registerFunctionsMakeDate(factory); - registerFunctionsMap(factory); - registerFunctionsBitmap(factory); - registerFunctionsBinaryRepr(factory); - registerFunctionsCoding(factory); - registerFunctionsCodingUUID(factory); - registerFunctionChar(factory); - registerFunctionsComparison(factory); - registerFunctionsConditional(factory); - registerFunctionsConversion(factory); - registerFunctionCastOrDefault(factory); - registerFunctionsDateTime(factory); - registerFunctionsEmbeddedDictionaries(factory); - registerFunctionsExternalDictionaries(factory); - registerFunctionsExternalModels(factory); - registerFunctionsFormatting(factory); - registerFunctionsHashing(factory); - registerFunctionsHigherOrder(factory); - registerFunctionsLogical(factory); - registerFunctionsMiscellaneous(factory); - registerFunctionsRandom(factory); - registerFunctionsReinterpret(factory); - registerFunctionsRound(factory); - registerFunctionsString(factory); - registerFunctionsStringArray(factory); - registerFunctionsStringSearch(factory); - registerFunctionsStringRegexp(factory); - registerFunctionsStringSimilarity(factory); - registerFunctionsStringTokenExtractor(factory); - registerFunctionsURL(factory); - registerFunctionsVisitParam(factory); - registerFunctionsMath(factory); - registerFunctionsGeo(factory); - registerFunctionsNull(factory); - registerFunctionsJSON(factory); - registerFunctionsSQLJSON(factory); - registerFunctionToJSONString(factory); - registerFunctionsIntrospection(factory); - registerFunctionsConsistentHashing(factory); - registerFunctionsUnixTimestamp64(factory); - registerFunctionBitHammingDistance(factory); - registerFunctionTupleHammingDistance(factory); - registerFunctionsStringHash(factory); - registerFunctionValidateNestedArraySizes(factory); - registerFunctionsSnowflake(factory); - registerFunctionsTimeWindow(factory); - registerFunctionToBool(factory); - registerFunctionMinSampleSize(factory); - registerFunctionTid(factory); - registerFunctionLogTrace(factory); - registerFunctionHashID(factory); - - /// For Meilisearch - registerFunctionMeiliMatch(factory); - -#if USE_SSL - registerFunctionEncrypt(factory); - registerFunctionDecrypt(factory); - registerFunctionAESEncryptMysql(factory); - registerFunctionAESDecryptMysql(factory); - registerFunctionShowCertificate(factory); -#endif + for (const auto & [_, reg] : FunctionRegisterMap::instance()) + reg(factory); } } diff --git a/src/Functions/registerFunctionsArithmetic.cpp b/src/Functions/registerFunctionsArithmetic.cpp deleted file mode 100644 index 96e77d34882..00000000000 --- a/src/Functions/registerFunctionsArithmetic.cpp +++ /dev/null @@ -1,92 +0,0 @@ -namespace DB -{ - -class FunctionFactory; - -void registerFunctionPlus(FunctionFactory & factory); -void registerFunctionMinus(FunctionFactory & factory); -void registerFunctionMultiply(FunctionFactory & factory); -void registerFunctionDivide(FunctionFactory & factory); -void registerFunctionIntDiv(FunctionFactory & factory); -void registerFunctionIntDivOrZero(FunctionFactory & factory); -void registerFunctionModulo(FunctionFactory & factory); -void registerFunctionModuloOrZero(FunctionFactory & factory); -void registerFunctionModuloLegacy(FunctionFactory & factory); -void registerFunctionNegate(FunctionFactory & factory); -void registerFunctionAbs(FunctionFactory & factory); -void registerFunctionBitAnd(FunctionFactory & factory); -void registerFunctionBitOr(FunctionFactory & factory); -void registerFunctionBitXor(FunctionFactory & factory); -void registerFunctionBitNot(FunctionFactory & factory); -void registerFunctionBitShiftLeft(FunctionFactory & factory); -void registerFunctionBitShiftRight(FunctionFactory & factory); -void registerFunctionBitSlice(FunctionFactory & factory); -void registerFunctionBitRotateLeft(FunctionFactory & factory); -void registerFunctionBitRotateRight(FunctionFactory & factory); -void registerFunctionBitCount(FunctionFactory & factory); -void registerFunctionLeast(FunctionFactory & factory); -void registerFunctionGreatest(FunctionFactory & factory); -void registerFunctionBitTest(FunctionFactory & factory); -void registerFunctionBitTestAny(FunctionFactory & factory); -void registerFunctionBitTestAll(FunctionFactory & factory); -void registerFunctionGCD(FunctionFactory & factory); -void registerFunctionLCM(FunctionFactory & factory); -void registerFunctionIntExp2(FunctionFactory & factory); -void registerFunctionIntExp10(FunctionFactory & factory); -void registerFunctionRoundToExp2(FunctionFactory & factory); -void registerFunctionRoundDuration(FunctionFactory & factory); -void registerFunctionRoundAge(FunctionFactory & factory); - -void registerFunctionBitBoolMaskOr(FunctionFactory & factory); -void registerFunctionBitBoolMaskAnd(FunctionFactory & factory); -void registerFunctionBitWrapperFunc(FunctionFactory & factory); -void registerFunctionBitSwapLastTwo(FunctionFactory & factory); -void registerFunctionZTest(FunctionFactory & factory); - - -void registerFunctionsArithmetic(FunctionFactory & factory) -{ - registerFunctionPlus(factory); - registerFunctionMinus(factory); - registerFunctionMultiply(factory); - registerFunctionDivide(factory); - registerFunctionIntDiv(factory); - registerFunctionIntDivOrZero(factory); - registerFunctionModulo(factory); - registerFunctionModuloOrZero(factory); - registerFunctionModuloLegacy(factory); - registerFunctionNegate(factory); - registerFunctionAbs(factory); - registerFunctionBitAnd(factory); - registerFunctionBitOr(factory); - registerFunctionBitXor(factory); - registerFunctionBitNot(factory); - registerFunctionBitShiftLeft(factory); - registerFunctionBitShiftRight(factory); - registerFunctionBitRotateLeft(factory); - registerFunctionBitRotateRight(factory); - registerFunctionBitCount(factory); - registerFunctionBitSlice(factory); - registerFunctionLeast(factory); - registerFunctionGreatest(factory); - registerFunctionBitTest(factory); - registerFunctionBitTestAny(factory); - registerFunctionBitTestAll(factory); - registerFunctionGCD(factory); - registerFunctionLCM(factory); - registerFunctionIntExp2(factory); - registerFunctionIntExp10(factory); - registerFunctionRoundToExp2(factory); - registerFunctionRoundDuration(factory); - registerFunctionRoundAge(factory); - - /// Not for external use. - registerFunctionBitBoolMaskOr(factory); - registerFunctionBitBoolMaskAnd(factory); - registerFunctionBitWrapperFunc(factory); - registerFunctionBitSwapLastTwo(factory); - - registerFunctionZTest(factory); -} - -} diff --git a/src/Functions/registerFunctionsComparison.cpp b/src/Functions/registerFunctionsComparison.cpp deleted file mode 100644 index af5cbed6191..00000000000 --- a/src/Functions/registerFunctionsComparison.cpp +++ /dev/null @@ -1,24 +0,0 @@ -namespace DB -{ - -class FunctionFactory; - -void registerFunctionEquals(FunctionFactory & factory); -void registerFunctionNotEquals(FunctionFactory & factory); -void registerFunctionLess(FunctionFactory & factory); -void registerFunctionGreater(FunctionFactory & factory); -void registerFunctionLessOrEquals(FunctionFactory & factory); -void registerFunctionGreaterOrEquals(FunctionFactory & factory); - - -void registerFunctionsComparison(FunctionFactory & factory) -{ - registerFunctionEquals(factory); - registerFunctionNotEquals(factory); - registerFunctionLess(factory); - registerFunctionGreater(factory); - registerFunctionLessOrEquals(factory); - registerFunctionGreaterOrEquals(factory); -} - -} diff --git a/src/Functions/registerFunctionsConditional.cpp b/src/Functions/registerFunctionsConditional.cpp deleted file mode 100644 index d58d2508dee..00000000000 --- a/src/Functions/registerFunctionsConditional.cpp +++ /dev/null @@ -1,18 +0,0 @@ -namespace DB -{ - -class FunctionFactory; - -void registerFunctionIf(FunctionFactory & factory); -void registerFunctionMultiIf(FunctionFactory & factory); -void registerFunctionCaseWithExpression(FunctionFactory & factory); - - -void registerFunctionsConditional(FunctionFactory & factory) -{ - registerFunctionIf(factory); - registerFunctionMultiIf(factory); - registerFunctionCaseWithExpression(factory); -} - -} diff --git a/src/Functions/registerFunctionsConsistentHashing.cpp b/src/Functions/registerFunctionsConsistentHashing.cpp deleted file mode 100644 index 84a78cd6765..00000000000 --- a/src/Functions/registerFunctionsConsistentHashing.cpp +++ /dev/null @@ -1,14 +0,0 @@ -namespace DB -{ -class FunctionFactory; - -void registerFunctionKostikConsistentHash(FunctionFactory & factory); -void registerFunctionJumpConsistentHash(FunctionFactory & factory); - -void registerFunctionsConsistentHashing(FunctionFactory & factory) -{ - registerFunctionKostikConsistentHash(factory); - registerFunctionJumpConsistentHash(factory); -} - -} diff --git a/src/Functions/registerFunctionsDateTime.cpp b/src/Functions/registerFunctionsDateTime.cpp deleted file mode 100644 index b3fcca42bed..00000000000 --- a/src/Functions/registerFunctionsDateTime.cpp +++ /dev/null @@ -1,170 +0,0 @@ -namespace DB -{ - -class FunctionFactory; - -void registerFunctionToYear(FunctionFactory &); -void registerFunctionToQuarter(FunctionFactory &); -void registerFunctionToMonth(FunctionFactory &); -void registerFunctionToDayOfMonth(FunctionFactory &); -void registerFunctionToDayOfWeek(FunctionFactory &); -void registerFunctionToDayOfYear(FunctionFactory &); -void registerFunctionToHour(FunctionFactory &); -void registerFunctionToMinute(FunctionFactory &); -void registerFunctionToStartOfNanosecond(FunctionFactory &); -void registerFunctionToStartOfMicrosecond(FunctionFactory &); -void registerFunctionToStartOfMillisecond(FunctionFactory &); -void registerFunctionToStartOfSecond(FunctionFactory &); -void registerFunctionToSecond(FunctionFactory &); -void registerFunctionToStartOfDay(FunctionFactory &); -void registerFunctionToMonday(FunctionFactory &); -void registerFunctionToISOWeek(FunctionFactory &); -void registerFunctionToISOYear(FunctionFactory &); -void registerFunctionToCustomWeek(FunctionFactory &); -void registerFunctionToModifiedJulianDay(FunctionFactory &); -void registerFunctionToStartOfMonth(FunctionFactory &); -void registerFunctionToLastDayOfMonth(FunctionFactory &); -void registerFunctionToStartOfQuarter(FunctionFactory &); -void registerFunctionToStartOfYear(FunctionFactory &); -void registerFunctionToStartOfMinute(FunctionFactory &); -void registerFunctionToStartOfFiveMinutes(FunctionFactory &); -void registerFunctionToStartOfTenMinutes(FunctionFactory &); -void registerFunctionToStartOfFifteenMinutes(FunctionFactory &); -void registerFunctionToStartOfHour(FunctionFactory &); -void registerFunctionToStartOfInterval(FunctionFactory &); -void registerFunctionToStartOfISOYear(FunctionFactory &); -void registerFunctionToRelativeYearNum(FunctionFactory &); -void registerFunctionToRelativeQuarterNum(FunctionFactory &); -void registerFunctionToRelativeMonthNum(FunctionFactory &); -void registerFunctionToRelativeWeekNum(FunctionFactory &); -void registerFunctionToRelativeDayNum(FunctionFactory &); -void registerFunctionToRelativeHourNum(FunctionFactory &); -void registerFunctionToRelativeMinuteNum(FunctionFactory &); -void registerFunctionToRelativeSecondNum(FunctionFactory &); -void registerFunctionToTime(FunctionFactory &); -void registerFunctionNow(FunctionFactory &); -void registerFunctionNowInBlock(FunctionFactory &); -void registerFunctionNow64(FunctionFactory &); -void registerFunctionToday(FunctionFactory &); -void registerFunctionYesterday(FunctionFactory &); -void registerFunctionTimeSlot(FunctionFactory &); -void registerFunctionTimeSlots(FunctionFactory &); -void registerFunctionToYYYYMM(FunctionFactory &); -void registerFunctionToYYYYMMDD(FunctionFactory &); -void registerFunctionToYYYYMMDDhhmmss(FunctionFactory &); -void registerFunctionAddNanoseconds(FunctionFactory &); -void registerFunctionAddMicroseconds(FunctionFactory &); -void registerFunctionAddMilliseconds(FunctionFactory &); -void registerFunctionAddSeconds(FunctionFactory &); -void registerFunctionAddMinutes(FunctionFactory &); -void registerFunctionAddHours(FunctionFactory &); -void registerFunctionAddDays(FunctionFactory &); -void registerFunctionAddWeeks(FunctionFactory &); -void registerFunctionAddMonths(FunctionFactory &); -void registerFunctionAddQuarters(FunctionFactory &); -void registerFunctionAddYears(FunctionFactory &); -void registerFunctionSubtractNanoseconds(FunctionFactory &); -void registerFunctionSubtractMicroseconds(FunctionFactory &); -void registerFunctionSubtractMilliseconds(FunctionFactory &); -void registerFunctionSubtractSeconds(FunctionFactory &); -void registerFunctionSubtractMinutes(FunctionFactory &); -void registerFunctionSubtractHours(FunctionFactory &); -void registerFunctionSubtractDays(FunctionFactory &); -void registerFunctionSubtractWeeks(FunctionFactory &); -void registerFunctionSubtractMonths(FunctionFactory &); -void registerFunctionSubtractQuarters(FunctionFactory &); -void registerFunctionSubtractYears(FunctionFactory &); -void registerFunctionDateDiff(FunctionFactory &); -void registerFunctionDateName(FunctionFactory &); -void registerFunctionMonthName(FunctionFactory &); -void registerFunctionToTimeZone(FunctionFactory &); -void registerFunctionFormatDateTime(FunctionFactory &); -void registerFunctionFromModifiedJulianDay(FunctionFactory &); -void registerFunctionDateTrunc(FunctionFactory &); - -void registerFunctiontimezoneOffset(FunctionFactory &); - -void registerFunctionsDateTime(FunctionFactory & factory) -{ - registerFunctionToYear(factory); - registerFunctionToQuarter(factory); - registerFunctionToMonth(factory); - registerFunctionToDayOfMonth(factory); - registerFunctionToDayOfWeek(factory); - registerFunctionToDayOfYear(factory); - registerFunctionToHour(factory); - registerFunctionToMinute(factory); - registerFunctionToSecond(factory); - registerFunctionToStartOfDay(factory); - registerFunctionToMonday(factory); - registerFunctionToISOWeek(factory); - registerFunctionToISOYear(factory); - registerFunctionToCustomWeek(factory); - registerFunctionToModifiedJulianDay(factory); - registerFunctionToStartOfMonth(factory); - registerFunctionToLastDayOfMonth(factory); - registerFunctionToStartOfQuarter(factory); - registerFunctionToStartOfYear(factory); - registerFunctionToStartOfNanosecond(factory); - registerFunctionToStartOfMicrosecond(factory); - registerFunctionToStartOfMillisecond(factory); - registerFunctionToStartOfSecond(factory); - registerFunctionToStartOfMinute(factory); - registerFunctionToStartOfFiveMinutes(factory); - registerFunctionToStartOfTenMinutes(factory); - registerFunctionToStartOfFifteenMinutes(factory); - registerFunctionToStartOfHour(factory); - registerFunctionToStartOfInterval(factory); - registerFunctionToStartOfISOYear(factory); - registerFunctionToRelativeYearNum(factory); - registerFunctionToRelativeQuarterNum(factory); - registerFunctionToRelativeMonthNum(factory); - registerFunctionToRelativeWeekNum(factory); - registerFunctionToRelativeDayNum(factory); - registerFunctionToRelativeHourNum(factory); - registerFunctionToRelativeMinuteNum(factory); - registerFunctionToRelativeSecondNum(factory); - registerFunctionToTime(factory); - registerFunctionNow(factory); - registerFunctionNow64(factory); - registerFunctionNowInBlock(factory); - registerFunctionToday(factory); - registerFunctionYesterday(factory); - registerFunctionTimeSlot(factory); - registerFunctionTimeSlots(factory); - registerFunctionToYYYYMM(factory); - registerFunctionToYYYYMMDD(factory); - registerFunctionToYYYYMMDDhhmmss(factory); - registerFunctionAddNanoseconds(factory); - registerFunctionAddMicroseconds(factory); - registerFunctionAddMilliseconds(factory); - registerFunctionAddSeconds(factory); - registerFunctionAddMinutes(factory); - registerFunctionAddHours(factory); - registerFunctionAddDays(factory); - registerFunctionAddWeeks(factory); - registerFunctionAddMonths(factory); - registerFunctionAddQuarters(factory); - registerFunctionAddYears(factory); - registerFunctionSubtractNanoseconds(factory); - registerFunctionSubtractMicroseconds(factory); - registerFunctionSubtractMilliseconds(factory); - registerFunctionSubtractSeconds(factory); - registerFunctionSubtractMinutes(factory); - registerFunctionSubtractHours(factory); - registerFunctionSubtractDays(factory); - registerFunctionSubtractWeeks(factory); - registerFunctionSubtractMonths(factory); - registerFunctionSubtractQuarters(factory); - registerFunctionSubtractYears(factory); - registerFunctionDateDiff(factory); - registerFunctionDateName(factory); - registerFunctionMonthName(factory); - registerFunctionToTimeZone(factory); - registerFunctionFormatDateTime(factory); - registerFunctionFromModifiedJulianDay(factory); - registerFunctionDateTrunc(factory); - registerFunctiontimezoneOffset(factory); -} - -} diff --git a/src/Functions/registerFunctionsFormatting.cpp b/src/Functions/registerFunctionsFormatting.cpp deleted file mode 100644 index 02353fac812..00000000000 --- a/src/Functions/registerFunctionsFormatting.cpp +++ /dev/null @@ -1,21 +0,0 @@ -namespace DB -{ - -class FunctionFactory; - -void registerFunctionsBitToArray(FunctionFactory &); -void registerFunctionFormatReadableSize(FunctionFactory &); -void registerFunctionFormatReadableQuantity(FunctionFactory &); -void registerFunctionFormatReadableTimeDelta(FunctionFactory &); -void registerFunctionParseTimeDelta(FunctionFactory &); - -void registerFunctionsFormatting(FunctionFactory & factory) -{ - registerFunctionsBitToArray(factory); - registerFunctionFormatReadableSize(factory); - registerFunctionFormatReadableQuantity(factory); - registerFunctionFormatReadableTimeDelta(factory); - registerFunctionParseTimeDelta(factory); -} - -} diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp deleted file mode 100644 index 006693a93be..00000000000 --- a/src/Functions/registerFunctionsGeo.cpp +++ /dev/null @@ -1,166 +0,0 @@ -#include "config_functions.h" - -namespace DB -{ - -class FunctionFactory; - -void registerFunctionGeoDistance(FunctionFactory & factory); -void registerFunctionPointInEllipses(FunctionFactory & factory); -void registerFunctionPointInPolygon(FunctionFactory & factory); -void registerFunctionPolygonsIntersection(FunctionFactory & factory); -void registerFunctionPolygonsUnion(FunctionFactory & factory); -void registerFunctionPolygonArea(FunctionFactory & factory); -void registerFunctionPolygonConvexHull(FunctionFactory & factory); -void registerFunctionPolygonsSymDifference(FunctionFactory & factory); -void registerFunctionPolygonsEquals(FunctionFactory & factory); -void registerFunctionPolygonsDistance(FunctionFactory & factory); -void registerFunctionPolygonsWithin(FunctionFactory & factory); -void registerFunctionPolygonPerimeter(FunctionFactory & factory); -void registerFunctionGeohashEncode(FunctionFactory & factory); -void registerFunctionGeohashDecode(FunctionFactory & factory); -void registerFunctionGeohashesInBox(FunctionFactory & factory); -void registerFunctionWkt(FunctionFactory & factory); -void registerFunctionReadWKT(FunctionFactory & factory); -void registerFunctionSvg(FunctionFactory & factory); - -#if USE_H3 -void registerFunctionGeoToH3(FunctionFactory &); -void registerFunctionH3ToGeo(FunctionFactory &); -void registerFunctionH3ToGeoBoundary(FunctionFactory &); -void registerFunctionH3EdgeAngle(FunctionFactory &); -void registerFunctionH3EdgeLengthM(FunctionFactory &); -void registerFunctionH3EdgeLengthKm(FunctionFactory &); -void registerFunctionH3ExactEdgeLengthM(FunctionFactory &); -void registerFunctionH3ExactEdgeLengthKm(FunctionFactory &); -void registerFunctionH3ExactEdgeLengthRads(FunctionFactory &); -void registerFunctionH3GetResolution(FunctionFactory &); -void registerFunctionH3IsValid(FunctionFactory &); -void registerFunctionH3KRing(FunctionFactory &); -void registerFunctionH3GetBaseCell(FunctionFactory &); -void registerFunctionH3ToParent(FunctionFactory &); -void registerFunctionH3ToChildren(FunctionFactory &); -void registerFunctionH3ToCenterChild(FunctionFactory &); -void registerFunctionH3IndexesAreNeighbors(FunctionFactory &); -void registerFunctionStringToH3(FunctionFactory &); -void registerFunctionH3ToString(FunctionFactory &); -void registerFunctionH3HexAreaM2(FunctionFactory &); -void registerFunctionH3IsResClassIII(FunctionFactory &); -void registerFunctionH3IsPentagon(FunctionFactory &); -void registerFunctionH3GetFaces(FunctionFactory &); -void registerFunctionH3HexAreaKm2(FunctionFactory &); -void registerFunctionH3CellAreaM2(FunctionFactory &); -void registerFunctionH3CellAreaRads2(FunctionFactory &); -void registerFunctionH3NumHexagons(FunctionFactory &); -void registerFunctionH3PointDistM(FunctionFactory &); -void registerFunctionH3PointDistKm(FunctionFactory &); -void registerFunctionH3PointDistRads(FunctionFactory &); -void registerFunctionH3GetRes0Indexes(FunctionFactory &); -void registerFunctionH3GetPentagonIndexes(FunctionFactory &); -void registerFunctionH3Line(FunctionFactory &); -void registerFunctionH3Distance(FunctionFactory &); -void registerFunctionH3HexRing(FunctionFactory &); -void registerFunctionH3GetUnidirectionalEdge(FunctionFactory &); -void registerFunctionH3UnidirectionalEdgeIsValid(FunctionFactory &); -void registerFunctionH3GetOriginIndexFromUnidirectionalEdge(FunctionFactory &); -void registerFunctionH3GetDestinationIndexFromUnidirectionalEdge(FunctionFactory &); -void registerFunctionH3GetIndexesFromUnidirectionalEdge(FunctionFactory &); -void registerFunctionH3GetUnidirectionalEdgesFromHexagon(FunctionFactory &); -void registerFunctionH3GetUnidirectionalEdgeBoundary(FunctionFactory &); -#endif - -#if USE_S2_GEOMETRY -void registerFunctionGeoToS2(FunctionFactory &); -void registerFunctionS2ToGeo(FunctionFactory &); -void registerFunctionS2GetNeighbors(FunctionFactory &); -void registerFunctionS2CellsIntersect(FunctionFactory &); -void registerFunctionS2CapContains(FunctionFactory &); -void registerFunctionS2CapUnion(FunctionFactory &); -void registerFunctionS2RectAdd(FunctionFactory &); -void registerFunctionS2RectContains(FunctionFactory &); -void registerFunctionS2RectUnion(FunctionFactory &); -void registerFunctionS2RectIntersection(FunctionFactory &); -#endif - - -void registerFunctionsGeo(FunctionFactory & factory) -{ - registerFunctionGeoDistance(factory); - registerFunctionPointInEllipses(factory); - registerFunctionPointInPolygon(factory); - registerFunctionPolygonsIntersection(factory); - registerFunctionPolygonsUnion(factory); - registerFunctionPolygonArea(factory); - registerFunctionPolygonConvexHull(factory); - registerFunctionPolygonsSymDifference(factory); - registerFunctionPolygonsEquals(factory); - registerFunctionPolygonsDistance(factory); - registerFunctionPolygonsWithin(factory); - registerFunctionPolygonPerimeter(factory); - registerFunctionGeohashEncode(factory); - registerFunctionGeohashDecode(factory); - registerFunctionGeohashesInBox(factory); - registerFunctionWkt(factory); - registerFunctionReadWKT(factory); - registerFunctionSvg(factory); - -#if USE_H3 - registerFunctionGeoToH3(factory); - registerFunctionH3ToGeo(factory); - registerFunctionH3ToGeoBoundary(factory); - registerFunctionH3EdgeAngle(factory); - registerFunctionH3EdgeLengthM(factory); - registerFunctionH3EdgeLengthKm(factory); - registerFunctionH3ExactEdgeLengthM(factory); - registerFunctionH3ExactEdgeLengthKm(factory); - registerFunctionH3ExactEdgeLengthRads(factory); - registerFunctionH3GetResolution(factory); - registerFunctionH3IsValid(factory); - registerFunctionH3KRing(factory); - registerFunctionH3GetBaseCell(factory); - registerFunctionH3ToParent(factory); - registerFunctionH3ToChildren(factory); - registerFunctionH3ToCenterChild(factory); - registerFunctionH3IndexesAreNeighbors(factory); - registerFunctionStringToH3(factory); - registerFunctionH3ToString(factory); - registerFunctionH3HexAreaM2(factory); - registerFunctionH3IsResClassIII(factory); - registerFunctionH3IsPentagon(factory); - registerFunctionH3GetFaces(factory); - registerFunctionH3HexAreaKm2(factory); - registerFunctionH3CellAreaM2(factory); - registerFunctionH3CellAreaRads2(factory); - registerFunctionH3NumHexagons(factory); - registerFunctionH3PointDistM(factory); - registerFunctionH3PointDistKm(factory); - registerFunctionH3PointDistRads(factory); - registerFunctionH3GetRes0Indexes(factory); - registerFunctionH3GetPentagonIndexes(factory); - registerFunctionH3Line(factory); - registerFunctionH3Distance(factory); - registerFunctionH3HexRing(factory); - registerFunctionH3GetUnidirectionalEdge(factory); - registerFunctionH3UnidirectionalEdgeIsValid(factory); - registerFunctionH3GetOriginIndexFromUnidirectionalEdge(factory); - registerFunctionH3GetDestinationIndexFromUnidirectionalEdge(factory); - registerFunctionH3GetIndexesFromUnidirectionalEdge(factory); - registerFunctionH3GetUnidirectionalEdgesFromHexagon(factory); - registerFunctionH3GetUnidirectionalEdgeBoundary(factory); -#endif - -#if USE_S2_GEOMETRY - registerFunctionGeoToS2(factory); - registerFunctionS2ToGeo(factory); - registerFunctionS2GetNeighbors(factory); - registerFunctionS2CellsIntersect(factory); - registerFunctionS2CapContains(factory); - registerFunctionS2CapUnion(factory); - registerFunctionS2RectAdd(factory); - registerFunctionS2RectContains(factory); - registerFunctionS2RectUnion(factory); - registerFunctionS2RectIntersection(factory); -#endif -} - -} diff --git a/src/Functions/registerFunctionsHigherOrder.cpp b/src/Functions/registerFunctionsHigherOrder.cpp deleted file mode 100644 index 00bea58b918..00000000000 --- a/src/Functions/registerFunctionsHigherOrder.cpp +++ /dev/null @@ -1,43 +0,0 @@ -namespace DB -{ - -class FunctionFactory; - -void registerFunctionArrayMap(FunctionFactory & factory); -void registerFunctionArrayFilter(FunctionFactory & factory); -void registerFunctionArrayCount(FunctionFactory & factory); -void registerFunctionArrayExists(FunctionFactory & factory); -void registerFunctionArrayAll(FunctionFactory & factory); -void registerFunctionArrayCompact(FunctionFactory & factory); -void registerFunctionArrayAggregation(FunctionFactory & factory); -void registerFunctionArrayFirst(FunctionFactory & factory); -void registerFunctionArrayFirstIndex(FunctionFactory & factory); -void registerFunctionsArrayFill(FunctionFactory & factory); -void registerFunctionsArraySplit(FunctionFactory & factory); -void registerFunctionsArraySort(FunctionFactory & factory); -void registerFunctionArrayCumSum(FunctionFactory & factory); -void registerFunctionArrayCumSumNonNegative(FunctionFactory & factory); -void registerFunctionArrayDifference(FunctionFactory & factory); -void registerFunctionMapApply(FunctionFactory & factory); - -void registerFunctionsHigherOrder(FunctionFactory & factory) -{ - registerFunctionArrayMap(factory); - registerFunctionArrayFilter(factory); - registerFunctionArrayCount(factory); - registerFunctionArrayExists(factory); - registerFunctionArrayAll(factory); - registerFunctionArrayCompact(factory); - registerFunctionArrayAggregation(factory); - registerFunctionArrayFirst(factory); - registerFunctionArrayFirstIndex(factory); - registerFunctionsArrayFill(factory); - registerFunctionsArraySplit(factory); - registerFunctionsArraySort(factory); - registerFunctionArrayCumSum(factory); - registerFunctionArrayCumSumNonNegative(factory); - registerFunctionArrayDifference(factory); - registerFunctionMapApply(factory); -} - -} diff --git a/src/Functions/registerFunctionsIntrospection.cpp b/src/Functions/registerFunctionsIntrospection.cpp deleted file mode 100644 index 76a92847d8e..00000000000 --- a/src/Functions/registerFunctionsIntrospection.cpp +++ /dev/null @@ -1,27 +0,0 @@ -namespace DB -{ - -class FunctionFactory; - -#if defined(OS_LINUX) -void registerFunctionAddressToSymbol(FunctionFactory & factory); -void registerFunctionAddressToLine(FunctionFactory & factory); -void registerFunctionAddressToLineWithInlines(FunctionFactory & factory); -#endif - -void registerFunctionDemangle(FunctionFactory & factory); -void registerFunctionTrap(FunctionFactory & factory); - - -void registerFunctionsIntrospection(FunctionFactory & factory) -{ -#if defined(OS_LINUX) - registerFunctionAddressToSymbol(factory); - registerFunctionAddressToLine(factory); - registerFunctionAddressToLineWithInlines(factory); -#endif - registerFunctionDemangle(factory); - registerFunctionTrap(factory); -} - -} diff --git a/src/Functions/registerFunctionsMath.cpp b/src/Functions/registerFunctionsMath.cpp deleted file mode 100644 index 865ec23174a..00000000000 --- a/src/Functions/registerFunctionsMath.cpp +++ /dev/null @@ -1,85 +0,0 @@ -namespace DB -{ -class FunctionFactory; - -void registerFunctionE(FunctionFactory & factory); -void registerFunctionPi(FunctionFactory & factory); -void registerFunctionExp(FunctionFactory & factory); -void registerFunctionLog(FunctionFactory & factory); -void registerFunctionExp2(FunctionFactory & factory); -void registerFunctionLog2(FunctionFactory & factory); -void registerFunctionLog1p(FunctionFactory & factory); -void registerFunctionExp10(FunctionFactory & factory); -void registerFunctionLog10(FunctionFactory & factory); -void registerFunctionSqrt(FunctionFactory & factory); -void registerFunctionCbrt(FunctionFactory & factory); -void registerFunctionErf(FunctionFactory & factory); -void registerFunctionErfc(FunctionFactory & factory); -void registerFunctionLGamma(FunctionFactory & factory); -void registerFunctionTGamma(FunctionFactory & factory); -void registerFunctionSin(FunctionFactory & factory); -void registerFunctionCos(FunctionFactory & factory); -void registerFunctionTan(FunctionFactory & factory); -void registerFunctionAsin(FunctionFactory & factory); -void registerFunctionAcos(FunctionFactory & factory); -void registerFunctionAtan(FunctionFactory & factory); -void registerFunctionAtan2(FunctionFactory & factory); -void registerFunctionSigmoid(FunctionFactory & factory); -void registerFunctionHypot(FunctionFactory & factory); -void registerFunctionSinh(FunctionFactory & factory); -void registerFunctionCosh(FunctionFactory & factory); -void registerFunctionTanh(FunctionFactory & factory); -void registerFunctionAsinh(FunctionFactory & factory); -void registerFunctionAcosh(FunctionFactory & factory); -void registerFunctionAtanh(FunctionFactory & factory); -void registerFunctionPow(FunctionFactory & factory); -void registerFunctionSign(FunctionFactory & factory); -void registerFunctionMax2(FunctionFactory & factory); -void registerFunctionMin2(FunctionFactory & factory); -void registerVectorFunctions(FunctionFactory &); -void registerFunctionDegrees(FunctionFactory & factory); -void registerFunctionRadians(FunctionFactory & factory); - - -void registerFunctionsMath(FunctionFactory & factory) -{ - registerFunctionE(factory); - registerFunctionPi(factory); - registerFunctionExp(factory); - registerFunctionLog(factory); - registerFunctionExp2(factory); - registerFunctionLog2(factory); - registerFunctionLog1p(factory); - registerFunctionExp10(factory); - registerFunctionLog10(factory); - registerFunctionSqrt(factory); - registerFunctionCbrt(factory); - registerFunctionErf(factory); - registerFunctionErfc(factory); - registerFunctionLGamma(factory); - registerFunctionTGamma(factory); - registerFunctionSin(factory); - registerFunctionCos(factory); - registerFunctionTan(factory); - registerFunctionAsin(factory); - registerFunctionAcos(factory); - registerFunctionAtan(factory); - registerFunctionAtan2(factory); - registerFunctionSigmoid(factory); - registerFunctionHypot(factory); - registerFunctionSinh(factory); - registerFunctionCosh(factory); - registerFunctionTanh(factory); - registerFunctionAsinh(factory); - registerFunctionAcosh(factory); - registerFunctionAtanh(factory); - registerFunctionPow(factory); - registerFunctionSign(factory); - registerFunctionMax2(factory); - registerFunctionMin2(factory); - registerVectorFunctions(factory); - registerFunctionDegrees(factory); - registerFunctionRadians(factory); -} - -} diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp deleted file mode 100644 index 755d38409a6..00000000000 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ /dev/null @@ -1,187 +0,0 @@ -#include - -namespace DB -{ - -class FunctionFactory; - -void registerFunctionCurrentDatabase(FunctionFactory &); -void registerFunctionCurrentUser(FunctionFactory &); -void registerFunctionCurrentProfiles(FunctionFactory &); -void registerFunctionCurrentRoles(FunctionFactory &); -void registerFunctionHostName(FunctionFactory &); -void registerFunctionFQDN(FunctionFactory &); -void registerFunctionVisibleWidth(FunctionFactory &); -void registerFunctionToTypeName(FunctionFactory &); -void registerFunctionGetSizeOfEnumType(FunctionFactory &); -void registerFunctionBlockSerializedSize(FunctionFactory &); -void registerFunctionToColumnTypeName(FunctionFactory &); -void registerFunctionDumpColumnStructure(FunctionFactory &); -void registerFunctionDefaultValueOfArgumentType(FunctionFactory &); -void registerFunctionDefaultValueOfTypeName(FunctionFactory &); -void registerFunctionBlockSize(FunctionFactory &); -void registerFunctionBlockNumber(FunctionFactory &); -void registerFunctionRowNumberInBlock(FunctionFactory &); -void registerFunctionRowNumberInAllBlocks(FunctionFactory &); -void registerFunctionNeighbor(FunctionFactory &); -void registerFunctionSleep(FunctionFactory &); -void registerFunctionSleepEachRow(FunctionFactory &); -void registerFunctionMaterialize(FunctionFactory &); -void registerFunctionIgnore(FunctionFactory &); -void registerFunctionIndexHint(FunctionFactory &); -void registerFunctionIdentity(FunctionFactory &); -void registerFunctionArrayJoin(FunctionFactory &); -void registerFunctionReplicate(FunctionFactory &); -void registerFunctionBar(FunctionFactory &); -void registerFunctionHasColumnInTable(FunctionFactory &); -void registerFunctionIsFinite(FunctionFactory &); -void registerFunctionIsInfinite(FunctionFactory &); -void registerFunctionIsNaN(FunctionFactory &); -void registerFunctionIfNotFinite(FunctionFactory &); -void registerFunctionThrowIf(FunctionFactory &); -void registerFunctionVersion(FunctionFactory &); -void registerFunctionRevision(FunctionFactory &); -void registerFunctionBuildId(FunctionFactory &); -void registerFunctionUptime(FunctionFactory &); -void registerFunctionTimezone(FunctionFactory &); -void registerFunctionTimezoneOf(FunctionFactory &); -void registerFunctionRunningAccumulate(FunctionFactory &); -void registerFunctionRunningDifference(FunctionFactory &); -void registerFunctionRunningDifferenceStartingWithFirstValue(FunctionFactory &); -void registerFunctionRunningConcurrency(FunctionFactory &); -void registerFunctionFinalizeAggregation(FunctionFactory &); -void registerFunctionToLowCardinality(FunctionFactory &); -void registerFunctionLowCardinalityIndices(FunctionFactory &); -void registerFunctionLowCardinalityKeys(FunctionFactory &); -void registerFunctionsIn(FunctionFactory &); -void registerFunctionJoinGet(FunctionFactory &); -void registerFunctionFilesystem(FunctionFactory &); -void registerFunctionEvalMLMethod(FunctionFactory &); -void registerFunctionBasename(FunctionFactory &); -void registerFunctionTransform(FunctionFactory &); -void registerFunctionGetMacro(FunctionFactory &); -void registerFunctionGetScalar(FunctionFactory &); -void registerFunctionGetSetting(FunctionFactory &); -void registerFunctionIsConstant(FunctionFactory &); -void registerFunctionIsDecimalOverflow(FunctionFactory &); -void registerFunctionCountDigits(FunctionFactory &); -void registerFunctionGlobalVariable(FunctionFactory &); -void registerFunctionHasThreadFuzzer(FunctionFactory &); -void registerFunctionInitializeAggregation(FunctionFactory &); -void registerFunctionErrorCodeToName(FunctionFactory &); -void registerFunctionTcpPort(FunctionFactory &); -void registerFunctionGetServerPort(FunctionFactory &); -void registerFunctionByteSize(FunctionFactory &); -void registerFunctionFile(FunctionFactory &); -void registerFunctionConnectionId(FunctionFactory &); -void registerFunctionPartitionId(FunctionFactory &); -void registerFunctionIsIPAddressContainedIn(FunctionFactory &); -void registerFunctionsTransactionCounters(FunctionFactory & factory); -void registerFunctionQueryID(FunctionFactory &); -void registerFunctionInitialQueryID(FunctionFactory &); -void registerFunctionServerUUID(FunctionFactory &); -void registerFunctionZooKeeperSessionUptime(FunctionFactory &); -void registerFunctionGetOSKernelVersion(FunctionFactory &); -void registerFunctionGetTypeSerializationStreams(FunctionFactory &); -void registerFunctionFlattenTuple(FunctionFactory &); - -#if USE_ICU -void registerFunctionConvertCharset(FunctionFactory &); -#endif - -#ifdef FUZZING_MODE -void registerFunctionGetFuzzerData(FunctionFactory & factory); -#endif - -void registerFunctionsMiscellaneous(FunctionFactory & factory) -{ - registerFunctionCurrentDatabase(factory); - registerFunctionCurrentUser(factory); - registerFunctionCurrentProfiles(factory); - registerFunctionCurrentRoles(factory); - registerFunctionHostName(factory); - registerFunctionFQDN(factory); - registerFunctionVisibleWidth(factory); - registerFunctionToTypeName(factory); - registerFunctionGetSizeOfEnumType(factory); - registerFunctionBlockSerializedSize(factory); - registerFunctionToColumnTypeName(factory); - registerFunctionDumpColumnStructure(factory); - registerFunctionDefaultValueOfArgumentType(factory); - registerFunctionDefaultValueOfTypeName(factory); - registerFunctionBlockSize(factory); - registerFunctionBlockNumber(factory); - registerFunctionRowNumberInBlock(factory); - registerFunctionRowNumberInAllBlocks(factory); - registerFunctionNeighbor(factory); - registerFunctionSleep(factory); - registerFunctionSleepEachRow(factory); - registerFunctionMaterialize(factory); - registerFunctionIgnore(factory); - registerFunctionIndexHint(factory); - registerFunctionIdentity(factory); - registerFunctionArrayJoin(factory); - registerFunctionReplicate(factory); - registerFunctionBar(factory); - registerFunctionHasColumnInTable(factory); - registerFunctionIsFinite(factory); - registerFunctionIsInfinite(factory); - registerFunctionIsNaN(factory); - registerFunctionIfNotFinite(factory); - registerFunctionThrowIf(factory); - registerFunctionVersion(factory); - registerFunctionRevision(factory); - registerFunctionBuildId(factory); - registerFunctionUptime(factory); - registerFunctionTimezone(factory); - registerFunctionTimezoneOf(factory); - registerFunctionRunningAccumulate(factory); - registerFunctionRunningDifference(factory); - registerFunctionRunningDifferenceStartingWithFirstValue(factory); - registerFunctionRunningConcurrency(factory); - registerFunctionFinalizeAggregation(factory); - registerFunctionToLowCardinality(factory); - registerFunctionLowCardinalityIndices(factory); - registerFunctionLowCardinalityKeys(factory); - registerFunctionsIn(factory); - registerFunctionJoinGet(factory); - registerFunctionFilesystem(factory); - registerFunctionEvalMLMethod(factory); - registerFunctionBasename(factory); - registerFunctionTransform(factory); - registerFunctionGetMacro(factory); - registerFunctionGetScalar(factory); - registerFunctionGetSetting(factory); - registerFunctionIsConstant(factory); - registerFunctionIsDecimalOverflow(factory); - registerFunctionCountDigits(factory); - registerFunctionGlobalVariable(factory); - registerFunctionHasThreadFuzzer(factory); - registerFunctionInitializeAggregation(factory); - registerFunctionErrorCodeToName(factory); - registerFunctionTcpPort(factory); - registerFunctionGetServerPort(factory); - registerFunctionByteSize(factory); - registerFunctionFile(factory); - registerFunctionConnectionId(factory); - registerFunctionPartitionId(factory); - registerFunctionIsIPAddressContainedIn(factory); - registerFunctionsTransactionCounters(factory); - registerFunctionQueryID(factory); - registerFunctionInitialQueryID(factory); - registerFunctionServerUUID(factory); - registerFunctionZooKeeperSessionUptime(factory); - registerFunctionGetOSKernelVersion(factory); - registerFunctionGetTypeSerializationStreams(factory); - registerFunctionFlattenTuple(factory); - -#if USE_ICU - registerFunctionConvertCharset(factory); -#endif - -#ifdef FUZZING_MODE - registerFunctionGetFuzzerData(factory); -#endif -} - -} diff --git a/src/Functions/registerFunctionsNull.cpp b/src/Functions/registerFunctionsNull.cpp deleted file mode 100644 index d94ee95d28f..00000000000 --- a/src/Functions/registerFunctionsNull.cpp +++ /dev/null @@ -1,30 +0,0 @@ -namespace DB -{ - -class FunctionFactory; - -void registerFunctionIsNull(FunctionFactory & factory); -void registerFunctionIsNotNull(FunctionFactory & factory); -void registerFunctionCoalesce(FunctionFactory & factory); -void registerFunctionIfNull(FunctionFactory & factory); -void registerFunctionNullIf(FunctionFactory & factory); -void registerFunctionAssumeNotNull(FunctionFactory & factory); -void registerFunctionToNullable(FunctionFactory & factory); -void registerFunctionIsZeroOrNull(FunctionFactory & factory); -void registerFunctionIsNullable(FunctionFactory & factory); - - -void registerFunctionsNull(FunctionFactory & factory) -{ - registerFunctionIsNull(factory); - registerFunctionIsNotNull(factory); - registerFunctionCoalesce(factory); - registerFunctionIfNull(factory); - registerFunctionNullIf(factory); - registerFunctionAssumeNotNull(factory); - registerFunctionToNullable(factory); - registerFunctionIsZeroOrNull(factory); - registerFunctionIsNullable(factory); -} - -} diff --git a/src/Functions/registerFunctionsRandom.cpp b/src/Functions/registerFunctionsRandom.cpp deleted file mode 100644 index 6a2cb82dc4c..00000000000 --- a/src/Functions/registerFunctionsRandom.cpp +++ /dev/null @@ -1,28 +0,0 @@ -namespace DB -{ -class FunctionFactory; - -void registerFunctionRand(FunctionFactory & factory); -void registerFunctionRand64(FunctionFactory & factory); -void registerFunctionRandConstant(FunctionFactory & factory); -void registerFunctionGenerateUUIDv4(FunctionFactory & factory); -void registerFunctionRandomPrintableASCII(FunctionFactory & factory); -void registerFunctionRandomString(FunctionFactory & factory); -void registerFunctionRandomFixedString(FunctionFactory & factory); -void registerFunctionRandomStringUTF8(FunctionFactory & factory); -void registerFunctionFuzzBits(FunctionFactory & factory); - -void registerFunctionsRandom(FunctionFactory & factory) -{ - registerFunctionRand(factory); - registerFunctionRand64(factory); - registerFunctionRandConstant(factory); - registerFunctionGenerateUUIDv4(factory); - registerFunctionRandomPrintableASCII(factory); - registerFunctionRandomString(factory); - registerFunctionRandomFixedString(factory); - registerFunctionRandomStringUTF8(factory); - registerFunctionFuzzBits(factory); -} - -} diff --git a/src/Functions/registerFunctionsReinterpret.cpp b/src/Functions/registerFunctionsReinterpret.cpp deleted file mode 100644 index e6fa0402071..00000000000 --- a/src/Functions/registerFunctionsReinterpret.cpp +++ /dev/null @@ -1,13 +0,0 @@ -namespace DB -{ - -class FunctionFactory; - -void registerFunctionsReinterpretAs(FunctionFactory & factory); - -void registerFunctionsReinterpret(FunctionFactory & factory) -{ - registerFunctionsReinterpretAs(factory); -} - -} diff --git a/src/Functions/registerFunctionsSnowflake.cpp b/src/Functions/registerFunctionsSnowflake.cpp deleted file mode 100644 index 7a0569ee16a..00000000000 --- a/src/Functions/registerFunctionsSnowflake.cpp +++ /dev/null @@ -1,22 +0,0 @@ -namespace DB -{ - -class FunctionFactory; - -void registerDateTimeToSnowflake(FunctionFactory &); -void registerSnowflakeToDateTime(FunctionFactory &); - -void registerDateTime64ToSnowflake(FunctionFactory &); -void registerSnowflakeToDateTime64(FunctionFactory &); - - -void registerFunctionsSnowflake(FunctionFactory & factory) -{ - registerDateTimeToSnowflake(factory); - registerSnowflakeToDateTime(factory); - - registerDateTime64ToSnowflake(factory); - registerSnowflakeToDateTime64(factory); -} - -} diff --git a/src/Functions/registerFunctionsString.cpp b/src/Functions/registerFunctionsString.cpp deleted file mode 100644 index 9e75e9900b7..00000000000 --- a/src/Functions/registerFunctionsString.cpp +++ /dev/null @@ -1,126 +0,0 @@ -#include "config_functions.h" -#include "config_core.h" - -namespace DB -{ - -class FunctionFactory; - -void registerFunctionRepeat(FunctionFactory &); -void registerFunctionEmpty(FunctionFactory &); -void registerFunctionNotEmpty(FunctionFactory &); -void registerFunctionLength(FunctionFactory &); -void registerFunctionLengthUTF8(FunctionFactory &); -void registerFunctionIsValidUTF8(FunctionFactory &); -void registerFunctionToValidUTF8(FunctionFactory &); -void registerFunctionLower(FunctionFactory &); -void registerFunctionUpper(FunctionFactory &); -void registerFunctionLowerUTF8(FunctionFactory &); -void registerFunctionUpperUTF8(FunctionFactory &); -void registerFunctionReverse(FunctionFactory &); -void registerFunctionReverseUTF8(FunctionFactory &); -void registerFunctionsConcat(FunctionFactory &); -void registerFunctionFormat(FunctionFactory &); -void registerFunctionFormatRow(FunctionFactory &); -void registerFunctionSubstring(FunctionFactory &); -void registerFunctionLeft(FunctionFactory &); -void registerFunctionRight(FunctionFactory &); -void registerFunctionCRC(FunctionFactory &); -void registerFunctionAppendTrailingCharIfAbsent(FunctionFactory &); -void registerFunctionStartsWith(FunctionFactory &); -void registerFunctionEndsWith(FunctionFactory &); -void registerFunctionTrim(FunctionFactory &); -void registerFunctionPadString(FunctionFactory &); -void registerFunctionRegexpQuoteMeta(FunctionFactory &); -void registerFunctionNormalizeQuery(FunctionFactory &); -void registerFunctionNormalizedQueryHash(FunctionFactory &); -void registerFunctionCountMatches(FunctionFactory &); -void registerFunctionEncodeXMLComponent(FunctionFactory &); -void registerFunctionDecodeXMLComponent(FunctionFactory &); -void registerFunctionExtractTextFromHTML(FunctionFactory &); -void registerFunctionToStringCutToZero(FunctionFactory &); -void registerFunctionDetectCharset(FunctionFactory &); -void registerFunctionDetectTonality(FunctionFactory &); -void registerFunctionDetectProgrammingLanguage(FunctionFactory &); - -#if USE_BASE64 -void registerFunctionBase64Encode(FunctionFactory &); -void registerFunctionBase64Decode(FunctionFactory &); -void registerFunctionTryBase64Decode(FunctionFactory &); -#endif - -void registerFunctionBase58Encode(FunctionFactory &); -void registerFunctionBase58Decode(FunctionFactory &); - -#if USE_NLP -void registerFunctionStem(FunctionFactory &); -void registerFunctionSynonyms(FunctionFactory &); -void registerFunctionLemmatize(FunctionFactory &); -void registerFunctionsDetectLanguage(FunctionFactory &); -#endif - -#if USE_ICU -void registerFunctionNormalizeUTF8(FunctionFactory &); -#endif - -void registerFunctionsString(FunctionFactory & factory) -{ - registerFunctionRepeat(factory); - registerFunctionEmpty(factory); - registerFunctionNotEmpty(factory); - registerFunctionLength(factory); - registerFunctionLengthUTF8(factory); - registerFunctionIsValidUTF8(factory); - registerFunctionToValidUTF8(factory); - registerFunctionLower(factory); - registerFunctionUpper(factory); - registerFunctionLowerUTF8(factory); - registerFunctionUpperUTF8(factory); - registerFunctionReverse(factory); - registerFunctionCRC(factory); - registerFunctionReverseUTF8(factory); - registerFunctionsConcat(factory); - registerFunctionFormat(factory); - registerFunctionFormatRow(factory); - registerFunctionSubstring(factory); - registerFunctionLeft(factory); - registerFunctionRight(factory); - registerFunctionAppendTrailingCharIfAbsent(factory); - registerFunctionStartsWith(factory); - registerFunctionEndsWith(factory); - registerFunctionTrim(factory); - registerFunctionPadString(factory); - registerFunctionRegexpQuoteMeta(factory); - registerFunctionNormalizeQuery(factory); - registerFunctionNormalizedQueryHash(factory); - registerFunctionCountMatches(factory); - registerFunctionEncodeXMLComponent(factory); - registerFunctionDecodeXMLComponent(factory); - registerFunctionExtractTextFromHTML(factory); - registerFunctionToStringCutToZero(factory); - registerFunctionDetectCharset(factory); - registerFunctionDetectTonality(factory); - registerFunctionDetectProgrammingLanguage(factory); - -#if USE_BASE64 - registerFunctionBase64Encode(factory); - registerFunctionBase64Decode(factory); - registerFunctionTryBase64Decode(factory); -#endif - - registerFunctionBase58Encode(factory); - registerFunctionBase58Decode(factory); - -#if USE_NLP - registerFunctionStem(factory); - registerFunctionSynonyms(factory); - registerFunctionLemmatize(factory); - registerFunctionsDetectLanguage(factory); -#endif - -#if USE_ICU - registerFunctionNormalizeUTF8(factory); -#endif -} - -} diff --git a/src/Functions/registerFunctionsStringRegexp.cpp b/src/Functions/registerFunctionsStringRegexp.cpp deleted file mode 100644 index df7e8f58396..00000000000 --- a/src/Functions/registerFunctionsStringRegexp.cpp +++ /dev/null @@ -1,50 +0,0 @@ -namespace DB -{ - -class FunctionFactory; - -void registerFunctionLike(FunctionFactory &); -void registerFunctionILike(FunctionFactory &); -void registerFunctionNotLike(FunctionFactory &); -void registerFunctionNotILike(FunctionFactory &); -void registerFunctionMatch(FunctionFactory &); -void registerFunctionExtract(FunctionFactory &); -void registerFunctionTranslate(FunctionFactory &); -void registerFunctionReplaceOne(FunctionFactory &); -void registerFunctionReplaceAll(FunctionFactory &); -void registerFunctionReplaceRegexpOne(FunctionFactory &); -void registerFunctionReplaceRegexpAll(FunctionFactory &); -void registerFunctionMultiMatchAny(FunctionFactory &); -void registerFunctionMultiMatchAnyIndex(FunctionFactory &); -void registerFunctionMultiMatchAllIndices(FunctionFactory &); -void registerFunctionMultiFuzzyMatchAny(FunctionFactory &); -void registerFunctionMultiFuzzyMatchAnyIndex(FunctionFactory &); -void registerFunctionMultiFuzzyMatchAllIndices(FunctionFactory &); -void registerFunctionExtractGroups(FunctionFactory &); -void registerFunctionExtractAllGroupsVertical(FunctionFactory &); -void registerFunctionExtractAllGroupsHorizontal(FunctionFactory &); - -void registerFunctionsStringRegexp(FunctionFactory & factory) -{ - registerFunctionLike(factory); - registerFunctionILike(factory); - registerFunctionNotLike(factory); - registerFunctionNotILike(factory); - registerFunctionMatch(factory); - registerFunctionExtract(factory); - registerFunctionTranslate(factory); - registerFunctionReplaceOne(factory); - registerFunctionReplaceAll(factory); - registerFunctionReplaceRegexpOne(factory); - registerFunctionReplaceRegexpAll(factory); - registerFunctionMultiMatchAny(factory); - registerFunctionMultiMatchAnyIndex(factory); - registerFunctionMultiMatchAllIndices(factory); - registerFunctionMultiFuzzyMatchAny(factory); - registerFunctionMultiFuzzyMatchAnyIndex(factory); - registerFunctionMultiFuzzyMatchAllIndices(factory); - registerFunctionExtractGroups(factory); - registerFunctionExtractAllGroupsVertical(factory); - registerFunctionExtractAllGroupsHorizontal(factory); -} -} diff --git a/src/Functions/registerFunctionsStringSearch.cpp b/src/Functions/registerFunctionsStringSearch.cpp deleted file mode 100644 index bdf94cf1134..00000000000 --- a/src/Functions/registerFunctionsStringSearch.cpp +++ /dev/null @@ -1,74 +0,0 @@ -namespace DB -{ - -class FunctionFactory; - -void registerFunctionPosition(FunctionFactory &); -void registerFunctionPositionUTF8(FunctionFactory &); -void registerFunctionPositionCaseInsensitive(FunctionFactory &); -void registerFunctionPositionCaseInsensitiveUTF8(FunctionFactory &); - -void registerFunctionMultiSearchAny(FunctionFactory &); -void registerFunctionMultiSearchAnyUTF8(FunctionFactory &); -void registerFunctionMultiSearchAnyCaseInsensitive(FunctionFactory &); -void registerFunctionMultiSearchAnyCaseInsensitiveUTF8(FunctionFactory &); - -void registerFunctionMultiSearchFirstIndex(FunctionFactory &); -void registerFunctionMultiSearchFirstIndexUTF8(FunctionFactory &); -void registerFunctionMultiSearchFirstIndexCaseInsensitive(FunctionFactory &); -void registerFunctionMultiSearchFirstIndexCaseInsensitiveUTF8(FunctionFactory &); - -void registerFunctionMultiSearchFirstPosition(FunctionFactory &); -void registerFunctionMultiSearchFirstPositionUTF8(FunctionFactory &); -void registerFunctionMultiSearchFirstPositionCaseInsensitive(FunctionFactory &); -void registerFunctionMultiSearchFirstPositionCaseInsensitiveUTF8(FunctionFactory &); - -void registerFunctionMultiSearchAllPositions(FunctionFactory &); -void registerFunctionMultiSearchAllPositionsUTF8(FunctionFactory &); -void registerFunctionMultiSearchAllPositionsCaseInsensitive(FunctionFactory &); -void registerFunctionMultiSearchAllPositionsCaseInsensitiveUTF8(FunctionFactory &); - -void registerFunctionHasToken(FunctionFactory &); -void registerFunctionHasTokenCaseInsensitive(FunctionFactory &); - -void registerFunctionCountSubstrings(FunctionFactory &); -void registerFunctionCountSubstringsCaseInsensitive(FunctionFactory &); -void registerFunctionCountSubstringsCaseInsensitiveUTF8(FunctionFactory &); - - -void registerFunctionsStringSearch(FunctionFactory & factory) -{ - registerFunctionPosition(factory); - registerFunctionPositionUTF8(factory); - registerFunctionPositionCaseInsensitive(factory); - registerFunctionPositionCaseInsensitiveUTF8(factory); - - registerFunctionMultiSearchAny(factory); - registerFunctionMultiSearchAnyUTF8(factory); - registerFunctionMultiSearchAnyCaseInsensitive(factory); - registerFunctionMultiSearchAnyCaseInsensitiveUTF8(factory); - - registerFunctionMultiSearchFirstIndex(factory); - registerFunctionMultiSearchFirstIndexUTF8(factory); - registerFunctionMultiSearchFirstIndexCaseInsensitive(factory); - registerFunctionMultiSearchFirstIndexCaseInsensitiveUTF8(factory); - - registerFunctionMultiSearchFirstPosition(factory); - registerFunctionMultiSearchFirstPositionUTF8(factory); - registerFunctionMultiSearchFirstPositionCaseInsensitive(factory); - registerFunctionMultiSearchFirstPositionCaseInsensitiveUTF8(factory); - - registerFunctionMultiSearchAllPositions(factory); - registerFunctionMultiSearchAllPositionsUTF8(factory); - registerFunctionMultiSearchAllPositionsCaseInsensitive(factory); - registerFunctionMultiSearchAllPositionsCaseInsensitiveUTF8(factory); - - registerFunctionHasToken(factory); - registerFunctionHasTokenCaseInsensitive(factory); - - registerFunctionCountSubstrings(factory); - registerFunctionCountSubstringsCaseInsensitive(factory); - registerFunctionCountSubstringsCaseInsensitiveUTF8(factory); -} - -} diff --git a/src/Functions/registerFunctionsTuple.cpp b/src/Functions/registerFunctionsTuple.cpp deleted file mode 100644 index 33f078675e9..00000000000 --- a/src/Functions/registerFunctionsTuple.cpp +++ /dev/null @@ -1,17 +0,0 @@ -namespace DB -{ - -class FunctionFactory; - -void registerFunctionTuple(FunctionFactory &); -void registerFunctionTupleElement(FunctionFactory &); -void registerFunctionTupleToNameValuePairs(FunctionFactory &); - -void registerFunctionsTuple(FunctionFactory & factory) -{ - registerFunctionTuple(factory); - registerFunctionTupleElement(factory); - registerFunctionTupleToNameValuePairs(factory); -} - -} diff --git a/src/Functions/registerFunctionsUnixTimestamp64.cpp b/src/Functions/registerFunctionsUnixTimestamp64.cpp deleted file mode 100644 index d7f3a4a4fc2..00000000000 --- a/src/Functions/registerFunctionsUnixTimestamp64.cpp +++ /dev/null @@ -1,25 +0,0 @@ -namespace DB -{ - -class FunctionFactory; - -void registerToUnixTimestamp64Milli(FunctionFactory &); -void registerToUnixTimestamp64Micro(FunctionFactory &); -void registerToUnixTimestamp64Nano(FunctionFactory &); - -void registerFromUnixTimestamp64Milli(FunctionFactory &); -void registerFromUnixTimestamp64Micro(FunctionFactory &); -void registerFromUnixTimestamp64Nano(FunctionFactory &); - -void registerFunctionsUnixTimestamp64(FunctionFactory & factory) -{ - registerToUnixTimestamp64Milli(factory); - registerToUnixTimestamp64Micro(factory); - registerToUnixTimestamp64Nano(factory); - - registerFromUnixTimestamp64Milli(factory); - registerFromUnixTimestamp64Micro(factory); - registerFromUnixTimestamp64Nano(factory); -} - -} diff --git a/src/Functions/registerFunctionsVisitParam.cpp b/src/Functions/registerFunctionsVisitParam.cpp deleted file mode 100644 index 01084594f08..00000000000 --- a/src/Functions/registerFunctionsVisitParam.cpp +++ /dev/null @@ -1,25 +0,0 @@ -namespace DB -{ - -class FunctionFactory; - -void registerFunctionVisitParamHas(FunctionFactory & factory); -void registerFunctionVisitParamExtractUInt(FunctionFactory & factory); -void registerFunctionVisitParamExtractInt(FunctionFactory & factory); -void registerFunctionVisitParamExtractFloat(FunctionFactory & factory); -void registerFunctionVisitParamExtractBool(FunctionFactory & factory); -void registerFunctionVisitParamExtractRaw(FunctionFactory & factory); -void registerFunctionVisitParamExtractString(FunctionFactory & factory); - -void registerFunctionsVisitParam(FunctionFactory & factory) -{ - registerFunctionVisitParamHas(factory); - registerFunctionVisitParamExtractUInt(factory); - registerFunctionVisitParamExtractInt(factory); - registerFunctionVisitParamExtractFloat(factory); - registerFunctionVisitParamExtractBool(factory); - registerFunctionVisitParamExtractRaw(factory); - registerFunctionVisitParamExtractString(factory); -} - -} diff --git a/src/Functions/reinterpretAs.cpp b/src/Functions/reinterpretAs.cpp index a31b41b55f2..8e656863cdb 100644 --- a/src/Functions/reinterpretAs.cpp +++ b/src/Functions/reinterpretAs.cpp @@ -471,7 +471,7 @@ using FunctionReinterpretAsFixedString = FunctionReinterpretAs(); factory.registerFunction(); diff --git a/src/Functions/repeat.cpp b/src/Functions/repeat.cpp index 6100b523420..40f33f2b9c4 100644 --- a/src/Functions/repeat.cpp +++ b/src/Functions/repeat.cpp @@ -252,7 +252,7 @@ public: } -void registerFunctionRepeat(FunctionFactory & factory) +REGISTER_FUNCTION(Repeat) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/replaceAll.cpp b/src/Functions/replaceAll.cpp index cc29e57ea69..7c5cd82ca5d 100644 --- a/src/Functions/replaceAll.cpp +++ b/src/Functions/replaceAll.cpp @@ -17,7 +17,7 @@ using FunctionReplaceAll = FunctionStringReplace, NameR } -void registerFunctionReplaceAll(FunctionFactory & factory) +REGISTER_FUNCTION(ReplaceAll) { factory.registerFunction(); factory.registerAlias("replace", NameReplaceAll::name, FunctionFactory::CaseInsensitive); diff --git a/src/Functions/replaceOne.cpp b/src/Functions/replaceOne.cpp index d9bcbef0e2d..c0c21dbf51f 100644 --- a/src/Functions/replaceOne.cpp +++ b/src/Functions/replaceOne.cpp @@ -17,7 +17,7 @@ using FunctionReplaceOne = FunctionStringReplace, NameRe } -void registerFunctionReplaceOne(FunctionFactory & factory) +REGISTER_FUNCTION(ReplaceOne) { factory.registerFunction(); } diff --git a/src/Functions/replaceRegexpAll.cpp b/src/Functions/replaceRegexpAll.cpp index 07ffbdae792..0250b4a5ba6 100644 --- a/src/Functions/replaceRegexpAll.cpp +++ b/src/Functions/replaceRegexpAll.cpp @@ -17,7 +17,7 @@ using FunctionReplaceRegexpAll = FunctionStringReplace, } -void registerFunctionReplaceRegexpAll(FunctionFactory & factory) +REGISTER_FUNCTION(ReplaceRegexpAll) { factory.registerFunction(); factory.registerAlias("REGEXP_REPLACE", NameReplaceRegexpAll::name, FunctionFactory::CaseInsensitive); diff --git a/src/Functions/replaceRegexpOne.cpp b/src/Functions/replaceRegexpOne.cpp index 9844f77ee26..b40992b73fc 100644 --- a/src/Functions/replaceRegexpOne.cpp +++ b/src/Functions/replaceRegexpOne.cpp @@ -17,7 +17,7 @@ using FunctionReplaceRegexpOne = FunctionStringReplace, } -void registerFunctionReplaceRegexpOne(FunctionFactory & factory) +REGISTER_FUNCTION(ReplaceRegexpOne) { factory.registerFunction(); } diff --git a/src/Functions/replicate.cpp b/src/Functions/replicate.cpp index 5f6f187031f..f55f89bcac1 100644 --- a/src/Functions/replicate.cpp +++ b/src/Functions/replicate.cpp @@ -59,7 +59,7 @@ ColumnPtr FunctionReplicate::executeImpl(const ColumnsWithTypeAndName & argument return ColumnArray::create(first_column->replicate(offsets_data)->convertToFullColumnIfConst(), offsets); } -void registerFunctionReplicate(FunctionFactory & factory) +REGISTER_FUNCTION(Replicate) { factory.registerFunction(); } diff --git a/src/Functions/reverse.cpp b/src/Functions/reverse.cpp index 9edd2927706..56397958b10 100644 --- a/src/Functions/reverse.cpp +++ b/src/Functions/reverse.cpp @@ -148,7 +148,7 @@ private: } -void registerFunctionReverse(FunctionFactory & factory) +REGISTER_FUNCTION(Reverse) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/reverseUTF8.cpp b/src/Functions/reverseUTF8.cpp index b19808c091e..06567f0b325 100644 --- a/src/Functions/reverseUTF8.cpp +++ b/src/Functions/reverseUTF8.cpp @@ -76,7 +76,7 @@ using FunctionReverseUTF8 = FunctionStringToString(); } diff --git a/src/Functions/right.cpp b/src/Functions/right.cpp index ca0df8b2d98..4c26630c9ff 100644 --- a/src/Functions/right.cpp +++ b/src/Functions/right.cpp @@ -4,7 +4,7 @@ namespace DB { -void registerFunctionRight(FunctionFactory & factory) +REGISTER_FUNCTION(Right) { factory.registerFunction>(FunctionFactory::CaseInsensitive); factory.registerFunction>(FunctionFactory::CaseSensitive); diff --git a/src/Functions/roundAge.cpp b/src/Functions/roundAge.cpp index 54e34dc7266..d2503bb6938 100644 --- a/src/Functions/roundAge.cpp +++ b/src/Functions/roundAge.cpp @@ -36,7 +36,7 @@ using FunctionRoundAge = FunctionUnaryArithmetic struct FunctionUnaryArithmeticMonotonicity : PositiveMonotonicity {}; -void registerFunctionRoundAge(FunctionFactory & factory) +REGISTER_FUNCTION(RoundAge) { factory.registerFunction(); } diff --git a/src/Functions/roundDuration.cpp b/src/Functions/roundDuration.cpp index 40018753d5e..62d35ea194d 100644 --- a/src/Functions/roundDuration.cpp +++ b/src/Functions/roundDuration.cpp @@ -45,7 +45,7 @@ using FunctionRoundDuration = FunctionUnaryArithmetic struct FunctionUnaryArithmeticMonotonicity : PositiveMonotonicity {}; -void registerFunctionRoundDuration(FunctionFactory & factory) +REGISTER_FUNCTION(RoundDuration) { factory.registerFunction(); } diff --git a/src/Functions/roundToExp2.cpp b/src/Functions/roundToExp2.cpp index 846890bc5c8..7893773fb61 100644 --- a/src/Functions/roundToExp2.cpp +++ b/src/Functions/roundToExp2.cpp @@ -83,7 +83,7 @@ using FunctionRoundToExp2 = FunctionUnaryArithmetic struct FunctionUnaryArithmeticMonotonicity : PositiveMonotonicity {}; -void registerFunctionRoundToExp2(FunctionFactory & factory) +REGISTER_FUNCTION(RoundToExp2) { factory.registerFunction(); } diff --git a/src/Functions/rowNumberInAllBlocks.cpp b/src/Functions/rowNumberInAllBlocks.cpp index b12fe351553..91da512d221 100644 --- a/src/Functions/rowNumberInAllBlocks.cpp +++ b/src/Functions/rowNumberInAllBlocks.cpp @@ -74,7 +74,7 @@ public: } -void registerFunctionRowNumberInAllBlocks(FunctionFactory & factory) +REGISTER_FUNCTION(RowNumberInAllBlocks) { factory.registerFunction(); } diff --git a/src/Functions/rowNumberInBlock.cpp b/src/Functions/rowNumberInBlock.cpp index 89a9124691f..b3f95d27a93 100644 --- a/src/Functions/rowNumberInBlock.cpp +++ b/src/Functions/rowNumberInBlock.cpp @@ -62,7 +62,7 @@ public: } -void registerFunctionRowNumberInBlock(FunctionFactory & factory) +REGISTER_FUNCTION(RowNumberInBlock) { factory.registerFunction(); } diff --git a/src/Functions/runningAccumulate.cpp b/src/Functions/runningAccumulate.cpp index ad0cbdb8758..667f722ee92 100644 --- a/src/Functions/runningAccumulate.cpp +++ b/src/Functions/runningAccumulate.cpp @@ -138,7 +138,7 @@ public: } -void registerFunctionRunningAccumulate(FunctionFactory & factory) +REGISTER_FUNCTION(RunningAccumulate) { factory.registerFunction(); } diff --git a/src/Functions/runningConcurrency.cpp b/src/Functions/runningConcurrency.cpp index 4d190dd0f79..37fa11bce8f 100644 --- a/src/Functions/runningConcurrency.cpp +++ b/src/Functions/runningConcurrency.cpp @@ -220,7 +220,7 @@ namespace DB static constexpr auto name = "runningConcurrency"; }; - void registerFunctionRunningConcurrency(FunctionFactory & factory) + REGISTER_FUNCTION(RunningConcurrency) { factory.registerFunction>(); } diff --git a/src/Functions/runningDifference.cpp b/src/Functions/runningDifference.cpp index 07acaf6522b..dd3baaa82c1 100644 --- a/src/Functions/runningDifference.cpp +++ b/src/Functions/runningDifference.cpp @@ -5,7 +5,7 @@ namespace DB { -void registerFunctionRunningDifference(FunctionFactory & factory) +REGISTER_FUNCTION(RunningDifference) { factory.registerFunction>(); } diff --git a/src/Functions/runningDifferenceStartingWithFirstValue.cpp b/src/Functions/runningDifferenceStartingWithFirstValue.cpp index 4419e413da7..02cb5c3cd4e 100644 --- a/src/Functions/runningDifferenceStartingWithFirstValue.cpp +++ b/src/Functions/runningDifferenceStartingWithFirstValue.cpp @@ -5,7 +5,7 @@ namespace DB { -void registerFunctionRunningDifferenceStartingWithFirstValue(FunctionFactory & factory) +REGISTER_FUNCTION(RunningDifferenceStartingWithFirstValue) { factory.registerFunction>(); } diff --git a/src/Functions/s2CapContains.cpp b/src/Functions/s2CapContains.cpp index 482c4a22c63..f7a31120e0f 100644 --- a/src/Functions/s2CapContains.cpp +++ b/src/Functions/s2CapContains.cpp @@ -154,7 +154,7 @@ public: } -void registerFunctionS2CapContains(FunctionFactory & factory) +REGISTER_FUNCTION(S2CapContains) { factory.registerFunction(); } diff --git a/src/Functions/s2CapUnion.cpp b/src/Functions/s2CapUnion.cpp index ea1f2f7534d..da329065553 100644 --- a/src/Functions/s2CapUnion.cpp +++ b/src/Functions/s2CapUnion.cpp @@ -169,7 +169,7 @@ public: } -void registerFunctionS2CapUnion(FunctionFactory & factory) +REGISTER_FUNCTION(S2CapUnion) { factory.registerFunction(); } diff --git a/src/Functions/s2CellsIntersect.cpp b/src/Functions/s2CellsIntersect.cpp index 617910e1a59..51cef79285f 100644 --- a/src/Functions/s2CellsIntersect.cpp +++ b/src/Functions/s2CellsIntersect.cpp @@ -115,7 +115,7 @@ public: } -void registerFunctionS2CellsIntersect(FunctionFactory & factory) +REGISTER_FUNCTION(S2CellsIntersect) { factory.registerFunction(); } diff --git a/src/Functions/s2GetNeighbors.cpp b/src/Functions/s2GetNeighbors.cpp index f16e531a645..906a0e01195 100644 --- a/src/Functions/s2GetNeighbors.cpp +++ b/src/Functions/s2GetNeighbors.cpp @@ -115,7 +115,7 @@ public: } -void registerFunctionS2GetNeighbors(FunctionFactory & factory) +REGISTER_FUNCTION(S2GetNeighbors) { factory.registerFunction(); } diff --git a/src/Functions/s2RectAdd.cpp b/src/Functions/s2RectAdd.cpp index 44f240588be..fe74f8b2507 100644 --- a/src/Functions/s2RectAdd.cpp +++ b/src/Functions/s2RectAdd.cpp @@ -135,7 +135,7 @@ public: } -void registerFunctionS2RectAdd(FunctionFactory & factory) +REGISTER_FUNCTION(S2RectAdd) { factory.registerFunction(); } diff --git a/src/Functions/s2RectContains.cpp b/src/Functions/s2RectContains.cpp index 84218704ef1..c10a4e5ecae 100644 --- a/src/Functions/s2RectContains.cpp +++ b/src/Functions/s2RectContains.cpp @@ -125,7 +125,7 @@ public: } -void registerFunctionS2RectContains(FunctionFactory & factory) +REGISTER_FUNCTION(S2RectContains) { factory.registerFunction(); } diff --git a/src/Functions/s2RectIntersection.cpp b/src/Functions/s2RectIntersection.cpp index 064dad4166d..cf4f7c8aa9d 100644 --- a/src/Functions/s2RectIntersection.cpp +++ b/src/Functions/s2RectIntersection.cpp @@ -150,7 +150,7 @@ public: } -void registerFunctionS2RectIntersection(FunctionFactory & factory) +REGISTER_FUNCTION(S2RectIntersection) { factory.registerFunction(); } diff --git a/src/Functions/s2RectUnion.cpp b/src/Functions/s2RectUnion.cpp index 91664d3b65f..845dcb982b6 100644 --- a/src/Functions/s2RectUnion.cpp +++ b/src/Functions/s2RectUnion.cpp @@ -148,7 +148,7 @@ public: } -void registerFunctionS2RectUnion(FunctionFactory & factory) +REGISTER_FUNCTION(S2RectUnion) { factory.registerFunction(); } diff --git a/src/Functions/s2ToGeo.cpp b/src/Functions/s2ToGeo.cpp index 082c300536d..63edfc84f97 100644 --- a/src/Functions/s2ToGeo.cpp +++ b/src/Functions/s2ToGeo.cpp @@ -113,7 +113,7 @@ public: } -void registerFunctionS2ToGeo(FunctionFactory & factory) +REGISTER_FUNCTION(S2ToGeo) { factory.registerFunction(); } diff --git a/src/Functions/serverConstants.cpp b/src/Functions/serverConstants.cpp index e809ec7c298..cbb567339bd 100644 --- a/src/Functions/serverConstants.cpp +++ b/src/Functions/serverConstants.cpp @@ -121,63 +121,62 @@ namespace } - -void registerFunctionBuildId([[maybe_unused]] FunctionFactory & factory) -{ #if defined(__ELF__) && !defined(OS_FREEBSD) +REGISTER_FUNCTION(BuildId) +{ factory.registerFunction(); -#endif } +#endif -void registerFunctionHostName(FunctionFactory & factory) +REGISTER_FUNCTION(HostName) { factory.registerFunction(); factory.registerAlias("hostname", "hostName"); } -void registerFunctionServerUUID(FunctionFactory & factory) +REGISTER_FUNCTION(ServerUUID) { factory.registerFunction(); } -void registerFunctionTcpPort(FunctionFactory & factory) +REGISTER_FUNCTION(TcpPort) { factory.registerFunction(); } -void registerFunctionTimezone(FunctionFactory & factory) +REGISTER_FUNCTION(Timezone) { factory.registerFunction(); factory.registerAlias("timeZone", "timezone"); } -void registerFunctionUptime(FunctionFactory & factory) +REGISTER_FUNCTION(Uptime) { factory.registerFunction(); } -void registerFunctionVersion(FunctionFactory & factory) +REGISTER_FUNCTION(Version) { factory.registerFunction(FunctionFactory::CaseInsensitive); } -void registerFunctionRevision(FunctionFactory & factory) +REGISTER_FUNCTION(Revision) { factory.registerFunction(FunctionFactory::CaseInsensitive); } -void registerFunctionZooKeeperSessionUptime(FunctionFactory & factory) +REGISTER_FUNCTION(ZooKeeperSessionUptime) { factory.registerFunction(); } -void registerFunctionGetOSKernelVersion([[maybe_unused]] FunctionFactory & factory) -{ #if defined(OS_LINUX) +REGISTER_FUNCTION(GetOSKernelVersion) +{ factory.registerFunction(); +} #endif -} } diff --git a/src/Functions/sigmoid.cpp b/src/Functions/sigmoid.cpp index d675413ae67..d121bdc7389 100644 --- a/src/Functions/sigmoid.cpp +++ b/src/Functions/sigmoid.cpp @@ -41,7 +41,7 @@ using FunctionSigmoid = FunctionMathUnary(); } diff --git a/src/Functions/sign.cpp b/src/Functions/sign.cpp index 5dfe6538b32..ae87ff8e8b6 100644 --- a/src/Functions/sign.cpp +++ b/src/Functions/sign.cpp @@ -43,7 +43,7 @@ struct FunctionUnaryArithmeticMonotonicity } }; -void registerFunctionSign(FunctionFactory & factory) +REGISTER_FUNCTION(Sign) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/sin.cpp b/src/Functions/sin.cpp index 6fd5d189767..536b2635b9a 100644 --- a/src/Functions/sin.cpp +++ b/src/Functions/sin.cpp @@ -11,7 +11,7 @@ using FunctionSin = FunctionMathUnary>; } -void registerFunctionSin(FunctionFactory & factory) +REGISTER_FUNCTION(Sin) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/sinh.cpp b/src/Functions/sinh.cpp index 84fe0e805b7..2d83fad36df 100644 --- a/src/Functions/sinh.cpp +++ b/src/Functions/sinh.cpp @@ -13,7 +13,7 @@ namespace } -void registerFunctionSinh(FunctionFactory & factory) +REGISTER_FUNCTION(Sinh) { factory.registerFunction(); } diff --git a/src/Functions/sleep.cpp b/src/Functions/sleep.cpp index ad1c6680363..abd1226b8cf 100644 --- a/src/Functions/sleep.cpp +++ b/src/Functions/sleep.cpp @@ -5,7 +5,7 @@ namespace DB { -void registerFunctionSleep(FunctionFactory & factory) +REGISTER_FUNCTION(Sleep) { factory.registerFunction>(); } diff --git a/src/Functions/sleepEachRow.cpp b/src/Functions/sleepEachRow.cpp index c1c983e850e..595beb15e19 100644 --- a/src/Functions/sleepEachRow.cpp +++ b/src/Functions/sleepEachRow.cpp @@ -5,7 +5,7 @@ namespace DB { -void registerFunctionSleepEachRow(FunctionFactory & factory) +REGISTER_FUNCTION(SleepEachRow) { factory.registerFunction>(); } diff --git a/src/Functions/snowflake.cpp b/src/Functions/snowflake.cpp index 5ac1d229d17..4849d6512ca 100644 --- a/src/Functions/snowflake.cpp +++ b/src/Functions/snowflake.cpp @@ -4,27 +4,27 @@ namespace DB { -void registerDateTimeToSnowflake(FunctionFactory & factory) +REGISTER_FUNCTION(DateTimeToSnowflake) { factory.registerFunction("dateTimeToSnowflake", [](ContextPtr){ return std::make_unique( std::make_shared("dateTimeToSnowflake")); }); } -void registerDateTime64ToSnowflake(FunctionFactory & factory) +REGISTER_FUNCTION(DateTime64ToSnowflake) { factory.registerFunction("dateTime64ToSnowflake", [](ContextPtr){ return std::make_unique( std::make_shared("dateTime64ToSnowflake")); }); } -void registerSnowflakeToDateTime(FunctionFactory & factory) +REGISTER_FUNCTION(SnowflakeToDateTime) { factory.registerFunction("snowflakeToDateTime", [](ContextPtr){ return std::make_unique( std::make_shared("snowflakeToDateTime")); }); } -void registerSnowflakeToDateTime64(FunctionFactory & factory) +REGISTER_FUNCTION(SnowflakeToDateTime64) { factory.registerFunction("snowflakeToDateTime64", [](ContextPtr){ return std::make_unique( diff --git a/src/Functions/sqrt.cpp b/src/Functions/sqrt.cpp index 725da874a51..63c1098d7e7 100644 --- a/src/Functions/sqrt.cpp +++ b/src/Functions/sqrt.cpp @@ -11,7 +11,7 @@ using FunctionSqrt = FunctionMathUnary>; } -void registerFunctionSqrt(FunctionFactory & factory) +REGISTER_FUNCTION(Sqrt) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/startsWith.cpp b/src/Functions/startsWith.cpp index 8e9a06cd17c..32f2b08d540 100644 --- a/src/Functions/startsWith.cpp +++ b/src/Functions/startsWith.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionStartsWith = FunctionStartsEndsWith; -void registerFunctionStartsWith(FunctionFactory & factory) +REGISTER_FUNCTION(StartsWith) { factory.registerFunction(); } diff --git a/src/Functions/stem.cpp b/src/Functions/stem.cpp index 3da4a7fdd07..25021ed74a4 100644 --- a/src/Functions/stem.cpp +++ b/src/Functions/stem.cpp @@ -125,7 +125,7 @@ public: } -void registerFunctionStem(FunctionFactory & factory) +REGISTER_FUNCTION(Stem) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/stringCutToZero.cpp b/src/Functions/stringCutToZero.cpp index baebf67c0e3..caeedaceae7 100644 --- a/src/Functions/stringCutToZero.cpp +++ b/src/Functions/stringCutToZero.cpp @@ -148,7 +148,7 @@ public: }; -void registerFunctionToStringCutToZero(FunctionFactory & factory) +REGISTER_FUNCTION(ToStringCutToZero) { factory.registerFunction(); } diff --git a/src/Functions/stringToH3.cpp b/src/Functions/stringToH3.cpp index db13534b3d2..8a90866e131 100644 --- a/src/Functions/stringToH3.cpp +++ b/src/Functions/stringToH3.cpp @@ -100,7 +100,7 @@ private: } -void registerFunctionStringToH3(FunctionFactory & factory) +REGISTER_FUNCTION(StringToH3) { factory.registerFunction(); } diff --git a/src/Functions/substring.cpp b/src/Functions/substring.cpp index ca94071187a..79b801a9ef6 100644 --- a/src/Functions/substring.cpp +++ b/src/Functions/substring.cpp @@ -186,7 +186,7 @@ public: } -void registerFunctionSubstring(FunctionFactory & factory) +REGISTER_FUNCTION(Substring) { factory.registerFunction>(FunctionFactory::CaseInsensitive); factory.registerAlias("substr", "substring", FunctionFactory::CaseInsensitive); diff --git a/src/Functions/subtractDays.cpp b/src/Functions/subtractDays.cpp index 21966a15311..b0060be2dd2 100644 --- a/src/Functions/subtractDays.cpp +++ b/src/Functions/subtractDays.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionSubtractDays = FunctionDateOrDateTimeAddInterval; -void registerFunctionSubtractDays(FunctionFactory & factory) +REGISTER_FUNCTION(SubtractDays) { factory.registerFunction(); } diff --git a/src/Functions/subtractHours.cpp b/src/Functions/subtractHours.cpp index e71c9d74a01..0c7cd83e8d6 100644 --- a/src/Functions/subtractHours.cpp +++ b/src/Functions/subtractHours.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionSubtractHours = FunctionDateOrDateTimeAddInterval; -void registerFunctionSubtractHours(FunctionFactory & factory) +REGISTER_FUNCTION(SubtractHours) { factory.registerFunction(); } diff --git a/src/Functions/subtractMinutes.cpp b/src/Functions/subtractMinutes.cpp index ba9d593a64d..562cf2e6fe6 100644 --- a/src/Functions/subtractMinutes.cpp +++ b/src/Functions/subtractMinutes.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionSubtractMinutes = FunctionDateOrDateTimeAddInterval; -void registerFunctionSubtractMinutes(FunctionFactory & factory) +REGISTER_FUNCTION(SubtractMinutes) { factory.registerFunction(); } diff --git a/src/Functions/subtractMonths.cpp b/src/Functions/subtractMonths.cpp index 64eeba2ce86..b4e34a03861 100644 --- a/src/Functions/subtractMonths.cpp +++ b/src/Functions/subtractMonths.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionSubtractMonths = FunctionDateOrDateTimeAddInterval; -void registerFunctionSubtractMonths(FunctionFactory & factory) +REGISTER_FUNCTION(SubtractMonths) { factory.registerFunction(); } diff --git a/src/Functions/subtractQuarters.cpp b/src/Functions/subtractQuarters.cpp index 6c066ed17a1..295e1ad37e0 100644 --- a/src/Functions/subtractQuarters.cpp +++ b/src/Functions/subtractQuarters.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionSubtractQuarters = FunctionDateOrDateTimeAddInterval; -void registerFunctionSubtractQuarters(FunctionFactory & factory) +REGISTER_FUNCTION(SubtractQuarters) { factory.registerFunction(); } diff --git a/src/Functions/subtractSeconds.cpp b/src/Functions/subtractSeconds.cpp index 81a7f7e2df1..97358c9b64f 100644 --- a/src/Functions/subtractSeconds.cpp +++ b/src/Functions/subtractSeconds.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionSubtractSeconds = FunctionDateOrDateTimeAddInterval; -void registerFunctionSubtractSeconds(FunctionFactory & factory) +REGISTER_FUNCTION(SubtractSeconds) { factory.registerFunction(); } diff --git a/src/Functions/subtractWeeks.cpp b/src/Functions/subtractWeeks.cpp index 55b52043dd0..e4a600fee3c 100644 --- a/src/Functions/subtractWeeks.cpp +++ b/src/Functions/subtractWeeks.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionSubtractWeeks = FunctionDateOrDateTimeAddInterval; -void registerFunctionSubtractWeeks(FunctionFactory & factory) +REGISTER_FUNCTION(SubtractWeeks) { factory.registerFunction(); } diff --git a/src/Functions/subtractYears.cpp b/src/Functions/subtractYears.cpp index 241142722d5..78f4bb3cc28 100644 --- a/src/Functions/subtractYears.cpp +++ b/src/Functions/subtractYears.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionSubtractYears = FunctionDateOrDateTimeAddInterval; -void registerFunctionSubtractYears(FunctionFactory & factory) +REGISTER_FUNCTION(SubtractYears) { factory.registerFunction(); } diff --git a/src/Functions/svg.cpp b/src/Functions/svg.cpp index e1d48ffc061..4cf1598857b 100644 --- a/src/Functions/svg.cpp +++ b/src/Functions/svg.cpp @@ -99,7 +99,7 @@ public: } }; -void registerFunctionSvg(FunctionFactory & factory) +REGISTER_FUNCTION(Svg) { factory.registerFunction(); factory.registerAlias("SVG", "svg"); diff --git a/src/Functions/synonyms.cpp b/src/Functions/synonyms.cpp index 6d879f9d42c..d68f9c76743 100644 --- a/src/Functions/synonyms.cpp +++ b/src/Functions/synonyms.cpp @@ -118,7 +118,7 @@ public: } }; -void registerFunctionSynonyms(FunctionFactory & factory) +REGISTER_FUNCTION(Synonyms) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/tan.cpp b/src/Functions/tan.cpp index db63ab13984..7d84055d0c3 100644 --- a/src/Functions/tan.cpp +++ b/src/Functions/tan.cpp @@ -11,7 +11,7 @@ using FunctionTan = FunctionMathUnary>; } -void registerFunctionTan(FunctionFactory & factory) +REGISTER_FUNCTION(Tan) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/tanh.cpp b/src/Functions/tanh.cpp index 6fc1d2f79e5..9461c2a5811 100644 --- a/src/Functions/tanh.cpp +++ b/src/Functions/tanh.cpp @@ -37,7 +37,7 @@ using FunctionTanh = FunctionMathUnary>; } -void registerFunctionTanh(FunctionFactory & factory) +REGISTER_FUNCTION(Tanh) { factory.registerFunction(FunctionFactory::CaseInsensitive); } diff --git a/src/Functions/tgamma.cpp b/src/Functions/tgamma.cpp index 3378d44388d..c1fb235efcd 100644 --- a/src/Functions/tgamma.cpp +++ b/src/Functions/tgamma.cpp @@ -11,7 +11,7 @@ using FunctionTGamma = FunctionMathUnary(); } diff --git a/src/Functions/throwIf.cpp b/src/Functions/throwIf.cpp index 32afa8b4702..bda8426150e 100644 --- a/src/Functions/throwIf.cpp +++ b/src/Functions/throwIf.cpp @@ -143,7 +143,7 @@ public: } -void registerFunctionThrowIf(FunctionFactory & factory) +REGISTER_FUNCTION(ThrowIf) { factory.registerFunction(); } diff --git a/src/Functions/tid.cpp b/src/Functions/tid.cpp index ec767e8ec84..1789c9ba7d4 100644 --- a/src/Functions/tid.cpp +++ b/src/Functions/tid.cpp @@ -32,7 +32,7 @@ namespace } -void registerFunctionTid(FunctionFactory & factory) +REGISTER_FUNCTION(Tid) { factory.registerFunction(); } diff --git a/src/Functions/timeSlot.cpp b/src/Functions/timeSlot.cpp index afe99c86eb2..697d2234db2 100644 --- a/src/Functions/timeSlot.cpp +++ b/src/Functions/timeSlot.cpp @@ -10,7 +10,7 @@ namespace DB using FunctionTimeSlot = FunctionDateOrDateTimeToSomething; -void registerFunctionTimeSlot(FunctionFactory & factory) +REGISTER_FUNCTION(TimeSlot) { factory.registerFunction(); } diff --git a/src/Functions/timeSlots.cpp b/src/Functions/timeSlots.cpp index a19ccf62565..15696dc719a 100644 --- a/src/Functions/timeSlots.cpp +++ b/src/Functions/timeSlots.cpp @@ -195,7 +195,7 @@ public: } -void registerFunctionTimeSlots(FunctionFactory & factory) +REGISTER_FUNCTION(TimeSlots) { factory.registerFunction(); } diff --git a/src/Functions/timezoneOf.cpp b/src/Functions/timezoneOf.cpp index 97e025bc0e0..6454b1cd735 100644 --- a/src/Functions/timezoneOf.cpp +++ b/src/Functions/timezoneOf.cpp @@ -67,7 +67,7 @@ public: } -void registerFunctionTimezoneOf(FunctionFactory & factory) +REGISTER_FUNCTION(TimezoneOf) { factory.registerFunction(); factory.registerAlias("timeZoneOf", "timezoneOf"); diff --git a/src/Functions/timezoneOffset.cpp b/src/Functions/timezoneOffset.cpp index 296603ce0a4..cc1945d23f9 100644 --- a/src/Functions/timezoneOffset.cpp +++ b/src/Functions/timezoneOffset.cpp @@ -9,7 +9,7 @@ namespace DB using FunctiontimezoneOffset = FunctionDateOrDateTimeToSomething; -void registerFunctiontimezoneOffset(FunctionFactory & factory) +REGISTER_FUNCTION(timezoneOffset) { factory.registerFunction(); factory.registerAlias("timeZoneOffset", "timezoneOffset"); diff --git a/src/Functions/toBool.cpp b/src/Functions/toBool.cpp index 7f167744f01..765da0c3206 100644 --- a/src/Functions/toBool.cpp +++ b/src/Functions/toBool.cpp @@ -56,7 +56,7 @@ namespace } -void registerFunctionToBool(FunctionFactory & factory) +REGISTER_FUNCTION(ToBool) { factory.registerFunction(); } diff --git a/src/Functions/toColumnTypeName.cpp b/src/Functions/toColumnTypeName.cpp index d8013f13340..89a44b2aeef 100644 --- a/src/Functions/toColumnTypeName.cpp +++ b/src/Functions/toColumnTypeName.cpp @@ -55,7 +55,7 @@ public: } -void registerFunctionToColumnTypeName(FunctionFactory & factory) +REGISTER_FUNCTION(ToColumnTypeName) { factory.registerFunction(); } diff --git a/src/Functions/toCustomWeek.cpp b/src/Functions/toCustomWeek.cpp index 5ba0b8e8b2a..13dc76b6389 100644 --- a/src/Functions/toCustomWeek.cpp +++ b/src/Functions/toCustomWeek.cpp @@ -11,7 +11,7 @@ using FunctionToWeek = FunctionCustomWeekToSomething; using FunctionToYearWeek = FunctionCustomWeekToSomething; using FunctionToStartOfWeek = FunctionCustomWeekToSomething; -void registerFunctionToCustomWeek(FunctionFactory & factory) +REGISTER_FUNCTION(ToCustomWeek) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/toDayOfMonth.cpp b/src/Functions/toDayOfMonth.cpp index ad397a15276..c5ed4629258 100644 --- a/src/Functions/toDayOfMonth.cpp +++ b/src/Functions/toDayOfMonth.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToDayOfMonth = FunctionDateOrDateTimeToSomething; -void registerFunctionToDayOfMonth(FunctionFactory & factory) +REGISTER_FUNCTION(ToDayOfMonth) { factory.registerFunction(); diff --git a/src/Functions/toDayOfWeek.cpp b/src/Functions/toDayOfWeek.cpp index 9b2bb055eca..2c04e30a607 100644 --- a/src/Functions/toDayOfWeek.cpp +++ b/src/Functions/toDayOfWeek.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToDayOfWeek = FunctionDateOrDateTimeToSomething; -void registerFunctionToDayOfWeek(FunctionFactory & factory) +REGISTER_FUNCTION(ToDayOfWeek) { factory.registerFunction(); diff --git a/src/Functions/toDayOfYear.cpp b/src/Functions/toDayOfYear.cpp index 510e3253f87..ac289e3a757 100644 --- a/src/Functions/toDayOfYear.cpp +++ b/src/Functions/toDayOfYear.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToDayOfYear = FunctionDateOrDateTimeToSomething; -void registerFunctionToDayOfYear(FunctionFactory & factory) +REGISTER_FUNCTION(ToDayOfYear) { factory.registerFunction(); diff --git a/src/Functions/toFixedString.cpp b/src/Functions/toFixedString.cpp index cfc357a055a..1299288b87e 100644 --- a/src/Functions/toFixedString.cpp +++ b/src/Functions/toFixedString.cpp @@ -5,7 +5,7 @@ namespace DB { -void registerFunctionFixedString(FunctionFactory & factory) +REGISTER_FUNCTION(FixedString) { factory.registerFunction(); } diff --git a/src/Functions/toHour.cpp b/src/Functions/toHour.cpp index 72709f990b0..172515aad58 100644 --- a/src/Functions/toHour.cpp +++ b/src/Functions/toHour.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToHour = FunctionDateOrDateTimeToSomething; -void registerFunctionToHour(FunctionFactory & factory) +REGISTER_FUNCTION(ToHour) { factory.registerFunction(); diff --git a/src/Functions/toISOWeek.cpp b/src/Functions/toISOWeek.cpp index 9cdee8b7a94..240c0c2bad4 100644 --- a/src/Functions/toISOWeek.cpp +++ b/src/Functions/toISOWeek.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToISOWeek = FunctionDateOrDateTimeToSomething; -void registerFunctionToISOWeek(FunctionFactory & factory) +REGISTER_FUNCTION(ToISOWeek) { factory.registerFunction(); } diff --git a/src/Functions/toISOYear.cpp b/src/Functions/toISOYear.cpp index 200ce7aa97e..9d88d154cf3 100644 --- a/src/Functions/toISOYear.cpp +++ b/src/Functions/toISOYear.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToISOYear = FunctionDateOrDateTimeToSomething; -void registerFunctionToISOYear(FunctionFactory & factory) +REGISTER_FUNCTION(ToISOYear) { factory.registerFunction(); } diff --git a/src/Functions/toJSONString.cpp b/src/Functions/toJSONString.cpp index 0ec13e019f7..de82d7cb50a 100644 --- a/src/Functions/toJSONString.cpp +++ b/src/Functions/toJSONString.cpp @@ -55,7 +55,7 @@ namespace }; } -void registerFunctionToJSONString(FunctionFactory & factory) +REGISTER_FUNCTION(ToJSONString) { factory.registerFunction(); } diff --git a/src/Functions/toLastDayOfMonth.cpp b/src/Functions/toLastDayOfMonth.cpp index 7a15ede4e96..38d42521f00 100644 --- a/src/Functions/toLastDayOfMonth.cpp +++ b/src/Functions/toLastDayOfMonth.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionToLastDayOfMonth = FunctionDateOrDateTimeToSomething; -void registerFunctionToLastDayOfMonth(FunctionFactory & factory) +REGISTER_FUNCTION(ToLastDayOfMonth) { factory.registerFunction(); diff --git a/src/Functions/toLowCardinality.cpp b/src/Functions/toLowCardinality.cpp index eff01b144d9..9b050d47a90 100644 --- a/src/Functions/toLowCardinality.cpp +++ b/src/Functions/toLowCardinality.cpp @@ -50,7 +50,7 @@ public: } -void registerFunctionToLowCardinality(FunctionFactory & factory) +REGISTER_FUNCTION(ToLowCardinality) { factory.registerFunction(); } diff --git a/src/Functions/toMinute.cpp b/src/Functions/toMinute.cpp index 27a79bb0e53..c84b0876a24 100644 --- a/src/Functions/toMinute.cpp +++ b/src/Functions/toMinute.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToMinute = FunctionDateOrDateTimeToSomething; -void registerFunctionToMinute(FunctionFactory & factory) +REGISTER_FUNCTION(ToMinute) { factory.registerFunction(); /// MysQL compatibility alias. diff --git a/src/Functions/toModifiedJulianDay.cpp b/src/Functions/toModifiedJulianDay.cpp index 7873baa11af..2e9b6eb1c79 100644 --- a/src/Functions/toModifiedJulianDay.cpp +++ b/src/Functions/toModifiedJulianDay.cpp @@ -228,7 +228,7 @@ namespace DB static constexpr auto name = "toModifiedJulianDayOrNull"; }; - void registerFunctionToModifiedJulianDay(FunctionFactory & factory) + REGISTER_FUNCTION(ToModifiedJulianDay) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/toMonday.cpp b/src/Functions/toMonday.cpp index 02a1d65b309..89145634e45 100644 --- a/src/Functions/toMonday.cpp +++ b/src/Functions/toMonday.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionToMonday = FunctionDateOrDateTimeToSomething; -void registerFunctionToMonday(FunctionFactory & factory) +REGISTER_FUNCTION(ToMonday) { factory.registerFunction(); } diff --git a/src/Functions/toMonth.cpp b/src/Functions/toMonth.cpp index a61543a897b..1364ad5a997 100644 --- a/src/Functions/toMonth.cpp +++ b/src/Functions/toMonth.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToMonth = FunctionDateOrDateTimeToSomething; -void registerFunctionToMonth(FunctionFactory & factory) +REGISTER_FUNCTION(ToMonth) { factory.registerFunction(); /// MysQL compatibility alias. diff --git a/src/Functions/toNullable.cpp b/src/Functions/toNullable.cpp index 16d9f9198cd..aea64c8d4ad 100644 --- a/src/Functions/toNullable.cpp +++ b/src/Functions/toNullable.cpp @@ -45,7 +45,7 @@ public: } -void registerFunctionToNullable(FunctionFactory & factory) +REGISTER_FUNCTION(ToNullable) { factory.registerFunction(); } diff --git a/src/Functions/toQuarter.cpp b/src/Functions/toQuarter.cpp index 55fc3e83b1b..e9c1795121f 100644 --- a/src/Functions/toQuarter.cpp +++ b/src/Functions/toQuarter.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToQuarter = FunctionDateOrDateTimeToSomething; -void registerFunctionToQuarter(FunctionFactory & factory) +REGISTER_FUNCTION(ToQuarter) { factory.registerFunction(); /// MysQL compatibility alias. diff --git a/src/Functions/toRelativeDayNum.cpp b/src/Functions/toRelativeDayNum.cpp index 7b4ca094843..241104493cd 100644 --- a/src/Functions/toRelativeDayNum.cpp +++ b/src/Functions/toRelativeDayNum.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToRelativeDayNum = FunctionDateOrDateTimeToSomething; -void registerFunctionToRelativeDayNum(FunctionFactory & factory) +REGISTER_FUNCTION(ToRelativeDayNum) { factory.registerFunction(); } diff --git a/src/Functions/toRelativeHourNum.cpp b/src/Functions/toRelativeHourNum.cpp index e49906c8d56..2404d73c450 100644 --- a/src/Functions/toRelativeHourNum.cpp +++ b/src/Functions/toRelativeHourNum.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToRelativeHourNum = FunctionDateOrDateTimeToSomething; -void registerFunctionToRelativeHourNum(FunctionFactory & factory) +REGISTER_FUNCTION(ToRelativeHourNum) { factory.registerFunction(); } diff --git a/src/Functions/toRelativeMinuteNum.cpp b/src/Functions/toRelativeMinuteNum.cpp index 5e2b737a814..a5ecada1e92 100644 --- a/src/Functions/toRelativeMinuteNum.cpp +++ b/src/Functions/toRelativeMinuteNum.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToRelativeMinuteNum = FunctionDateOrDateTimeToSomething; -void registerFunctionToRelativeMinuteNum(FunctionFactory & factory) +REGISTER_FUNCTION(ToRelativeMinuteNum) { factory.registerFunction(); } diff --git a/src/Functions/toRelativeMonthNum.cpp b/src/Functions/toRelativeMonthNum.cpp index 695ed89ec18..8f46e04e483 100644 --- a/src/Functions/toRelativeMonthNum.cpp +++ b/src/Functions/toRelativeMonthNum.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToRelativeMonthNum = FunctionDateOrDateTimeToSomething; -void registerFunctionToRelativeMonthNum(FunctionFactory & factory) +REGISTER_FUNCTION(ToRelativeMonthNum) { factory.registerFunction(); } diff --git a/src/Functions/toRelativeQuarterNum.cpp b/src/Functions/toRelativeQuarterNum.cpp index fdd5ed57c89..8ea0c42ef09 100644 --- a/src/Functions/toRelativeQuarterNum.cpp +++ b/src/Functions/toRelativeQuarterNum.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToRelativeQuarterNum = FunctionDateOrDateTimeToSomething; -void registerFunctionToRelativeQuarterNum(FunctionFactory & factory) +REGISTER_FUNCTION(ToRelativeQuarterNum) { factory.registerFunction(); } diff --git a/src/Functions/toRelativeSecondNum.cpp b/src/Functions/toRelativeSecondNum.cpp index 02b63d86133..7af41ab8334 100644 --- a/src/Functions/toRelativeSecondNum.cpp +++ b/src/Functions/toRelativeSecondNum.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToRelativeSecondNum = FunctionDateOrDateTimeToSomething; -void registerFunctionToRelativeSecondNum(FunctionFactory & factory) +REGISTER_FUNCTION(ToRelativeSecondNum) { factory.registerFunction(); } diff --git a/src/Functions/toRelativeWeekNum.cpp b/src/Functions/toRelativeWeekNum.cpp index b0c4aa730fb..fe7aec3fd9a 100644 --- a/src/Functions/toRelativeWeekNum.cpp +++ b/src/Functions/toRelativeWeekNum.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToRelativeWeekNum = FunctionDateOrDateTimeToSomething; -void registerFunctionToRelativeWeekNum(FunctionFactory & factory) +REGISTER_FUNCTION(ToRelativeWeekNum) { factory.registerFunction(); } diff --git a/src/Functions/toRelativeYearNum.cpp b/src/Functions/toRelativeYearNum.cpp index 94717a8abb3..4574d8513e0 100644 --- a/src/Functions/toRelativeYearNum.cpp +++ b/src/Functions/toRelativeYearNum.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToRelativeYearNum = FunctionDateOrDateTimeToSomething; -void registerFunctionToRelativeYearNum(FunctionFactory & factory) +REGISTER_FUNCTION(ToRelativeYearNum) { factory.registerFunction(); } diff --git a/src/Functions/toSecond.cpp b/src/Functions/toSecond.cpp index a631a2646d6..8ab329689f7 100644 --- a/src/Functions/toSecond.cpp +++ b/src/Functions/toSecond.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToSecond = FunctionDateOrDateTimeToSomething; -void registerFunctionToSecond(FunctionFactory & factory) +REGISTER_FUNCTION(ToSecond) { factory.registerFunction(); /// MysQL compatibility alias. diff --git a/src/Functions/toStartOfDay.cpp b/src/Functions/toStartOfDay.cpp index a9632733d9b..0f3304c2283 100644 --- a/src/Functions/toStartOfDay.cpp +++ b/src/Functions/toStartOfDay.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionToStartOfDay = FunctionDateOrDateTimeToSomething; -void registerFunctionToStartOfDay(FunctionFactory & factory) +REGISTER_FUNCTION(ToStartOfDay) { factory.registerFunction(); } diff --git a/src/Functions/toStartOfFifteenMinutes.cpp b/src/Functions/toStartOfFifteenMinutes.cpp index ceac5702915..a103c4ff0eb 100644 --- a/src/Functions/toStartOfFifteenMinutes.cpp +++ b/src/Functions/toStartOfFifteenMinutes.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionToStartOfFifteenMinutes = FunctionDateOrDateTimeToSomething; -void registerFunctionToStartOfFifteenMinutes(FunctionFactory & factory) +REGISTER_FUNCTION(ToStartOfFifteenMinutes) { factory.registerFunction(); } diff --git a/src/Functions/toStartOfFiveMinutes.cpp b/src/Functions/toStartOfFiveMinutes.cpp index b311c69d8c6..17db95841e7 100644 --- a/src/Functions/toStartOfFiveMinutes.cpp +++ b/src/Functions/toStartOfFiveMinutes.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionToStartOfFiveMinutes = FunctionDateOrDateTimeToSomething; -void registerFunctionToStartOfFiveMinutes(FunctionFactory & factory) +REGISTER_FUNCTION(ToStartOfFiveMinutes) { factory.registerFunction(); factory.registerAlias("toStartOfFiveMinute", FunctionToStartOfFiveMinutes::name); diff --git a/src/Functions/toStartOfHour.cpp b/src/Functions/toStartOfHour.cpp index a2273f2e78f..d12b37ab7b6 100644 --- a/src/Functions/toStartOfHour.cpp +++ b/src/Functions/toStartOfHour.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionToStartOfHour = FunctionDateOrDateTimeToSomething; -void registerFunctionToStartOfHour(FunctionFactory & factory) +REGISTER_FUNCTION(ToStartOfHour) { factory.registerFunction(); } diff --git a/src/Functions/toStartOfISOYear.cpp b/src/Functions/toStartOfISOYear.cpp index a9b338a0a82..366ba8dd09f 100644 --- a/src/Functions/toStartOfISOYear.cpp +++ b/src/Functions/toStartOfISOYear.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionToStartOfISOYear = FunctionDateOrDateTimeToSomething; -void registerFunctionToStartOfISOYear(FunctionFactory & factory) +REGISTER_FUNCTION(ToStartOfISOYear) { factory.registerFunction(); } diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 413ae7f63c3..49c3b56c186 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -537,7 +537,7 @@ private: } -void registerFunctionToStartOfInterval(FunctionFactory & factory) +REGISTER_FUNCTION(ToStartOfInterval) { factory.registerFunction(); } diff --git a/src/Functions/toStartOfMinute.cpp b/src/Functions/toStartOfMinute.cpp index 5fa7fc8ab4b..dddfaee93ec 100644 --- a/src/Functions/toStartOfMinute.cpp +++ b/src/Functions/toStartOfMinute.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionToStartOfMinute = FunctionDateOrDateTimeToSomething; -void registerFunctionToStartOfMinute(FunctionFactory & factory) +REGISTER_FUNCTION(ToStartOfMinute) { factory.registerFunction(); } diff --git a/src/Functions/toStartOfMonth.cpp b/src/Functions/toStartOfMonth.cpp index 8bad02acc81..9674462097b 100644 --- a/src/Functions/toStartOfMonth.cpp +++ b/src/Functions/toStartOfMonth.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionToStartOfMonth = FunctionDateOrDateTimeToSomething; -void registerFunctionToStartOfMonth(FunctionFactory & factory) +REGISTER_FUNCTION(ToStartOfMonth) { factory.registerFunction(); } diff --git a/src/Functions/toStartOfQuarter.cpp b/src/Functions/toStartOfQuarter.cpp index c2d2697e86c..c7d69743198 100644 --- a/src/Functions/toStartOfQuarter.cpp +++ b/src/Functions/toStartOfQuarter.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionToStartOfQuarter = FunctionDateOrDateTimeToSomething; -void registerFunctionToStartOfQuarter(FunctionFactory & factory) +REGISTER_FUNCTION(ToStartOfQuarter) { factory.registerFunction(); } diff --git a/src/Functions/toStartOfSecond.cpp b/src/Functions/toStartOfSecond.cpp index d4197a36e6c..3ceb6f94df7 100644 --- a/src/Functions/toStartOfSecond.cpp +++ b/src/Functions/toStartOfSecond.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionToStartOfSecond = FunctionDateOrDateTimeToSomething; -void registerFunctionToStartOfSecond(FunctionFactory & factory) +REGISTER_FUNCTION(ToStartOfSecond) { factory.registerFunction(); } diff --git a/src/Functions/toStartOfSubsecond.cpp b/src/Functions/toStartOfSubsecond.cpp index b2257c5e3cd..5878eaf1ed6 100644 --- a/src/Functions/toStartOfSubsecond.cpp +++ b/src/Functions/toStartOfSubsecond.cpp @@ -8,21 +8,21 @@ namespace DB using FunctionToStartOfMillisecond = FunctionDateOrDateTimeToSomething; -void registerFunctionToStartOfMillisecond(FunctionFactory & factory) +REGISTER_FUNCTION(ToStartOfMillisecond) { factory.registerFunction(); } using FunctionToStartOfMicrosecond = FunctionDateOrDateTimeToSomething; -void registerFunctionToStartOfMicrosecond(FunctionFactory & factory) +REGISTER_FUNCTION(ToStartOfMicrosecond) { factory.registerFunction(); } using FunctionToStartOfNanosecond = FunctionDateOrDateTimeToSomething; -void registerFunctionToStartOfNanosecond(FunctionFactory & factory) +REGISTER_FUNCTION(ToStartOfNanosecond) { factory.registerFunction(); } diff --git a/src/Functions/toStartOfTenMinutes.cpp b/src/Functions/toStartOfTenMinutes.cpp index 8cff4cc13cd..8f05e080b92 100644 --- a/src/Functions/toStartOfTenMinutes.cpp +++ b/src/Functions/toStartOfTenMinutes.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionToStartOfTenMinutes = FunctionDateOrDateTimeToSomething; -void registerFunctionToStartOfTenMinutes(FunctionFactory & factory) +REGISTER_FUNCTION(ToStartOfTenMinutes) { factory.registerFunction(); } diff --git a/src/Functions/toStartOfYear.cpp b/src/Functions/toStartOfYear.cpp index cef10c1534d..13729f2f812 100644 --- a/src/Functions/toStartOfYear.cpp +++ b/src/Functions/toStartOfYear.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionToStartOfYear = FunctionDateOrDateTimeToSomething; -void registerFunctionToStartOfYear(FunctionFactory & factory) +REGISTER_FUNCTION(ToStartOfYear) { factory.registerFunction(); } diff --git a/src/Functions/toTime.cpp b/src/Functions/toTime.cpp index b7ff56a18c4..2f5f3c9a3ca 100644 --- a/src/Functions/toTime.cpp +++ b/src/Functions/toTime.cpp @@ -8,7 +8,7 @@ namespace DB using FunctionToTime = FunctionDateOrDateTimeToSomething; -void registerFunctionToTime(FunctionFactory & factory) +REGISTER_FUNCTION(ToTime) { factory.registerFunction(); } diff --git a/src/Functions/toTimezone.cpp b/src/Functions/toTimezone.cpp index 092b79131d8..5f6784c5655 100644 --- a/src/Functions/toTimezone.cpp +++ b/src/Functions/toTimezone.cpp @@ -123,7 +123,7 @@ public: } -void registerFunctionToTimeZone(FunctionFactory & factory) +REGISTER_FUNCTION(ToTimeZone) { factory.registerFunction(); factory.registerAlias("toTimeZone", "toTimezone"); diff --git a/src/Functions/toTypeName.cpp b/src/Functions/toTypeName.cpp index d9ec08642ca..e66de5f71c6 100644 --- a/src/Functions/toTypeName.cpp +++ b/src/Functions/toTypeName.cpp @@ -62,7 +62,7 @@ public: } -void registerFunctionToTypeName(FunctionFactory & factory) +REGISTER_FUNCTION(ToTypeName) { factory.registerFunction(); } diff --git a/src/Functions/toUnixTimestamp64Micro.cpp b/src/Functions/toUnixTimestamp64Micro.cpp index c5b841a1a81..fd35e2a7a73 100644 --- a/src/Functions/toUnixTimestamp64Micro.cpp +++ b/src/Functions/toUnixTimestamp64Micro.cpp @@ -4,7 +4,7 @@ namespace DB { -void registerToUnixTimestamp64Micro(FunctionFactory & factory) +REGISTER_FUNCTION(ToUnixTimestamp64Micro) { factory.registerFunction("toUnixTimestamp64Micro", [](ContextPtr){ return std::make_unique( diff --git a/src/Functions/toUnixTimestamp64Milli.cpp b/src/Functions/toUnixTimestamp64Milli.cpp index bfceb3708d3..e6a680f941a 100644 --- a/src/Functions/toUnixTimestamp64Milli.cpp +++ b/src/Functions/toUnixTimestamp64Milli.cpp @@ -4,7 +4,7 @@ namespace DB { -void registerToUnixTimestamp64Milli(FunctionFactory & factory) +REGISTER_FUNCTION(ToUnixTimestamp64Milli) { factory.registerFunction("toUnixTimestamp64Milli", [](ContextPtr){ return std::make_unique( diff --git a/src/Functions/toUnixTimestamp64Nano.cpp b/src/Functions/toUnixTimestamp64Nano.cpp index 2256dc369b9..257f011603c 100644 --- a/src/Functions/toUnixTimestamp64Nano.cpp +++ b/src/Functions/toUnixTimestamp64Nano.cpp @@ -4,7 +4,7 @@ namespace DB { -void registerToUnixTimestamp64Nano(FunctionFactory & factory) +REGISTER_FUNCTION(ToUnixTimestamp64Nano) { factory.registerFunction("toUnixTimestamp64Nano", [](ContextPtr){ return std::make_unique( diff --git a/src/Functions/toValidUTF8.cpp b/src/Functions/toValidUTF8.cpp index 0ee62bd4961..9874e39baa4 100644 --- a/src/Functions/toValidUTF8.cpp +++ b/src/Functions/toValidUTF8.cpp @@ -166,7 +166,7 @@ using FunctionToValidUTF8 = FunctionStringToString(); } diff --git a/src/Functions/toYYYYMM.cpp b/src/Functions/toYYYYMM.cpp index 3dfefc4adb1..5b5f6bfe2fe 100644 --- a/src/Functions/toYYYYMM.cpp +++ b/src/Functions/toYYYYMM.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToYYYYMM = FunctionDateOrDateTimeToSomething; -void registerFunctionToYYYYMM(FunctionFactory & factory) +REGISTER_FUNCTION(ToYYYYMM) { factory.registerFunction(); } diff --git a/src/Functions/toYYYYMMDD.cpp b/src/Functions/toYYYYMMDD.cpp index fa5e85341fd..ccbb774d60b 100644 --- a/src/Functions/toYYYYMMDD.cpp +++ b/src/Functions/toYYYYMMDD.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToYYYYMMDD = FunctionDateOrDateTimeToSomething; -void registerFunctionToYYYYMMDD(FunctionFactory & factory) +REGISTER_FUNCTION(ToYYYYMMDD) { factory.registerFunction(); } diff --git a/src/Functions/toYYYYMMDDhhmmss.cpp b/src/Functions/toYYYYMMDDhhmmss.cpp index 7718bbab763..fbdf70ada02 100644 --- a/src/Functions/toYYYYMMDDhhmmss.cpp +++ b/src/Functions/toYYYYMMDDhhmmss.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToYYYYMMDDhhmmss = FunctionDateOrDateTimeToSomething; -void registerFunctionToYYYYMMDDhhmmss(FunctionFactory & factory) +REGISTER_FUNCTION(ToYYYYMMDDhhmmss) { factory.registerFunction(); } diff --git a/src/Functions/toYear.cpp b/src/Functions/toYear.cpp index 8db857c132e..6658bf0e927 100644 --- a/src/Functions/toYear.cpp +++ b/src/Functions/toYear.cpp @@ -9,7 +9,7 @@ namespace DB using FunctionToYear = FunctionDateOrDateTimeToSomething; -void registerFunctionToYear(FunctionFactory & factory) +REGISTER_FUNCTION(ToYear) { factory.registerFunction(); /// MysQL compatibility alias. diff --git a/src/Functions/today.cpp b/src/Functions/today.cpp index fe63197d127..504c840efe5 100644 --- a/src/Functions/today.cpp +++ b/src/Functions/today.cpp @@ -84,7 +84,7 @@ public: } -void registerFunctionToday(FunctionFactory & factory) +REGISTER_FUNCTION(Today) { factory.registerFunction(); } diff --git a/src/Functions/tokenExtractors.cpp b/src/Functions/tokenExtractors.cpp index 5902cb5f3e5..a29d759d2ca 100644 --- a/src/Functions/tokenExtractors.cpp +++ b/src/Functions/tokenExtractors.cpp @@ -147,7 +147,7 @@ private: } }; -void registerFunctionsStringTokenExtractor(FunctionFactory & factory) +REGISTER_FUNCTION(StringTokenExtractor) { factory.registerFunction>(); factory.registerFunction>(); diff --git a/src/Functions/transform.cpp b/src/Functions/transform.cpp index de9f1a5ba05..3337e8d40a8 100644 --- a/src/Functions/transform.cpp +++ b/src/Functions/transform.cpp @@ -1217,7 +1217,7 @@ private: } -void registerFunctionTransform(FunctionFactory & factory) +REGISTER_FUNCTION(Transform) { factory.registerFunction(); } diff --git a/src/Functions/translate.cpp b/src/Functions/translate.cpp index 8342bfe236b..b3f1d5ae460 100644 --- a/src/Functions/translate.cpp +++ b/src/Functions/translate.cpp @@ -355,7 +355,7 @@ using FunctionTranslateUTF8 = FunctionTranslate(); factory.registerFunction(); diff --git a/src/Functions/trap.cpp b/src/Functions/trap.cpp index 4fa0770e61b..5bb0ed210de 100644 --- a/src/Functions/trap.cpp +++ b/src/Functions/trap.cpp @@ -177,19 +177,11 @@ public: }; -void registerFunctionTrap(FunctionFactory & factory) +REGISTER_FUNCTION(Trap) { factory.registerFunction(); } } -#else - -namespace DB -{ - class FunctionFactory; - void registerFunctionTrap(FunctionFactory &) {} -} - #endif diff --git a/src/Functions/trim.cpp b/src/Functions/trim.cpp index a1737fc12a1..58760583e8d 100644 --- a/src/Functions/trim.cpp +++ b/src/Functions/trim.cpp @@ -107,7 +107,7 @@ using FunctionTrimBoth = FunctionStringToString, } -void registerFunctionTrim(FunctionFactory & factory) +REGISTER_FUNCTION(Trim) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/tryBase64Decode.cpp b/src/Functions/tryBase64Decode.cpp index b43ab2cbeea..1102c7a3418 100644 --- a/src/Functions/tryBase64Decode.cpp +++ b/src/Functions/tryBase64Decode.cpp @@ -5,7 +5,7 @@ namespace DB { -void registerFunctionTryBase64Decode(FunctionFactory & factory) +REGISTER_FUNCTION(TryBase64Decode) { factory.registerFunction>(); } diff --git a/src/Functions/tuple.cpp b/src/Functions/tuple.cpp index 5e85984bee0..545c6597811 100644 --- a/src/Functions/tuple.cpp +++ b/src/Functions/tuple.cpp @@ -83,7 +83,7 @@ public: } -void registerFunctionTuple(FunctionFactory & factory) +REGISTER_FUNCTION(Tuple) { factory.registerFunction(); } diff --git a/src/Functions/tupleElement.cpp b/src/Functions/tupleElement.cpp index 92ca6b85714..4f7ddda6b0b 100644 --- a/src/Functions/tupleElement.cpp +++ b/src/Functions/tupleElement.cpp @@ -259,7 +259,7 @@ private: } -void registerFunctionTupleElement(FunctionFactory & factory) +REGISTER_FUNCTION(TupleElement) { factory.registerFunction(); } diff --git a/src/Functions/tupleHammingDistance.cpp b/src/Functions/tupleHammingDistance.cpp index 11c8cf84b40..adc063bfa81 100644 --- a/src/Functions/tupleHammingDistance.cpp +++ b/src/Functions/tupleHammingDistance.cpp @@ -139,7 +139,7 @@ public: } }; -void registerFunctionTupleHammingDistance(FunctionFactory & factory) +REGISTER_FUNCTION(TupleHammingDistance) { factory.registerFunction(); } diff --git a/src/Functions/tupleToNameValuePairs.cpp b/src/Functions/tupleToNameValuePairs.cpp index 1d51b77bbee..998e0da4f0c 100644 --- a/src/Functions/tupleToNameValuePairs.cpp +++ b/src/Functions/tupleToNameValuePairs.cpp @@ -129,7 +129,7 @@ public: } -void registerFunctionTupleToNameValuePairs(FunctionFactory & factory) +REGISTER_FUNCTION(TupleToNameValuePairs) { factory.registerFunction(); } diff --git a/src/Functions/upper.cpp b/src/Functions/upper.cpp index 515574e2a09..05a125379d9 100644 --- a/src/Functions/upper.cpp +++ b/src/Functions/upper.cpp @@ -16,7 +16,7 @@ using FunctionUpper = FunctionStringToString, NameUpper } -void registerFunctionUpper(FunctionFactory & factory) +REGISTER_FUNCTION(Upper) { factory.registerFunction(FunctionFactory::CaseInsensitive); factory.registerAlias("ucase", FunctionUpper::name, FunctionFactory::CaseInsensitive); diff --git a/src/Functions/upperUTF8.cpp b/src/Functions/upperUTF8.cpp index 1a85b133757..659e67f0ef3 100644 --- a/src/Functions/upperUTF8.cpp +++ b/src/Functions/upperUTF8.cpp @@ -18,7 +18,7 @@ using FunctionUpperUTF8 = FunctionStringToString(); } diff --git a/src/Functions/validateNestedArraySizes.cpp b/src/Functions/validateNestedArraySizes.cpp index 42b0e401fa0..3b005f4d653 100644 --- a/src/Functions/validateNestedArraySizes.cpp +++ b/src/Functions/validateNestedArraySizes.cpp @@ -119,7 +119,7 @@ ColumnPtr FunctionValidateNestedArraySizes::executeImpl( return ColumnUInt8::create(input_rows_count, 1); } -void registerFunctionValidateNestedArraySizes(FunctionFactory & factory) +REGISTER_FUNCTION(ValidateNestedArraySizes) { factory.registerFunction(); } diff --git a/src/Functions/vectorFunctions.cpp b/src/Functions/vectorFunctions.cpp index d46d12379a7..20571f67eff 100644 --- a/src/Functions/vectorFunctions.cpp +++ b/src/Functions/vectorFunctions.cpp @@ -1272,7 +1272,7 @@ using TupleOrArrayFunctionLpDistance = TupleOrArrayFunction; using TupleOrArrayFunctionLinfDistance = TupleOrArrayFunction; using TupleOrArrayFunctionCosineDistance = TupleOrArrayFunction; -void registerVectorFunctions(FunctionFactory & factory) +REGISTER_FUNCTION(VectorFunctions) { factory.registerFunction(); factory.registerAlias("vectorSum", FunctionTuplePlus::name, FunctionFactory::CaseInsensitive); diff --git a/src/Functions/visibleWidth.cpp b/src/Functions/visibleWidth.cpp index 4e09aeb399a..d4f6de404ff 100644 --- a/src/Functions/visibleWidth.cpp +++ b/src/Functions/visibleWidth.cpp @@ -76,7 +76,7 @@ public: }; -void registerFunctionVisibleWidth(FunctionFactory & factory) +REGISTER_FUNCTION(VisibleWidth) { factory.registerFunction(); } diff --git a/src/Functions/visitParamExtractBool.cpp b/src/Functions/visitParamExtractBool.cpp index 48fb78ba9b6..e5a2277b443 100644 --- a/src/Functions/visitParamExtractBool.cpp +++ b/src/Functions/visitParamExtractBool.cpp @@ -22,7 +22,7 @@ using FunctionVisitParamExtractBool = FunctionsStringSearch>; -void registerFunctionVisitParamExtractBool(FunctionFactory & factory) +REGISTER_FUNCTION(VisitParamExtractBool) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/visitParamExtractFloat.cpp b/src/Functions/visitParamExtractFloat.cpp index e7967b6de2c..ee00f960f8f 100644 --- a/src/Functions/visitParamExtractFloat.cpp +++ b/src/Functions/visitParamExtractFloat.cpp @@ -12,7 +12,7 @@ using FunctionVisitParamExtractFloat = FunctionsStringSearch>>; -void registerFunctionVisitParamExtractFloat(FunctionFactory & factory) +REGISTER_FUNCTION(VisitParamExtractFloat) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/visitParamExtractInt.cpp b/src/Functions/visitParamExtractInt.cpp index b7f1050972c..30b373182ea 100644 --- a/src/Functions/visitParamExtractInt.cpp +++ b/src/Functions/visitParamExtractInt.cpp @@ -12,7 +12,7 @@ using FunctionVisitParamExtractInt = FunctionsStringSearch>>; -void registerFunctionVisitParamExtractInt(FunctionFactory & factory) +REGISTER_FUNCTION(VisitParamExtractInt) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/visitParamExtractRaw.cpp b/src/Functions/visitParamExtractRaw.cpp index 734fe107557..ab21fdf6e98 100644 --- a/src/Functions/visitParamExtractRaw.cpp +++ b/src/Functions/visitParamExtractRaw.cpp @@ -62,7 +62,7 @@ using FunctionVisitParamExtractRaw = FunctionsStringSearchToString, NameSimpleJSONExtractRaw>; -void registerFunctionVisitParamExtractRaw(FunctionFactory & factory) +REGISTER_FUNCTION(VisitParamExtractRaw) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/visitParamExtractString.cpp b/src/Functions/visitParamExtractString.cpp index 23f24b9e3b8..df640cef371 100644 --- a/src/Functions/visitParamExtractString.cpp +++ b/src/Functions/visitParamExtractString.cpp @@ -23,7 +23,7 @@ using FunctionVisitParamExtractString = FunctionsStringSearchToString, NameSimpleJSONExtractString>; -void registerFunctionVisitParamExtractString(FunctionFactory & factory) +REGISTER_FUNCTION(VisitParamExtractString) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/visitParamExtractUInt.cpp b/src/Functions/visitParamExtractUInt.cpp index d89b796263e..1612c91984d 100644 --- a/src/Functions/visitParamExtractUInt.cpp +++ b/src/Functions/visitParamExtractUInt.cpp @@ -13,7 +13,7 @@ struct NameSimpleJSONExtractUInt { static constexpr auto name = "simpleJSONExt using FunctionSimpleJSONExtractUInt = FunctionsStringSearch>>; -void registerFunctionVisitParamExtractUInt(FunctionFactory & factory) +REGISTER_FUNCTION(VisitParamExtractUInt) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/visitParamHas.cpp b/src/Functions/visitParamHas.cpp index 71d69ef5768..9e481fb44cc 100644 --- a/src/Functions/visitParamHas.cpp +++ b/src/Functions/visitParamHas.cpp @@ -22,7 +22,7 @@ using FunctionVisitParamHas = FunctionsStringSearch>; -void registerFunctionVisitParamHas(FunctionFactory & factory) +REGISTER_FUNCTION(VisitParamHas) { factory.registerFunction(); factory.registerFunction(); diff --git a/src/Functions/wkt.cpp b/src/Functions/wkt.cpp index 732441eeef2..fc9ef75a1e2 100644 --- a/src/Functions/wkt.cpp +++ b/src/Functions/wkt.cpp @@ -68,7 +68,7 @@ public: } }; -void registerFunctionWkt(FunctionFactory & factory) +REGISTER_FUNCTION(Wkt) { factory.registerFunction(); } diff --git a/src/Functions/yesterday.cpp b/src/Functions/yesterday.cpp index 364d4721b34..53b73a7a71e 100644 --- a/src/Functions/yesterday.cpp +++ b/src/Functions/yesterday.cpp @@ -81,7 +81,7 @@ public: } }; -void registerFunctionYesterday(FunctionFactory & factory) +REGISTER_FUNCTION(Yesterday) { factory.registerFunction(); } diff --git a/src/Functions/ztest.cpp b/src/Functions/ztest.cpp index c80b92960e9..26e55a4f934 100644 --- a/src/Functions/ztest.cpp +++ b/src/Functions/ztest.cpp @@ -217,7 +217,7 @@ public: }; -void registerFunctionZTest(FunctionFactory & factory) +REGISTER_FUNCTION(ZTest) { factory.registerFunction(); } From 83dfbe3b9a51e591910d99de25dc36c20fd7776b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 28 Jul 2022 19:19:37 +0300 Subject: [PATCH 083/139] Fix building keeper-data-dumper Signed-off-by: Azat Khuzhin --- utils/keeper-data-dumper/main.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/keeper-data-dumper/main.cpp b/utils/keeper-data-dumper/main.cpp index b55dab8399b..0762c740ac1 100644 --- a/utils/keeper-data-dumper/main.cpp +++ b/utils/keeper-data-dumper/main.cpp @@ -62,7 +62,8 @@ int main(int argc, char *argv[]) ResponsesQueue queue(std::numeric_limits::max()); SnapshotsQueue snapshots_queue{1}; CoordinationSettingsPtr settings = std::make_shared(); - auto state_machine = std::make_shared(queue, snapshots_queue, argv[1], settings); + KeeperContextPtr keeper_context = std::make_shared(); + auto state_machine = std::make_shared(queue, snapshots_queue, argv[1], settings, keeper_context); state_machine->init(); size_t last_commited_index = state_machine->last_commit_index(); From 2901ddc217dd80b3334fb902eaf7e146bcbb0bb6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 28 Jul 2022 19:53:48 +0300 Subject: [PATCH 084/139] Remove --verbose=1 for cmake on configuration stage --verbose is supported only for make/install, while 3.16 is OK (ubuntu 20.04), newer cmake (3.23) does not, and simply fails: # cmake --debug-trycompile --verbose=1 /foo debug trycompile on CMake Error: Unknown argument --verbose=1 CMake Error: Run 'cmake --help' for all supported options. And this is how it works with 3.16: # cmake --debug-trycompile --verbose=1 /foo debug trycompile on CMake Error: The source directory "/foo" does not exist. Specify --help for usage, or press the help button on the CMake GUI. Anyway this script sets CMAKE_VERBOSE_MAKEFILE, and this is what --verbose does. Signed-off-by: Azat Khuzhin --- docker/packager/binary/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 87f98df2ad8..da911a144f0 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -55,7 +55,7 @@ ccache --zero-stats ||: if [ "$BUILD_MUSL_KEEPER" == "1" ] then # build keeper with musl separately - cmake --debug-trycompile --verbose=1 -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" .. + cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" .. # shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty. ninja $NINJA_FLAGS clickhouse-keeper @@ -70,10 +70,10 @@ then rm -f CMakeCache.txt # Build the rest of binaries - cmake --debug-trycompile --verbose=1 -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" .. + cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" .. else # Build everything - cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" .. + cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" .. fi if [ "coverity" == "$COMBINED_OUTPUT" ] From 4b0594633b937f5e87ed0eb4a2494de33456ce70 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 28 Jul 2022 19:58:53 +0300 Subject: [PATCH 085/139] Build utils on CI when they was requested clickhouse-bundle does not includes utils, and to avoid modifying each utility it is better to build all, anyway this is not a common build. Signed-off-by: Azat Khuzhin --- docker/packager/binary/build.sh | 2 +- docker/packager/packager | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index da911a144f0..b5db874f33c 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -88,7 +88,7 @@ fi # No quotes because I want it to expand to nothing if empty. # shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty. -$SCAN_WRAPPER ninja $NINJA_FLAGS clickhouse-bundle +$SCAN_WRAPPER ninja $NINJA_FLAGS $BUILD_TARGET ls -la ./programs diff --git a/docker/packager/packager b/docker/packager/packager index 0b00bc4e9c0..1f41b3e69c6 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -134,6 +134,7 @@ def parse_env_variables( result = [] result.append("OUTPUT_DIR=/output") cmake_flags = ["$CMAKE_FLAGS"] + build_target = "clickhouse-bundle" is_cross_darwin = compiler.endswith(DARWIN_SUFFIX) is_cross_darwin_arm = compiler.endswith(DARWIN_ARM_SUFFIX) @@ -268,15 +269,20 @@ def parse_env_variables( # we have to build them at least in some way in CI. The split build is # probably the least heavy disk-wise. cmake_flags.append("-DENABLE_UTILS=1") + # utils are not included into clickhouse-bundle, so build everything + build_target = "all" if clang_tidy: cmake_flags.append("-DENABLE_CLANG_TIDY=1") - cmake_flags.append("-DENABLE_UTILS=1") cmake_flags.append("-DENABLE_TESTS=1") cmake_flags.append("-DENABLE_EXAMPLES=1") # Don't stop on first error to find more clang-tidy errors in one run. result.append("NINJA_FLAGS=-k0") + cmake_flags.append("-DENABLE_UTILS=1") + # utils are not included into clickhouse-bundle, so build everything + build_target = "all" + if with_coverage: cmake_flags.append("-DWITH_COVERAGE=1") @@ -290,6 +296,7 @@ def parse_env_variables( cmake_flags.append("-DCLICKHOUSE_OFFICIAL_BUILD=1") result.append('CMAKE_FLAGS="' + " ".join(cmake_flags) + '"') + result.append(f"BUILD_TARGET={build_target}") return result From b90152b6ec8ae50b34f2e48d3308e85f7dbb25ad Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 28 Jul 2022 22:52:19 +0300 Subject: [PATCH 086/139] Fix clickhouse-su building in splitted build - Add status log message - Add it to clickhouse-bundle in shared build - Move clickhouse-su.cpp into su.cpp, since executable does not have include directories of linked libraries (dbms here), only clickhouse-lib-su does, hence it cannot find includes CI: https://github.com/ClickHouse/ClickHouse/runs/7566319416?check_suite_focus=true Signed-off-by: Azat Khuzhin --- programs/CMakeLists.txt | 10 +++ programs/su/CMakeLists.txt | 2 +- programs/su/clickhouse-su.cpp | 147 +--------------------------------- programs/su/su.cpp | 145 +++++++++++++++++++++++++++++++++ 4 files changed, 158 insertions(+), 146 deletions(-) create mode 100644 programs/su/su.cpp diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 76799350119..6ebbe912724 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -195,6 +195,12 @@ else() message(STATUS "ClickHouse disks mode: OFF") endif() +if (ENABLE_CLICKHOUSE_SU) + message(STATUS "ClickHouse su: ON") +else() + message(STATUS "ClickHouse su: OFF") +endif() + configure_file (config_tools.h.in ${ConfigIncludePath}/config_tools.h) macro(clickhouse_target_link_split_lib target name) @@ -367,6 +373,10 @@ if (CLICKHOUSE_SPLIT_BINARY) list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-keeper-converter) endif () + if (ENABLE_CLICKHOUSE_SU) + list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-su) + endif () + set_target_properties(${CLICKHOUSE_ALL_TARGETS} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_ALL_TARGETS}) diff --git a/programs/su/CMakeLists.txt b/programs/su/CMakeLists.txt index df207e16f6e..3228eef8b8d 100644 --- a/programs/su/CMakeLists.txt +++ b/programs/su/CMakeLists.txt @@ -1,3 +1,3 @@ -set (CLICKHOUSE_SU_SOURCES clickhouse-su.cpp) +set (CLICKHOUSE_SU_SOURCES clickhouse-su.cpp su.cpp) set (CLICKHOUSE_SU_LINK PRIVATE dbms) clickhouse_program_add(su) diff --git a/programs/su/clickhouse-su.cpp b/programs/su/clickhouse-su.cpp index 0979abf353d..bb0967ca271 100644 --- a/programs/su/clickhouse-su.cpp +++ b/programs/su/clickhouse-su.cpp @@ -1,145 +1,2 @@ -#include -#include -#include -#include - -#include -#include -#include -#include - - -/// "su" means "set user" -/// In fact, this program can set Unix user and group. -/// -/// Usage: -/// clickhouse su user[:group] args... -/// -/// - will set user and, optionally, group and exec the remaining args. -/// user and group can be numeric identifiers or strings. -/// -/// The motivation for this tool is very obscure and idiosyncratic. It is needed for Docker. -/// People want to run programs inside Docker with dropped privileges (less than root). -/// But the standard Linux "su" program is not suitable for usage inside Docker, -/// because it is creating pseudoterminals to avoid hijacking input from the terminal, for security, -/// but Docker is also doing something with the terminal and it is incompatible. -/// For this reason, people use alternative and less "secure" versions of "su" tools like "gosu" or "su-exec". -/// But it would be very strange to use 3rd-party software only to do two-three syscalls. -/// That's why we provide this tool. -/// -/// Note: ClickHouse does not need Docker at all and works better without Docker. -/// ClickHouse has no dependencies, it is packaged and distributed in single binary. -/// There is no reason to use Docker unless you are already running all your software in Docker. - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; - extern const int SYSTEM_ERROR; -} - -void setUserAndGroup(std::string arg_uid, std::string arg_gid) -{ - static constexpr size_t buf_size = 16384; /// Linux man page says it is enough. Nevertheless, we will check if it's not enough and throw. - std::unique_ptr buf(new char[buf_size]); - - /// Set the group first, because if we set user, the privileges will be already dropped and we will not be able to set the group later. - - if (!arg_gid.empty()) - { - gid_t gid = 0; - if (!tryParse(gid, arg_gid) || gid == 0) - { - group entry{}; - group * result{}; - - if (0 != getgrnam_r(arg_gid.data(), &entry, buf.get(), buf_size, &result)) - throwFromErrno(fmt::format("Cannot do 'getgrnam_r' to obtain gid from group name ({})", arg_gid), ErrorCodes::SYSTEM_ERROR); - - if (!result) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Group {} is not found in the system", arg_gid); - - gid = entry.gr_gid; - } - - if (gid == 0 && getgid() != 0) - throw Exception("Group has id 0, but dropping privileges to gid 0 does not make sense", ErrorCodes::BAD_ARGUMENTS); - - if (0 != setgid(gid)) - throwFromErrno(fmt::format("Cannot do 'setgid' to user ({})", arg_gid), ErrorCodes::SYSTEM_ERROR); - } - - if (!arg_uid.empty()) - { - /// Is it numeric id or name? - uid_t uid = 0; - if (!tryParse(uid, arg_uid) || uid == 0) - { - passwd entry{}; - passwd * result{}; - - if (0 != getpwnam_r(arg_uid.data(), &entry, buf.get(), buf_size, &result)) - throwFromErrno(fmt::format("Cannot do 'getpwnam_r' to obtain uid from user name ({})", arg_uid), ErrorCodes::SYSTEM_ERROR); - - if (!result) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "User {} is not found in the system", arg_uid); - - uid = entry.pw_uid; - } - - if (uid == 0 && getuid() != 0) - throw Exception("User has id 0, but dropping privileges to uid 0 does not make sense", ErrorCodes::BAD_ARGUMENTS); - - if (0 != setuid(uid)) - throwFromErrno(fmt::format("Cannot do 'setuid' to user ({})", arg_uid), ErrorCodes::SYSTEM_ERROR); - } -} - -} - - -int mainEntryClickHouseSU(int argc, char ** argv) -try -{ - using namespace DB; - - if (argc < 3) - { - std::cout << "Usage: ./clickhouse su user:group ..." << std::endl; - exit(0); - } - - std::string_view user_and_group = argv[1]; - - std::string user; - std::string group; - - auto pos = user_and_group.find(':'); - if (pos == std::string_view::npos) - { - user = user_and_group; - } - else - { - user = user_and_group.substr(0, pos); - group = user_and_group.substr(pos + 1); - } - - setUserAndGroup(std::move(user), std::move(group)); - - std::vector new_argv; - new_argv.reserve(argc - 1); - new_argv.insert(new_argv.begin(), argv + 2, argv + argc); - new_argv.push_back(nullptr); - - execvp(new_argv.front(), new_argv.data()); - - throwFromErrno("Cannot execvp", ErrorCodes::SYSTEM_ERROR); -} -catch (...) -{ - std::cerr << DB::getCurrentExceptionMessage(false) << '\n'; - return 1; -} +int mainEntryClickHouseSU(int argc, char ** argv); +int main(int argc_, char ** argv_) { return mainEntryClickHouseSU(argc_, argv_); } diff --git a/programs/su/su.cpp b/programs/su/su.cpp new file mode 100644 index 00000000000..0979abf353d --- /dev/null +++ b/programs/su/su.cpp @@ -0,0 +1,145 @@ +#include +#include +#include +#include + +#include +#include +#include +#include + + +/// "su" means "set user" +/// In fact, this program can set Unix user and group. +/// +/// Usage: +/// clickhouse su user[:group] args... +/// +/// - will set user and, optionally, group and exec the remaining args. +/// user and group can be numeric identifiers or strings. +/// +/// The motivation for this tool is very obscure and idiosyncratic. It is needed for Docker. +/// People want to run programs inside Docker with dropped privileges (less than root). +/// But the standard Linux "su" program is not suitable for usage inside Docker, +/// because it is creating pseudoterminals to avoid hijacking input from the terminal, for security, +/// but Docker is also doing something with the terminal and it is incompatible. +/// For this reason, people use alternative and less "secure" versions of "su" tools like "gosu" or "su-exec". +/// But it would be very strange to use 3rd-party software only to do two-three syscalls. +/// That's why we provide this tool. +/// +/// Note: ClickHouse does not need Docker at all and works better without Docker. +/// ClickHouse has no dependencies, it is packaged and distributed in single binary. +/// There is no reason to use Docker unless you are already running all your software in Docker. + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int SYSTEM_ERROR; +} + +void setUserAndGroup(std::string arg_uid, std::string arg_gid) +{ + static constexpr size_t buf_size = 16384; /// Linux man page says it is enough. Nevertheless, we will check if it's not enough and throw. + std::unique_ptr buf(new char[buf_size]); + + /// Set the group first, because if we set user, the privileges will be already dropped and we will not be able to set the group later. + + if (!arg_gid.empty()) + { + gid_t gid = 0; + if (!tryParse(gid, arg_gid) || gid == 0) + { + group entry{}; + group * result{}; + + if (0 != getgrnam_r(arg_gid.data(), &entry, buf.get(), buf_size, &result)) + throwFromErrno(fmt::format("Cannot do 'getgrnam_r' to obtain gid from group name ({})", arg_gid), ErrorCodes::SYSTEM_ERROR); + + if (!result) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Group {} is not found in the system", arg_gid); + + gid = entry.gr_gid; + } + + if (gid == 0 && getgid() != 0) + throw Exception("Group has id 0, but dropping privileges to gid 0 does not make sense", ErrorCodes::BAD_ARGUMENTS); + + if (0 != setgid(gid)) + throwFromErrno(fmt::format("Cannot do 'setgid' to user ({})", arg_gid), ErrorCodes::SYSTEM_ERROR); + } + + if (!arg_uid.empty()) + { + /// Is it numeric id or name? + uid_t uid = 0; + if (!tryParse(uid, arg_uid) || uid == 0) + { + passwd entry{}; + passwd * result{}; + + if (0 != getpwnam_r(arg_uid.data(), &entry, buf.get(), buf_size, &result)) + throwFromErrno(fmt::format("Cannot do 'getpwnam_r' to obtain uid from user name ({})", arg_uid), ErrorCodes::SYSTEM_ERROR); + + if (!result) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "User {} is not found in the system", arg_uid); + + uid = entry.pw_uid; + } + + if (uid == 0 && getuid() != 0) + throw Exception("User has id 0, but dropping privileges to uid 0 does not make sense", ErrorCodes::BAD_ARGUMENTS); + + if (0 != setuid(uid)) + throwFromErrno(fmt::format("Cannot do 'setuid' to user ({})", arg_uid), ErrorCodes::SYSTEM_ERROR); + } +} + +} + + +int mainEntryClickHouseSU(int argc, char ** argv) +try +{ + using namespace DB; + + if (argc < 3) + { + std::cout << "Usage: ./clickhouse su user:group ..." << std::endl; + exit(0); + } + + std::string_view user_and_group = argv[1]; + + std::string user; + std::string group; + + auto pos = user_and_group.find(':'); + if (pos == std::string_view::npos) + { + user = user_and_group; + } + else + { + user = user_and_group.substr(0, pos); + group = user_and_group.substr(pos + 1); + } + + setUserAndGroup(std::move(user), std::move(group)); + + std::vector new_argv; + new_argv.reserve(argc - 1); + new_argv.insert(new_argv.begin(), argv + 2, argv + argc); + new_argv.push_back(nullptr); + + execvp(new_argv.front(), new_argv.data()); + + throwFromErrno("Cannot execvp", ErrorCodes::SYSTEM_ERROR); +} +catch (...) +{ + std::cerr << DB::getCurrentExceptionMessage(false) << '\n'; + return 1; +} From 498c8b3c52bd1d393e64a7cbc36cb020a0912e6e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 29 Jul 2022 09:20:44 +0300 Subject: [PATCH 087/139] Fix clang-tidy in utils/examples Signed-off-by: Azat Khuzhin --- src/Access/examples/kerberos_init.cpp | 2 +- src/Common/CompactArray.h | 2 +- src/Core/examples/coro.cpp | 8 +-- src/Interpreters/examples/hash_map3.cpp | 8 +-- .../get_current_inserts_in_replicated.cpp | 2 +- utils/compressor/decompress_perf.cpp | 8 +-- utils/db-generator/query_db_generator.cpp | 10 ++-- .../graphite-rollup/graphite-rollup-bench.cpp | 59 +++++++------------ utils/iotest/iotest.cpp | 5 +- utils/iotest/iotest_aio.cpp | 6 +- utils/iotest/iotest_nonblock.cpp | 10 ++-- .../main.cpp | 4 +- utils/zookeeper-test/main.cpp | 4 +- 13 files changed, 53 insertions(+), 75 deletions(-) diff --git a/src/Access/examples/kerberos_init.cpp b/src/Access/examples/kerberos_init.cpp index 5dbe92a5b57..f9f9d882ece 100644 --- a/src/Access/examples/kerberos_init.cpp +++ b/src/Access/examples/kerberos_init.cpp @@ -25,7 +25,7 @@ int main(int argc, char ** argv) return 0; } - String cache_name = ""; + const char * cache_name = ""; if (argc == 4) cache_name = argv[3]; diff --git a/src/Common/CompactArray.h b/src/Common/CompactArray.h index cf97206edb8..c2dd696d651 100644 --- a/src/Common/CompactArray.h +++ b/src/Common/CompactArray.h @@ -130,7 +130,7 @@ public: private: ReadBuffer & in; /// The physical location of the current cell. - Locus locus; + Locus locus{}; /// The current position in the file as a cell number. BucketIndex current_bucket_index = 0; /// The number of bytes read. diff --git a/src/Core/examples/coro.cpp b/src/Core/examples/coro.cpp index 956b4e988e4..370820a228d 100644 --- a/src/Core/examples/coro.cpp +++ b/src/Core/examples/coro.cpp @@ -12,7 +12,7 @@ #if defined(__clang__) #include -namespace std +namespace std // NOLINT(cert-dcl58-cpp) { using namespace experimental::coroutines_v1; } @@ -97,14 +97,14 @@ struct Task static bool resumeImpl(Task *r) { if (r->value) - return false; + return false; auto & next = r->my.promise().next; if (next) { if (resumeImpl(next.promise().r)) - return true; + return true; next = {}; } @@ -183,7 +183,7 @@ int main() auto t = foo("foo"); std::cout << ".. started" << std::endl; while (t.resume()) - std::cout << ".. yielded" << std::endl; + std::cout << ".. yielded" << std::endl; std::cout << ".. done: " << t.res() << std::endl; } catch (DB::Exception & e) diff --git a/src/Interpreters/examples/hash_map3.cpp b/src/Interpreters/examples/hash_map3.cpp index a8673e06e9b..110dcab3f76 100644 --- a/src/Interpreters/examples/hash_map3.cpp +++ b/src/Interpreters/examples/hash_map3.cpp @@ -1,13 +1,9 @@ -#include - #define DBMS_HASH_MAP_DEBUG_RESIZES #define DBMS_HASH_MAP_COUNT_COLLISIONS - -#include - +#include +#include #include - #include #include diff --git a/src/Storages/examples/get_current_inserts_in_replicated.cpp b/src/Storages/examples/get_current_inserts_in_replicated.cpp index d7dedbcab9c..0d04cbd260e 100644 --- a/src/Storages/examples/get_current_inserts_in_replicated.cpp +++ b/src/Storages/examples/get_current_inserts_in_replicated.cpp @@ -85,7 +85,7 @@ try for (BlockInfo & block : block_infos) { Coordination::GetResponse resp = block.contents_future.get(); - if (resp.error == Coordination::Error::ZOK && lock_holder_paths.count(resp.data)) + if (resp.error == Coordination::Error::ZOK && lock_holder_paths.contains(resp.data)) { ++total_count; current_inserts[block.partition].insert(block.number); diff --git a/utils/compressor/decompress_perf.cpp b/utils/compressor/decompress_perf.cpp index 20f974aab7f..e3210164d79 100644 --- a/utils/compressor/decompress_perf.cpp +++ b/utils/compressor/decompress_perf.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include @@ -60,7 +60,7 @@ protected: compressed_in->readStrict(reinterpret_cast(&checksum), sizeof(checksum)); own_compressed_buffer.resize(COMPRESSED_BLOCK_HEADER_SIZE); - compressed_in->readStrict(&own_compressed_buffer[0], COMPRESSED_BLOCK_HEADER_SIZE); + compressed_in->readStrict(own_compressed_buffer.data(), COMPRESSED_BLOCK_HEADER_SIZE); UInt8 method = own_compressed_buffer[0]; /// See CompressedWriteBuffer.h @@ -90,7 +90,7 @@ protected: else { own_compressed_buffer.resize(size_compressed + (variant == LZ4_REFERENCE ? 0 : LZ4::ADDITIONAL_BYTES_AT_END_OF_BUFFER)); - compressed_buffer = &own_compressed_buffer[0]; + compressed_buffer = own_compressed_buffer.data(); compressed_in->readStrict(compressed_buffer + COMPRESSED_BLOCK_HEADER_SIZE, size_compressed - COMPRESSED_BLOCK_HEADER_SIZE); } @@ -143,7 +143,7 @@ private: return false; memory.resize(size_decompressed + LZ4::ADDITIONAL_BYTES_AT_END_OF_BUFFER); - working_buffer = Buffer(&memory[0], &memory[size_decompressed]); + working_buffer = Buffer(memory.data(), &memory[size_decompressed]); decompress(working_buffer.begin(), size_decompressed, size_compressed_without_checksum); diff --git a/utils/db-generator/query_db_generator.cpp b/utils/db-generator/query_db_generator.cpp index 18c9bdbca38..00785af89f7 100644 --- a/utils/db-generator/query_db_generator.cpp +++ b/utils/db-generator/query_db_generator.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include @@ -59,8 +59,8 @@ std::string randomDate() int32_t month = rng() % 12 + 1; int32_t day = rng() % 12 + 1; char answer[13]; - sprintf(answer, "'%04u-%02u-%02u'", year, month, day); - return std::string(answer); + size_t size = sprintf(answer, "'%04u-%02u-%02u'", year, month, day); + return std::string(answer, size); } std::string randomDatetime() @@ -72,7 +72,7 @@ std::string randomDatetime() int32_t minutes = rng() % 60; int32_t seconds = rng() % 60; char answer[22]; - sprintf( + size_t size = sprintf( answer, "'%04u-%02u-%02u %02u:%02u:%02u'", year, @@ -81,7 +81,7 @@ std::string randomDatetime() hours, minutes, seconds); - return std::string(answer); + return std::string(answer, size); } TableAndColumn get_table_a_column(const std::string & c) { diff --git a/utils/graphite-rollup/graphite-rollup-bench.cpp b/utils/graphite-rollup/graphite-rollup-bench.cpp index 49a3d509be6..a46d214edbf 100644 --- a/utils/graphite-rollup/graphite-rollup-bench.cpp +++ b/utils/graphite-rollup/graphite-rollup-bench.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include @@ -15,50 +14,35 @@ #include #include #include +#include +#include using namespace DB; static SharedContextHolder shared_context = Context::createShared(); -std::vector loadMetrics(const std::string & metrics_file) +auto loadMetrics(const std::string & metrics_file) { - std::vector metrics; + std::vector metrics; + ReadBufferFromFile in(metrics_file); + String line; - FILE * stream; - char * line = nullptr; - size_t len = 0; - ssize_t nread; - - stream = fopen(metrics_file.c_str(), "r"); - if (stream == nullptr) + while (!in.eof()) { - throw std::runtime_error(strerror(errno)); - } - - while ((nread = getline(&line, &len, stream)) != -1) /// NOLINT - { - size_t l = strlen(line); - if (l > 0) + readEscapedStringUntilEOL(line, in); + if (!in.eof()) { - if (line[l - 1] == '\n') - { - line[l - 1] = '\0'; - l--; - } - if (l > 0) - { - metrics.emplace_back(std::string_view(strdup(line), l)); - } + ++in.position(); + } + if (!line.empty() && line.back() == '\n') + { + line.pop_back(); + } + if (!line.empty()) + { + metrics.emplace_back(line); } } - free(line); - if (ferror(stream)) - { - fclose(stream); - throw std::runtime_error(strerror(errno)); - } - - fclose(stream); return metrics; } @@ -80,7 +64,7 @@ void bench(const std::string & config_path, const std::string & metrics_file, si Graphite::Params params; setGraphitePatternsFromConfig(context, "graphite_rollup", params); - std::vector metrics = loadMetrics(metrics_file); + auto metrics = loadMetrics(metrics_file); std::vector durations(metrics.size()); size_t j, i; @@ -99,15 +83,14 @@ void bench(const std::string & config_path, const std::string & metrics_file, si if (j == 0 && verbose) { - std::cout << metrics[i].data() << ": rule with regexp '" << rule.second->regexp_str << "' found\n"; + std::cout << metrics[i] << ": rule with regexp '" << rule.second->regexp_str << "' found\n"; } } } for (i = 0; i < metrics.size(); i++) { - std::cout << metrics[i].data() << " " << durations[i] / n << " ns\n"; - free(const_cast(static_cast(metrics[i].data()))); + std::cout << metrics[i] << " " << durations[i] / n << " ns\n"; } } diff --git a/utils/iotest/iotest.cpp b/utils/iotest/iotest.cpp index f23ee6929fa..7a1f35ddd52 100644 --- a/utils/iotest/iotest.cpp +++ b/utils/iotest/iotest.cpp @@ -15,8 +15,7 @@ #include #include -#include -#include +#include #include @@ -54,7 +53,7 @@ void thread(int fd, int mode, size_t min_offset, size_t max_offset, size_t block if ((mode & MODE_DIRECT)) buf = direct_buf.data(); else - buf = &simple_buf[0]; + buf = simple_buf.data(); pcg64 rng(randomSeed()); diff --git a/utils/iotest/iotest_aio.cpp b/utils/iotest/iotest_aio.cpp index 05074bc85f4..c0cf002ce58 100644 --- a/utils/iotest/iotest_aio.cpp +++ b/utils/iotest/iotest_aio.cpp @@ -110,12 +110,12 @@ void thread(int fd, int mode, size_t min_offset, size_t max_offset, size_t block } /// Send queries. - if (io_submit(ctx.ctx, query_cbs.size(), &query_cbs[0]) < 0) + if (io_submit(ctx.ctx, query_cbs.size(), query_cbs.data()) < 0) throwFromErrno("io_submit failed", ErrorCodes::CANNOT_IO_SUBMIT); /// Receive answers. If we have something else to send, then receive at least one answer (after that send them), otherwise wait all answers. - memset(&events[0], 0, buffers_count * sizeof(events[0])); - int evs = io_getevents(ctx.ctx, (blocks_sent < count ? 1 : in_progress), buffers_count, &events[0], nullptr); + memset(events.data(), 0, buffers_count * sizeof(events[0])); + int evs = io_getevents(ctx.ctx, (blocks_sent < count ? 1 : in_progress), buffers_count, events.data(), nullptr); if (evs < 0) throwFromErrno("io_getevents failed", ErrorCodes::CANNOT_IO_GETEVENTS); diff --git a/utils/iotest/iotest_nonblock.cpp b/utils/iotest/iotest_nonblock.cpp index 430e951d36b..33fab4d04e6 100644 --- a/utils/iotest/iotest_nonblock.cpp +++ b/utils/iotest/iotest_nonblock.cpp @@ -13,8 +13,8 @@ #include #include -#include -#include +#include +#include #include #if defined (OS_LINUX) @@ -101,7 +101,7 @@ int mainImpl(int argc, char ** argv) size_t ops = 0; while (ops < count) { - if (poll(&polls[0], descriptors, -1) <= 0) + if (poll(polls.data(), descriptors, -1) <= 0) throwFromErrno("poll failed", ErrorCodes::SYSTEM_ERROR); for (size_t i = 0; i < descriptors; ++i) { @@ -123,12 +123,12 @@ int mainImpl(int argc, char ** argv) if (mode == MODE_READ) { - if (static_cast(block_size) != pread(fds[i], &buf[0], block_size, offset)) + if (static_cast(block_size) != pread(fds[i], buf.data(), block_size, offset)) throwFromErrno("Cannot read", ErrorCodes::CANNOT_READ_FROM_FILE_DESCRIPTOR); } else { - if (static_cast(block_size) != pwrite(fds[i], &buf[0], block_size, offset)) + if (static_cast(block_size) != pwrite(fds[i], buf.data(), block_size, offset)) throwFromErrno("Cannot write", ErrorCodes::CANNOT_WRITE_TO_FILE_DESCRIPTOR); } } diff --git a/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp b/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp index 5c694ee04ef..7736921a9c6 100644 --- a/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp +++ b/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp @@ -179,8 +179,8 @@ void setCurrentBlockNumber(zkutil::ZooKeeper & zk, const std::string & path, Int if (number != current_block_number) { char suffix[11] = ""; - sprintf(suffix, "%010lld", current_block_number); - std::string expected_path = block_prefix + suffix; + size_t size = sprintf(suffix, "%010lld", current_block_number); + std::string expected_path = block_prefix + std::string(suffix, size); std::cerr << "\t" << path_created << ": Ephemeral node has been created with an unexpected path (expected something like " << expected_path << ")." << std::endl; return false; diff --git a/utils/zookeeper-test/main.cpp b/utils/zookeeper-test/main.cpp index 5d0b54aa74b..2d5f2ba34ab 100644 --- a/utils/zookeeper-test/main.cpp +++ b/utils/zookeeper-test/main.cpp @@ -238,9 +238,9 @@ std::string currentDateTime() tstruct = *localtime(&now); // Visit http://en.cppreference.com/w/cpp/chrono/c/strftime // for more information about date/time format - strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", &tstruct); + size_t size = strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", &tstruct); - return buf; + return std::string(buf, size); } From 672e67aa8e03108bc983d684d52b8f7ae29c85a5 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 29 Jul 2022 11:52:20 +0200 Subject: [PATCH 088/139] Change mysql-odbc url --- docker/test/integration/base/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index eaf0f01e36d..9b6318a5426 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -34,13 +34,14 @@ RUN apt-get update \ ARG TARGETARCH # Install MySQL ODBC driver from RHEL rpm +# For reference https://downloads.mysql.com/archives/c-odbc/ RHEL RUN arch=${TARGETARCH:-amd64} \ && case $arch in \ amd64) rarch=x86_64 ;; \ arm64) rarch=aarch64 ;; \ esac \ && cd /tmp \ - && curl -o mysql-odbc.rpm "https://cdn.mysql.com/Downloads/Connector-ODBC/8.0/mysql-connector-odbc-8.0.27-1.el8.${rarch}.rpm" \ + && curl -o mysql-odbc.rpm "https://cdn.mysql.com/archives/mysql-connector-odbc-8.0/mysql-connector-odbc-8.0.27-1.el8.${rarch}.rpm" \ && rpm2archive mysql-odbc.rpm \ && tar xf mysql-odbc.rpm.tgz -C / ./usr/lib64/ \ && LINK_DIR=$(dpkg -L libodbc1 | grep '^/usr/lib/.*-linux-gnu/odbc$') \ From 11d5f1ea9a2ec52cfd6c0c3cbe8ad22021bdc993 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Fri, 29 Jul 2022 10:06:36 +0000 Subject: [PATCH 089/139] More tests --- ...7_distinct_in_order_optimization.reference | 24 +++++++++++++++---- .../02317_distinct_in_order_optimization.sql | 12 ++++++---- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference b/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference index 79927cf8884..efc9e28bcce 100644 --- a/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference +++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference @@ -79,12 +79,28 @@ 0 0 -- distinct with constants columns -- { echoOn } -select distinct 1 as a, 2 as b from distinct_in_order; +select distinct 1 as x, 2 as y from distinct_in_order; 1 2 -select distinct 1 as a, 2 as b from distinct_in_order order by a; +select distinct 1 as x, 2 as y from distinct_in_order order by x; 1 2 -select distinct 1 as a, 2 as b from distinct_in_order order by a, b; +select distinct 1 as x, 2 as y from distinct_in_order order by x, y; 1 2 -select distinct x, y from (select 1 as x, 2 as y from distinct_in_order order by x) order by x; +select distinct a, 1 as x from distinct_in_order order by x; +0 1 +select distinct a, 1 as x, 2 as y from distinct_in_order order by a; +0 1 2 +select distinct a, b, 1 as x, 2 as y from distinct_in_order order by a; +0 0 1 2 +0 1 1 2 +0 2 1 2 +0 3 1 2 +0 4 1 2 +select distinct x, y from (select 1 as x, 2 as y from distinct_in_order order by x) order by y; 1 2 +select distinct a, b, x, y from (select a, b, 1 as x, 2 as y from distinct_in_order order by a) order by b; +0 0 1 2 +0 1 1 2 +0 2 1 2 +0 3 1 2 +0 4 1 2 -- check that distinct in order has the same result as ordinary distinct diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql b/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql index 945c9625e99..7a70e2ef873 100644 --- a/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql +++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql @@ -45,10 +45,14 @@ select distinct b,c from distinct_in_order order by c desc; select '-- distinct with constants columns'; -- { echoOn } -select distinct 1 as a, 2 as b from distinct_in_order; -select distinct 1 as a, 2 as b from distinct_in_order order by a; -select distinct 1 as a, 2 as b from distinct_in_order order by a, b; -select distinct x, y from (select 1 as x, 2 as y from distinct_in_order order by x) order by x; +select distinct 1 as x, 2 as y from distinct_in_order; +select distinct 1 as x, 2 as y from distinct_in_order order by x; +select distinct 1 as x, 2 as y from distinct_in_order order by x, y; +select distinct a, 1 as x from distinct_in_order order by x; +select distinct a, 1 as x, 2 as y from distinct_in_order order by a; +select distinct a, b, 1 as x, 2 as y from distinct_in_order order by a; +select distinct x, y from (select 1 as x, 2 as y from distinct_in_order order by x) order by y; +select distinct a, b, x, y from (select a, b, 1 as x, 2 as y from distinct_in_order order by a) order by b; -- { echoOff } drop table if exists distinct_in_order sync; From b390bcfe7cbec203d44ab371d5357b562ed7be61 Mon Sep 17 00:00:00 2001 From: zvonand Date: Fri, 29 Jul 2022 13:25:40 +0300 Subject: [PATCH 090/139] fix wrong data type cast --- src/Functions/timeSlots.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Functions/timeSlots.cpp b/src/Functions/timeSlots.cpp index 12aeddcf621..382d7131bf3 100644 --- a/src/Functions/timeSlots.cpp +++ b/src/Functions/timeSlots.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -258,7 +259,7 @@ public: else { auto start_time_scale = assert_cast(*arguments[0].type).getScale(); - auto duration_scale = assert_cast &>(*arguments[1].type).getScale(); + auto duration_scale = assert_cast(*arguments[1].type).getScale(); return std::make_shared( std::make_shared(std::max(start_time_scale, duration_scale), extractTimeZoneNameFromFunctionArguments(arguments, 3, 0))); } @@ -318,7 +319,7 @@ public: if (time_slot_size = time_slot_column->getValue(); time_slot_size <= 0) throw Exception("Third argument for function " + getName() + " must be greater than zero", ErrorCodes::ILLEGAL_COLUMN); - time_slot_scale = assert_cast *>(arguments[2].type.get())->getScale(); + time_slot_scale = assert_cast(arguments[2].type.get())->getScale(); } const auto * starts = checkAndGetColumn(arguments[0].column.get()); @@ -328,7 +329,7 @@ public: const auto * const_durations = checkAndGetColumnConst>(arguments[1].column.get()); const auto start_time_scale = assert_cast(arguments[0].type.get())->getScale(); - const auto duration_scale = assert_cast *>(arguments[1].type.get())->getScale(); + const auto duration_scale = assert_cast(arguments[1].type.get())->getScale(); auto res = ColumnArray::create(DataTypeDateTime64(start_time_scale).createColumn()); DataTypeDateTime64::ColumnType::Container & res_values = typeid_cast(res->getData()).getData(); From 2f3b27d1ac94754407c17d5e83cbcec95460a9dd Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 28 Jul 2022 22:43:51 +0200 Subject: [PATCH 091/139] Fix seeking while reading from encrypted disk. --- src/IO/FileEncryptionCommon.h | 1 + src/IO/ReadBufferFromEncryptedFile.cpp | 8 ++++---- src/IO/tests/gtest_file_encryption.cpp | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/IO/FileEncryptionCommon.h b/src/IO/FileEncryptionCommon.h index bb6c8d14893..496c9e66b20 100644 --- a/src/IO/FileEncryptionCommon.h +++ b/src/IO/FileEncryptionCommon.h @@ -80,6 +80,7 @@ public: /// the initialization vector is increased by an index of the current block /// and the index of the current block is calculated from this offset. void setOffset(size_t offset_) { offset = offset_; } + size_t getOffset() const { return offset; } /// Encrypts some data. /// Also the function moves `offset` by `size` (for successive encryptions). diff --git a/src/IO/ReadBufferFromEncryptedFile.cpp b/src/IO/ReadBufferFromEncryptedFile.cpp index 16517422e26..22f994bb44b 100644 --- a/src/IO/ReadBufferFromEncryptedFile.cpp +++ b/src/IO/ReadBufferFromEncryptedFile.cpp @@ -21,7 +21,6 @@ ReadBufferFromEncryptedFile::ReadBufferFromEncryptedFile( , encryptor(header_.algorithm, key_, header_.init_vector) { offset = offset_; - encryptor.setOffset(offset_); need_seek = true; } @@ -60,9 +59,6 @@ off_t ReadBufferFromEncryptedFile::seek(off_t off, int whence) assert(!hasPendingData()); } - /// The encryptor always needs to know what the current offset is. - encryptor.setOffset(new_pos); - return new_pos; } @@ -94,6 +90,10 @@ bool ReadBufferFromEncryptedFile::nextImpl() /// The used cipher algorithms generate the same number of bytes in output as it were in input, /// so after deciphering the numbers of bytes will be still `bytes_read`. working_buffer.resize(bytes_read); + + /// The decryptor needs to know what the current offset is (because it's used in the decryption algorithm). + encryptor.setOffset(offset); + encryptor.decrypt(encrypted_buffer.data(), bytes_read, working_buffer.begin()); offset += bytes_read; diff --git a/src/IO/tests/gtest_file_encryption.cpp b/src/IO/tests/gtest_file_encryption.cpp index f53c85c422a..cae40afbb38 100644 --- a/src/IO/tests/gtest_file_encryption.cpp +++ b/src/IO/tests/gtest_file_encryption.cpp @@ -242,7 +242,7 @@ TEST(FileEncryptionPositionUpdateTest, Decryption) rb.ignore(5); rb.ignore(5); ASSERT_EQ(rb.getPosition(), 15); - + String res; readStringUntilEOF(res, rb); ASSERT_EQ(res, data.substr(15)); From ac90aa435ea98049a4038ead6bfbb585d124ed5a Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 29 Jul 2022 11:00:33 +0000 Subject: [PATCH 092/139] Avoid recursive destruction of AST. --- src/Parsers/IAST.cpp | 33 +++++ src/Parsers/IAST.h | 6 +- src/Parsers/tests/gtest_ast_deleter.cpp | 152 ++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 src/Parsers/tests/gtest_ast_deleter.cpp diff --git a/src/Parsers/IAST.cpp b/src/Parsers/IAST.cpp index 3a21d704eb9..72fdbd924f2 100644 --- a/src/Parsers/IAST.cpp +++ b/src/Parsers/IAST.cpp @@ -26,6 +26,39 @@ const char * IAST::hilite_substitution = "\033[1;36m"; const char * IAST::hilite_none = "\033[0m"; +IAST::~IAST() +{ + /// If deleter was set, move our children to it. + /// Will avoid recursive destruction. + if (deleter) + { + deleter->push_back(std::move(children)); + return; + } + + std::list queue; + queue.push_back(std::move(children)); + while (!queue.empty()) + { + for (auto & node : queue.front()) + { + /// If two threads remove ASTPtr concurrently, + /// it is possible that neither thead will see use_count == 1. + /// It is ok. Will need one more extra stack frame in this case. + if (node.use_count() == 1) + { + /// Deleter is only set when current thread is the single owner. + /// No extra synchronisation is needed. + ASTPtr to_delete; + node.swap(to_delete); + to_delete->deleter = &queue; + } + } + + queue.pop_front(); + } +} + size_t IAST::size() const { size_t res = 1; diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h index 1999eff37a8..5714a829693 100644 --- a/src/Parsers/IAST.h +++ b/src/Parsers/IAST.h @@ -9,6 +9,7 @@ #include #include +#include class SipHash; @@ -34,7 +35,7 @@ class IAST : public std::enable_shared_from_this, public TypePromotion * deleter = nullptr; }; template diff --git a/src/Parsers/tests/gtest_ast_deleter.cpp b/src/Parsers/tests/gtest_ast_deleter.cpp new file mode 100644 index 00000000000..2a94d6a3a20 --- /dev/null +++ b/src/Parsers/tests/gtest_ast_deleter.cpp @@ -0,0 +1,152 @@ +#include +#include +#include + +namespace DB::ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +struct StackDecrementer +{ + size_t & depth; + + explicit StackDecrementer(size_t & depth_) : depth(depth_) {} + + virtual ~StackDecrementer() + { + --depth; + } +}; + +struct ASTCounting : public StackDecrementer, public DB::IAST +{ + explicit ASTCounting(size_t & depth_) : StackDecrementer(depth_) {} + + String getID(char) const override { return ""; } + DB::ASTPtr clone() const override { return nullptr; } +}; + +class ASTWithMembers : public ASTCounting +{ +public: + explicit ASTWithMembers(DB::ASTs members_, size_t & depth_) : ASTCounting(depth_), members(std::move(members_)) + { + children = members; + } + + DB::ASTs members; +}; + +class ASTWithDecrementer : public ASTWithMembers +{ +public: + ASTWithDecrementer(DB::ASTs members_, size_t & depth_, size_t max_depth_) + : ASTWithMembers(std::move(members_), depth_), max_depth(max_depth_) + { + } + + size_t max_depth; + + ~ASTWithDecrementer() override + { + ++depth; + if (depth == max_depth) + std::cout << StackTrace().toString() << std::endl; + if (depth > max_depth) + EXPECT_LE(depth, max_depth); + } +}; + + +TEST(ASTDeleter, SimpleChain) +{ + size_t depth = 0; + size_t max_depth = 5; + size_t chain_lenght = 10; + + { + DB::ASTPtr ast = std::make_shared(DB::ASTs{}, depth, max_depth); + + for (size_t i = 0; i < chain_lenght; ++i) + ast = std::make_shared(DB::ASTs{std::move(ast)}, depth, max_depth); + } +} + +TEST(ASTDeleter, SimpleChainLong) +{ + size_t depth = 0; + size_t max_depth = 5; + size_t chain_lenght = 100000; + + { + DB::ASTPtr ast = std::make_shared(DB::ASTs{}, depth, max_depth); + + for (size_t i = 0; i < chain_lenght; ++i) + ast = std::make_shared(DB::ASTs{std::move(ast)}, depth, max_depth); + } +} + +TEST(ASTDeleter, ChainWithExtraMember) +{ + size_t depth = 0; + size_t max_depth = 5; + size_t member_depth = 10; + size_t chain_lenght = 100; + + { + DB::ASTPtr ast = std::make_shared(DB::ASTs{}, depth, max_depth); + + for (size_t i = 0; i < chain_lenght; ++i) + { + ast = std::make_shared(DB::ASTs{std::move(ast)}, depth, max_depth); + if (i > member_depth) + { + DB::ASTPtr member = ast; + for (size_t j = 0; j < member_depth; ++j) + member = member->children.front(); + ast->as()->members.push_back(std::move(member)); + } + } + } +} + +TEST(ASTDeleter, DoubleChain) +{ + size_t depth = 0; + size_t max_depth = 5; + size_t chain_lenght = 10; + + { + DB::ASTPtr ast1 = std::make_shared(DB::ASTs{}, depth, max_depth); + DB::ASTPtr ast2 = std::make_shared(DB::ASTs{}, depth, max_depth); + + for (size_t i = 0; i < chain_lenght; ++i) + { + ast1 = std::make_shared(DB::ASTs{std::move(ast1)}, depth, max_depth); + ast2 = std::make_shared(DB::ASTs{std::move(ast2)}, depth, max_depth); + ast1->as()->members.push_back(ast2->children.front()); + ast2->as()->members.push_back(ast1->children.front()); + } + } +} + +TEST(ASTDeleter, DoubleChainLong) +{ + size_t depth = 0; + size_t max_depth = 5; + size_t chain_lenght = 100000; + + { + DB::ASTPtr ast1 = std::make_shared(DB::ASTs{}, depth, max_depth); + DB::ASTPtr ast2 = std::make_shared(DB::ASTs{}, depth, max_depth); + + for (size_t i = 0; i < chain_lenght; ++i) + { + ast1 = std::make_shared(DB::ASTs{std::move(ast1)}, depth, max_depth); + ast2 = std::make_shared(DB::ASTs{std::move(ast2)}, depth, max_depth); + ast1->as()->members.push_back(ast2->children.front()); + ast2->as()->members.push_back(ast1->children.front()); + } + } +} From 0cef018c29937702fe3a0cbc0fc41793cf2b7688 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 29 Jul 2022 14:26:41 +0200 Subject: [PATCH 093/139] clear mmapio cache when dropping table from Ordinary --- src/Interpreters/InterpreterDropQuery.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index ac731ec6f4b..bb3f91d5210 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -245,6 +245,11 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue DatabaseCatalog::instance().tryRemoveLoadingDependencies(table_id, getContext()->getSettingsRef().check_table_dependencies, is_drop_or_detach_database); database->dropTable(context_, table_id.table_name, query.sync); + + /// We have to drop mmapio cache when dropping table from Ordinary database + /// to avoid reading old data if new table with the same name is created + if (database->getUUID() == UUIDHelpers::Nil) + context_->dropMMappedFileCache(); } db = database; From b08bdfcfc20033f463f511f96e274b776c7bf5a2 Mon Sep 17 00:00:00 2001 From: Rich Raposa Date: Fri, 29 Jul 2022 07:25:29 -0600 Subject: [PATCH 094/139] Update gui.md Typo in clickcat description --- docs/en/interfaces/third-party/gui.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/interfaces/third-party/gui.md b/docs/en/interfaces/third-party/gui.md index e23d68cbc2e..aefd763b21e 100644 --- a/docs/en/interfaces/third-party/gui.md +++ b/docs/en/interfaces/third-party/gui.md @@ -149,7 +149,7 @@ Features: ### ClickCat {#clickcat} -[ClickCat](https://github.com/clickcat-project/ClickCat) is a firendly user interface that lets you search, explore and visualize your ClickHouse Data. +[ClickCat](https://github.com/clickcat-project/ClickCat) is a friendly user interface that lets you search, explore and visualize your ClickHouse Data. Features: From 8fc6bad4f40e87567fc42655a8390a173c26dfaa Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 29 Jul 2022 18:30:50 +0200 Subject: [PATCH 095/139] Join enums refactoring --- src/Core/Joins.cpp | 55 +++++++++ src/Core/Joins.h | 95 +++++++++++++++ src/Core/Settings.h | 2 +- src/Core/SettingsEnums.cpp | 4 +- src/Core/SettingsEnums.h | 11 +- src/Interpreters/CollectJoinOnKeysVisitor.cpp | 11 +- src/Interpreters/CollectJoinOnKeysVisitor.h | 2 +- src/Interpreters/ConcurrentHashJoin.cpp | 4 +- src/Interpreters/CrossToInnerJoinVisitor.cpp | 18 +-- src/Interpreters/DirectJoin.cpp | 6 +- src/Interpreters/ExpressionAnalyzer.cpp | 6 +- src/Interpreters/GlobalSubqueriesVisitor.h | 2 +- src/Interpreters/HashJoin.cpp | 113 +++++++++--------- src/Interpreters/HashJoin.h | 20 ++-- .../InJoinSubqueriesPreprocessor.cpp | 4 +- .../JoinToSubqueryTransformVisitor.cpp | 2 +- src/Interpreters/JoinedTables.cpp | 2 +- src/Interpreters/MergeJoin.cpp | 14 +-- src/Interpreters/RowRefs.cpp | 25 ++-- src/Interpreters/RowRefs.h | 4 +- src/Interpreters/TableJoin.cpp | 34 +++--- src/Interpreters/TableJoin.h | 14 +-- src/Interpreters/TreeRewriter.cpp | 28 ++--- src/Interpreters/asof.h | 46 ------- src/Interpreters/joinDispatch.h | 85 +++++++------ src/Parsers/ASTTablesInSelectQuery.cpp | 36 +++--- src/Parsers/ASTTablesInSelectQuery.h | 51 +------- src/Parsers/ParserTablesInSelectQuery.cpp | 54 ++++----- .../Optimizations/filterPushDown.cpp | 10 +- .../Transforms/MergeJoinTransform.cpp | 8 +- .../Transforms/MergeJoinTransform.h | 4 +- src/Storages/StorageJoin.cpp | 60 +++++----- src/Storages/StorageJoin.h | 8 +- 33 files changed, 446 insertions(+), 392 deletions(-) create mode 100644 src/Core/Joins.cpp create mode 100644 src/Core/Joins.h delete mode 100644 src/Interpreters/asof.h diff --git a/src/Core/Joins.cpp b/src/Core/Joins.cpp new file mode 100644 index 00000000000..8c395d4d5cc --- /dev/null +++ b/src/Core/Joins.cpp @@ -0,0 +1,55 @@ +#include + +namespace DB +{ + +const char * toString(JoinKind kind) +{ + switch (kind) + { + case JoinKind::Inner: return "INNER"; + case JoinKind::Left: return "LEFT"; + case JoinKind::Right: return "RIGHT"; + case JoinKind::Full: return "FULL"; + case JoinKind::Cross: return "CROSS"; + case JoinKind::Comma: return "COMMA"; + } +}; + +const char * toString(JoinStrictness strictness) +{ + switch (strictness) + { + case JoinStrictness::Unspecified: return "UNSPECIFIED"; + case JoinStrictness::RightAny: return "RIGHT_ANY"; + case JoinStrictness::Any: return "ANY"; + case JoinStrictness::All: return "ALL"; + case JoinStrictness::Asof: return "ASOF"; + case JoinStrictness::Semi: return "SEMI"; + case JoinStrictness::Anti: return "ANTI"; + } +} + +const char * toString(JoinLocality locality) +{ + switch (locality) + { + case JoinLocality::Unspecified: return "UNSPECIFIED"; + case JoinLocality::Local: return "LOCAL"; + case JoinLocality::Global: return "GLOBAL"; + } +} + +const char * toString(ASOFJoinInequality asof_join_inequality) +{ + switch (asof_join_inequality) + { + case ASOFJoinInequality::None: return "NONE"; + case ASOFJoinInequality::Less: return "LESS"; + case ASOFJoinInequality::Greater: return "GREATER"; + case ASOFJoinInequality::LessOrEquals: return "LESS_OR_EQUALS"; + case ASOFJoinInequality::GreaterOrEquals: return "GREATER_OR_EQUALS"; + } +} + +} diff --git a/src/Core/Joins.h b/src/Core/Joins.h new file mode 100644 index 00000000000..41529e9888f --- /dev/null +++ b/src/Core/Joins.h @@ -0,0 +1,95 @@ +#pragma once + +namespace DB +{ + +/// Join method. +enum class JoinKind +{ + Inner, /// Leave only rows that was JOINed. + Left, /// If in "right" table there is no corresponding rows, use default values instead. + Right, + Full, + Cross, /// Direct product. Strictness and condition doesn't matter. + Comma /// Same as direct product. Intended to be converted to INNER JOIN with conditions from WHERE. +}; + +const char * toString(JoinKind kind); + +inline constexpr bool isLeft(JoinKind kind) { return kind == JoinKind::Left; } +inline constexpr bool isRight(JoinKind kind) { return kind == JoinKind::Right; } +inline constexpr bool isInner(JoinKind kind) { return kind == JoinKind::Inner; } +inline constexpr bool isFull(JoinKind kind) { return kind == JoinKind::Full; } +inline constexpr bool isCrossOrComma(JoinKind kind) { return kind == JoinKind::Comma || kind == JoinKind::Cross; } +inline constexpr bool isRightOrFull(JoinKind kind) { return kind == JoinKind::Right || kind == JoinKind::Full; } +inline constexpr bool isLeftOrFull(JoinKind kind) { return kind == JoinKind::Left || kind == JoinKind::Full; } +inline constexpr bool isInnerOrRight(JoinKind kind) { return kind == JoinKind::Inner || kind == JoinKind::Right; } +inline constexpr bool isInnerOrLeft(JoinKind kind) { return kind == JoinKind::Inner || kind == JoinKind::Left; } + +/// Allows more optimal JOIN for typical cases. +enum class JoinStrictness +{ + Unspecified, + RightAny, /// Old ANY JOIN. If there are many suitable rows in right table, use any from them to join. + Any, /// Semi Join with any value from filtering table. For LEFT JOIN with Any and RightAny are the same. + All, /// If there are many suitable rows to join, use all of them and replicate rows of "left" table (usual semantic of JOIN). + Asof, /// For the last JOIN column, pick the latest value + Semi, /// LEFT or RIGHT. SEMI LEFT JOIN filters left table by values exists in right table. SEMI RIGHT - otherwise. + Anti, /// LEFT or RIGHT. Same as SEMI JOIN but filter values that are NOT exists in other table. +}; + +const char * toString(JoinStrictness strictness); + +/// Algorithm for distributed query processing. +enum class JoinLocality +{ + Unspecified, + Local, /// Perform JOIN, using only data available on same servers (co-located data). + Global /// Collect and merge data from remote servers, and broadcast it to each server. +}; + +const char * toString(JoinLocality locality); + +/// ASOF JOIN inequality type +enum class ASOFJoinInequality +{ + None, + Less, + Greater, + LessOrEquals, + GreaterOrEquals, +}; + +const char * toString(ASOFJoinInequality asof_join_inequality); + +inline constexpr ASOFJoinInequality getASOFJoinInequality(std::string_view func_name) +{ + ASOFJoinInequality inequality = ASOFJoinInequality::None; + + if (func_name == "less") + inequality = ASOFJoinInequality::Less; + else if (func_name == "greater") + inequality = ASOFJoinInequality::Greater; + else if (func_name == "lessOrEquals") + inequality = ASOFJoinInequality::LessOrEquals; + else if (func_name == "greaterOrEquals") + inequality = ASOFJoinInequality::GreaterOrEquals; + + return inequality; +} + +inline constexpr ASOFJoinInequality reverseASOFJoinInequality(ASOFJoinInequality inequality) +{ + if (inequality == ASOFJoinInequality::Less) + return ASOFJoinInequality::Greater; + else if (inequality == ASOFJoinInequality::Greater) + return ASOFJoinInequality::Less; + else if (inequality == ASOFJoinInequality::LessOrEquals) + return ASOFJoinInequality::GreaterOrEquals; + else if (inequality == ASOFJoinInequality::GreaterOrEquals) + return ASOFJoinInequality::LessOrEquals; + + return ASOFJoinInequality::None; +} + +} diff --git a/src/Core/Settings.h b/src/Core/Settings.h index b370a749746..51197022908 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -247,7 +247,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) \ M(Bool, join_use_nulls, false, "Use NULLs for non-joined rows of outer JOINs for types that can be inside Nullable. If false, use default value of corresponding columns data type.", IMPORTANT) \ \ - M(JoinStrictness, join_default_strictness, JoinStrictness::ALL, "Set default strictness in JOIN query. Possible values: empty string, 'ANY', 'ALL'. If empty, query without strictness will throw exception.", 0) \ + M(JoinStrictness, join_default_strictness, JoinStrictness::All, "Set default strictness in JOIN query. Possible values: empty string, 'ANY', 'ALL'. If empty, query without strictness will throw exception.", 0) \ M(Bool, any_join_distinct_right_table_keys, false, "Enable old ANY JOIN logic with many-to-one left-to-right table keys mapping for all ANY JOINs. It leads to confusing not equal results for 't1 ANY LEFT JOIN t2' and 't2 ANY RIGHT JOIN t1'. ANY RIGHT JOIN needs one-to-many keys mapping to be consistent with LEFT one.", IMPORTANT) \ \ M(UInt64, preferred_block_size_bytes, 1000000, "", 0) \ diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index b832096c86c..616026520db 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -26,8 +26,8 @@ IMPLEMENT_SETTING_ENUM(LoadBalancing, ErrorCodes::UNKNOWN_LOAD_BALANCING, IMPLEMENT_SETTING_ENUM(JoinStrictness, ErrorCodes::UNKNOWN_JOIN, {{"", JoinStrictness::Unspecified}, - {"ALL", JoinStrictness::ALL}, - {"ANY", JoinStrictness::ANY}}) + {"ALL", JoinStrictness::All}, + {"ANY", JoinStrictness::Any}}) IMPLEMENT_SETTING_MULTI_ENUM(JoinAlgorithm, ErrorCodes::UNKNOWN_JOIN, diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index b8a2bdb48b0..18b5323dea5 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -1,12 +1,14 @@ #pragma once #include +#include #include #include namespace DB { + enum class LoadBalancing { /// among replicas with a minimum number of errors selected randomly @@ -26,14 +28,6 @@ enum class LoadBalancing DECLARE_SETTING_ENUM(LoadBalancing) - -enum class JoinStrictness -{ - Unspecified = 0, /// Query JOIN without strictness will throw Exception. - ALL, /// Query JOIN without strictness -> ALL JOIN ... - ANY, /// Query JOIN without strictness -> ANY JOIN ... -}; - DECLARE_SETTING_ENUM(JoinStrictness) enum class JoinAlgorithm @@ -200,4 +194,5 @@ DECLARE_SETTING_ENUM_WITH_RENAME(EnumComparingMode, FormatSettings::EnumComparin DECLARE_SETTING_ENUM_WITH_RENAME(EscapingRule, FormatSettings::EscapingRule) DECLARE_SETTING_ENUM_WITH_RENAME(MsgPackUUIDRepresentation, FormatSettings::MsgPackUUIDRepresentation) + } diff --git a/src/Interpreters/CollectJoinOnKeysVisitor.cpp b/src/Interpreters/CollectJoinOnKeysVisitor.cpp index 1c87138e1e3..15ecb822976 100644 --- a/src/Interpreters/CollectJoinOnKeysVisitor.cpp +++ b/src/Interpreters/CollectJoinOnKeysVisitor.cpp @@ -48,7 +48,7 @@ void CollectJoinOnKeysMatcher::Data::addJoinKeys(const ASTPtr & left_ast, const } void CollectJoinOnKeysMatcher::Data::addAsofJoinKeys(const ASTPtr & left_ast, const ASTPtr & right_ast, - JoinIdentifierPosPair table_pos, const ASOF::Inequality & inequality) + JoinIdentifierPosPair table_pos, const ASOFJoinInequality & inequality) { if (isLeftIdentifier(table_pos.first) && isRightIdentifier(table_pos.second)) { @@ -60,7 +60,7 @@ void CollectJoinOnKeysMatcher::Data::addAsofJoinKeys(const ASTPtr & left_ast, co { asof_left_key = right_ast->clone(); asof_right_key = left_ast->clone(); - analyzed_join.setAsofInequality(ASOF::reverseInequality(inequality)); + analyzed_join.setAsofInequality(reverseASOFJoinInequality(inequality)); } else { @@ -91,8 +91,9 @@ void CollectJoinOnKeysMatcher::visit(const ASTFunction & func, const ASTPtr & as if (func.name == "and") return; /// go into children - ASOF::Inequality inequality = ASOF::getInequality(func.name); - if (func.name == "equals" || inequality != ASOF::Inequality::None) + ASOFJoinInequality inequality = getASOFJoinInequality(func.name); + + if (func.name == "equals" || inequality != ASOFJoinInequality::None) { if (func.arguments->children.size() != 2) throw Exception("Function " + func.name + " takes two arguments, got '" + func.formatForErrorMessage() + "' instead", @@ -126,7 +127,7 @@ void CollectJoinOnKeysMatcher::visit(const ASTFunction & func, const ASTPtr & as return; } - if (data.is_asof && inequality != ASOF::Inequality::None) + if (data.is_asof && inequality != ASOFJoinInequality::None) { if (data.asof_left_key || data.asof_right_key) throw Exception("ASOF JOIN expects exactly one inequality in ON section. Unexpected '" + queryToString(ast) + "'", diff --git a/src/Interpreters/CollectJoinOnKeysVisitor.h b/src/Interpreters/CollectJoinOnKeysVisitor.h index 0647f58f79b..e52b0c69591 100644 --- a/src/Interpreters/CollectJoinOnKeysVisitor.h +++ b/src/Interpreters/CollectJoinOnKeysVisitor.h @@ -50,7 +50,7 @@ public: void addJoinKeys(const ASTPtr & left_ast, const ASTPtr & right_ast, JoinIdentifierPosPair table_pos); void addAsofJoinKeys(const ASTPtr & left_ast, const ASTPtr & right_ast, JoinIdentifierPosPair table_pos, - const ASOF::Inequality & asof_inequality); + const ASOFJoinInequality & asof_inequality); void asofToJoinKeys(); }; diff --git a/src/Interpreters/ConcurrentHashJoin.cpp b/src/Interpreters/ConcurrentHashJoin.cpp index 5d6318a8df1..8d91375e661 100644 --- a/src/Interpreters/ConcurrentHashJoin.cpp +++ b/src/Interpreters/ConcurrentHashJoin.cpp @@ -165,8 +165,8 @@ bool ConcurrentHashJoin::alwaysReturnsEmptySet() const std::shared_ptr ConcurrentHashJoin::getNonJoinedBlocks( const Block & /*left_sample_block*/, const Block & /*result_sample_block*/, UInt64 /*max_block_size*/) const { - if (table_join->strictness() == ASTTableJoin::Strictness::Asof || - table_join->strictness() == ASTTableJoin::Strictness::Semi || + if (table_join->strictness() == JoinStrictness::Asof || + table_join->strictness() == JoinStrictness::Semi || !isRightOrFull(table_join->kind())) { return {}; diff --git a/src/Interpreters/CrossToInnerJoinVisitor.cpp b/src/Interpreters/CrossToInnerJoinVisitor.cpp index cfa979f4036..52c457f2373 100644 --- a/src/Interpreters/CrossToInnerJoinVisitor.cpp +++ b/src/Interpreters/CrossToInnerJoinVisitor.cpp @@ -60,19 +60,19 @@ struct JoinedElement void rewriteCommaToCross() { - if (join && join->kind == ASTTableJoin::Kind::Comma) - join->kind = ASTTableJoin::Kind::Cross; + if (join && join->kind == JoinKind::Comma) + join->kind = JoinKind::Cross; } - ASTTableJoin::Kind getOriginalKind() const { return original_kind; } + JoinKind getOriginalKind() const { return original_kind; } bool rewriteCrossToInner(ASTPtr on_expression) { - if (join->kind != ASTTableJoin::Kind::Cross) + if (join->kind != JoinKind::Cross) return false; - join->kind = ASTTableJoin::Kind::Inner; - join->strictness = ASTTableJoin::Strictness::All; + join->kind = JoinKind::Inner; + join->strictness = JoinStrictness::All; join->on_expression = on_expression; join->children.push_back(join->on_expression); @@ -89,7 +89,7 @@ private: const ASTTablesInSelectQueryElement & element; ASTTableJoin * join = nullptr; - ASTTableJoin::Kind original_kind; + JoinKind original_kind; }; bool isAllowedToRewriteCrossJoin(const ASTPtr & node, const Aliases & aliases) @@ -243,7 +243,7 @@ void CrossToInnerJoinMatcher::visit(ASTSelectQuery & select, ASTPtr &, Data & da for (size_t i = 1; i < joined_tables.size(); ++i) { auto & joined = joined_tables[i]; - if (joined.tableJoin()->kind != ASTTableJoin::Kind::Cross) + if (joined.tableJoin()->kind != JoinKind::Cross) continue; String query_before = queryToString(*joined.tableJoin()); @@ -258,7 +258,7 @@ void CrossToInnerJoinMatcher::visit(ASTSelectQuery & select, ASTPtr &, Data & da } } - if (joined.getOriginalKind() == ASTTableJoin::Kind::Comma && + if (joined.getOriginalKind() == JoinKind::Comma && data.cross_to_inner_join_rewrite > 1 && !rewritten) { diff --git a/src/Interpreters/DirectJoin.cpp b/src/Interpreters/DirectJoin.cpp index 21baeb31945..af6bd484753 100644 --- a/src/Interpreters/DirectJoin.cpp +++ b/src/Interpreters/DirectJoin.cpp @@ -71,9 +71,9 @@ DirectKeyValueJoin::DirectKeyValueJoin(std::shared_ptr table_join_, throw DB::Exception(ErrorCodes::UNSUPPORTED_JOIN_KEYS, "Not supported by direct JOIN"); } - if (table_join->strictness() != ASTTableJoin::Strictness::All && - table_join->strictness() != ASTTableJoin::Strictness::Any && - table_join->strictness() != ASTTableJoin::Strictness::RightAny) + if (table_join->strictness() != JoinStrictness::All && + table_join->strictness() != JoinStrictness::Any && + table_join->strictness() != JoinStrictness::RightAny) { throw DB::Exception(ErrorCodes::NOT_IMPLEMENTED, "Not supported by direct JOIN"); } diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 96e4e11f17b..605495b6424 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1201,9 +1201,9 @@ std::shared_ptr tryKeyValueJoin(std::shared_ptr a return nullptr; } - if (analyzed_join->strictness() != ASTTableJoin::Strictness::All && - analyzed_join->strictness() != ASTTableJoin::Strictness::Any && - analyzed_join->strictness() != ASTTableJoin::Strictness::RightAny) + if (analyzed_join->strictness() != JoinStrictness::All && + analyzed_join->strictness() != JoinStrictness::Any && + analyzed_join->strictness() != JoinStrictness::RightAny) { return nullptr; } diff --git a/src/Interpreters/GlobalSubqueriesVisitor.h b/src/Interpreters/GlobalSubqueriesVisitor.h index 7086b1e950d..ee46920331d 100644 --- a/src/Interpreters/GlobalSubqueriesVisitor.h +++ b/src/Interpreters/GlobalSubqueriesVisitor.h @@ -239,7 +239,7 @@ private: static void visit(ASTTablesInSelectQueryElement & table_elem, ASTPtr &, Data & data) { if (table_elem.table_join - && (table_elem.table_join->as().locality == ASTTableJoin::Locality::Global + && (table_elem.table_join->as().locality == JoinLocality::Global || data.getContext()->getSettingsRef().prefer_global_in_and_join)) { data.addExternalStorage(table_elem.table_expression, true); diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index d9825a43c07..95a0008c257 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -73,7 +73,7 @@ namespace JoinStuff } /// for single disjunct - template + template void JoinUsedFlags::reinit(size_t size) { if constexpr (MapGetter::flagged) @@ -85,7 +85,7 @@ namespace JoinStuff } /// for multiple disjuncts - template + template void JoinUsedFlags::reinit(const Block * block_ptr) { if constexpr (MapGetter::flagged) @@ -307,7 +307,7 @@ HashJoin::HashJoin(std::shared_ptr table_join_, const Block & right_s std::get(data->maps[0]).create(Type::DICT); chooseMethod(kind, key_columns, key_sizes.emplace_back()); /// init key_sizes } - else if (strictness == ASTTableJoin::Strictness::Asof) + else if (strictness == JoinStrictness::Asof) { assert(disjuncts_num == 1); @@ -348,7 +348,7 @@ HashJoin::HashJoin(std::shared_ptr table_join_, const Block & right_s LOG_DEBUG(log, "Join type: {}, kind: {}, strictness: {}", data->type, kind, strictness); } -HashJoin::Type HashJoin::chooseMethod(ASTTableJoin::Kind kind, const ColumnRawPtrs & key_columns, Sizes & key_sizes) +HashJoin::Type HashJoin::chooseMethod(JoinKind kind, const ColumnRawPtrs & key_columns, Sizes & key_sizes) { size_t keys_size = key_columns.size(); @@ -531,7 +531,7 @@ void HashJoin::dataMapInit(MapsVariant & map) { if (data->type == Type::DICT) return; - if (kind == ASTTableJoin::Kind::Cross) + if (kind == JoinKind::Cross) return; joinDispatchInit(kind, strictness, map); joinDispatch(kind, strictness, map, [&](auto, auto, auto & map_) { map_.create(data->type); }); @@ -636,13 +636,13 @@ namespace }; - template + template size_t NO_INLINE insertFromBlockImplTypeCase( HashJoin & join, Map & map, size_t rows, const ColumnRawPtrs & key_columns, const Sizes & key_sizes, Block * stored_block, ConstNullMapPtr null_map, UInt8ColumnDataPtr join_mask, Arena & pool) { [[maybe_unused]] constexpr bool mapped_one = std::is_same_v; - constexpr bool is_asof_join = STRICTNESS == ASTTableJoin::Strictness::Asof; + constexpr bool is_asof_join = STRICTNESS == JoinStrictness::Asof; const IColumn * asof_column [[maybe_unused]] = nullptr; if constexpr (is_asof_join) @@ -670,7 +670,7 @@ namespace } - template + template size_t insertFromBlockImplType( HashJoin & join, Map & map, size_t rows, const ColumnRawPtrs & key_columns, const Sizes & key_sizes, Block * stored_block, ConstNullMapPtr null_map, UInt8ColumnDataPtr join_mask, Arena & pool) @@ -684,7 +684,7 @@ namespace } - template + template size_t insertFromBlockImpl( HashJoin & join, HashJoin::Type type, Maps & maps, size_t rows, const ColumnRawPtrs & key_columns, const Sizes & key_sizes, Block * stored_block, ConstNullMapPtr null_map, UInt8ColumnDataPtr join_mask, Arena & pool) @@ -723,7 +723,7 @@ void HashJoin::initRightBlockStructure(Block & saved_block_sample) { saved_block_sample = right_table_keys.cloneEmpty(); } - else if (strictness == ASTTableJoin::Strictness::Asof) + else if (strictness == JoinStrictness::Asof) { /// Save ASOF key saved_block_sample.insert(right_table_keys.safeGetByPosition(right_table_keys.columns() - 1)); @@ -824,7 +824,7 @@ bool HashJoin::addJoinedBlock(const Block & source_block, bool check_limits) } } - if (kind != ASTTableJoin::Kind::Cross) + if (kind != JoinKind::Cross) { joinDispatch(kind, strictness, data->maps[onexpr_idx], [&](auto kind_, auto strictness_, auto & map) { @@ -1072,20 +1072,20 @@ private: } }; -template +template struct JoinFeatures { - static constexpr bool is_any_join = STRICTNESS == ASTTableJoin::Strictness::Any; - static constexpr bool is_any_or_semi_join = STRICTNESS == ASTTableJoin::Strictness::Any || STRICTNESS == ASTTableJoin::Strictness::RightAny || (STRICTNESS == ASTTableJoin::Strictness::Semi && KIND == ASTTableJoin::Kind::Left); - static constexpr bool is_all_join = STRICTNESS == ASTTableJoin::Strictness::All; - static constexpr bool is_asof_join = STRICTNESS == ASTTableJoin::Strictness::Asof; - static constexpr bool is_semi_join = STRICTNESS == ASTTableJoin::Strictness::Semi; - static constexpr bool is_anti_join = STRICTNESS == ASTTableJoin::Strictness::Anti; + static constexpr bool is_any_join = STRICTNESS == JoinStrictness::Any; + static constexpr bool is_any_or_semi_join = STRICTNESS == JoinStrictness::Any || STRICTNESS == JoinStrictness::RightAny || (STRICTNESS == JoinStrictness::Semi && KIND == JoinKind::Left); + static constexpr bool is_all_join = STRICTNESS == JoinStrictness::All; + static constexpr bool is_asof_join = STRICTNESS == JoinStrictness::Asof; + static constexpr bool is_semi_join = STRICTNESS == JoinStrictness::Semi; + static constexpr bool is_anti_join = STRICTNESS == JoinStrictness::Anti; - static constexpr bool left = KIND == ASTTableJoin::Kind::Left; - static constexpr bool right = KIND == ASTTableJoin::Kind::Right; - static constexpr bool inner = KIND == ASTTableJoin::Kind::Inner; - static constexpr bool full = KIND == ASTTableJoin::Kind::Full; + static constexpr bool left = KIND == JoinKind::Left; + static constexpr bool right = KIND == JoinKind::Right; + static constexpr bool inner = KIND == JoinKind::Inner; + static constexpr bool full = KIND == JoinKind::Full; static constexpr bool need_replication = is_all_join || (is_any_join && right) || (is_semi_join && right); static constexpr bool need_filter = !need_replication && (inner || right || (is_semi_join && left) || (is_anti_join && left)); @@ -1239,7 +1239,7 @@ void setUsed(IColumn::Filter & filter [[maybe_unused]], size_t pos [[maybe_unuse /// Joins right table columns which indexes are present in right_indexes using specified map. /// Makes filter (1 if row presented in right table) and returns offsets to replicate (for ALL JOINS). -template +template NO_INLINE IColumn::Filter joinRightColumns( std::vector && key_getter_vector, const std::vector & mapv, @@ -1322,7 +1322,7 @@ NO_INLINE IColumn::Filter joinRightColumns( addFoundRowAll(mapped, added_columns, current_offset, known_rows, used_flags_opt); } } - else if constexpr (jf.is_any_join && KIND == ASTTableJoin::Kind::Inner) + else if constexpr (jf.is_any_join && KIND == JoinKind::Inner) { bool used_once = used_flags.template setUsedOnce(find_result); @@ -1389,7 +1389,7 @@ NO_INLINE IColumn::Filter joinRightColumns( return filter; } -template +template IColumn::Filter joinRightColumnsSwitchMultipleDisjuncts( std::vector && key_getter_vector, const std::vector & mapv, @@ -1401,7 +1401,7 @@ IColumn::Filter joinRightColumnsSwitchMultipleDisjuncts( : joinRightColumns(std::forward>(key_getter_vector), mapv, added_columns, used_flags); } -template +template IColumn::Filter joinRightColumnsSwitchNullability( std::vector && key_getter_vector, const std::vector & mapv, @@ -1426,14 +1426,14 @@ IColumn::Filter joinRightColumnsSwitchNullability( } } -template +template IColumn::Filter switchJoinRightColumns( const std::vector & mapv, AddedColumns & added_columns, HashJoin::Type type, JoinStuff::JoinUsedFlags & used_flags) { - constexpr bool is_asof_join = STRICTNESS == ASTTableJoin::Strictness::Asof; + constexpr bool is_asof_join = STRICTNESS == JoinStrictness::Asof; switch (type) { case HashJoin::Type::EMPTY: @@ -1476,13 +1476,13 @@ IColumn::Filter switchJoinRightColumns( } } -template +template IColumn::Filter dictionaryJoinRightColumns(const TableJoin & table_join, AddedColumns & added_columns) { - if constexpr (KIND == ASTTableJoin::Kind::Left && - (STRICTNESS == ASTTableJoin::Strictness::Any || - STRICTNESS == ASTTableJoin::Strictness::Semi || - STRICTNESS == ASTTableJoin::Strictness::Anti)) + if constexpr (KIND == JoinKind::Left && + (STRICTNESS == JoinStrictness::Any || + STRICTNESS == JoinStrictness::Semi || + STRICTNESS == JoinStrictness::Anti)) { assert(added_columns.join_on_keys.size() == 1); @@ -1500,7 +1500,7 @@ IColumn::Filter dictionaryJoinRightColumns(const TableJoin & table_join, AddedCo } /// nameless -template +template void HashJoin::joinBlockImpl( Block & block, const Block & block_with_columns_to_add, @@ -1745,11 +1745,11 @@ DataTypePtr HashJoin::joinGetCheckAndGetReturnType(const DataTypes & data_types, } /// TODO: return multiple columns as named tuple -/// TODO: return array of values when strictness == ASTTableJoin::Strictness::All +/// TODO: return array of values when strictness == JoinStrictness::All ColumnWithTypeAndName HashJoin::joinGet(const Block & block, const Block & block_with_columns_to_add) const { - bool is_valid = (strictness == ASTTableJoin::Strictness::Any || strictness == ASTTableJoin::Strictness::RightAny) - && kind == ASTTableJoin::Kind::Left; + bool is_valid = (strictness == JoinStrictness::Any || strictness == JoinStrictness::RightAny) + && kind == JoinKind::Left; if (!is_valid) throw Exception("joinGet only supports StorageJoin of type Left Any", ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN); const auto & key_names_right = table_join->getOnlyClause().key_names_right; @@ -1763,12 +1763,12 @@ ColumnWithTypeAndName HashJoin::joinGet(const Block & block, const Block & block keys.insert(std::move(key)); } - static_assert(!MapGetter::flagged, + static_assert(!MapGetter::flagged, "joinGet are not protected from hash table changes between block processing"); std::vector maps_vector; maps_vector.push_back(&std::get(data->maps[0])); - joinBlockImpl( + joinBlockImpl( keys, block_with_columns_to_add, maps_vector, true); return keys.getByPosition(keys.columns() - 1); } @@ -1791,46 +1791,43 @@ void HashJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) right_sample_block, onexpr.key_names_right, cond_column_name.second); } - if (kind == ASTTableJoin::Kind::Cross) + if (kind == JoinKind::Cross) { joinBlockImplCross(block, not_processed); return; } - if (kind == ASTTableJoin::Kind::Right || kind == ASTTableJoin::Kind::Full) + if (kind == JoinKind::Right || kind == JoinKind::Full) { materializeBlockInplace(block); } if (overDictionary()) { - using Kind = ASTTableJoin::Kind; - using Strictness = ASTTableJoin::Strictness; - auto & map = std::get(data->maps[0]); std::vector*> maps_vector; maps_vector.push_back(&map); - if (kind == Kind::Left) + if (kind == JoinKind::Left) { switch (strictness) { - case Strictness::Any: - case Strictness::All: - joinBlockImpl(block, sample_block_with_columns_to_add, maps_vector); + case JoinStrictness::Any: + case JoinStrictness::All: + joinBlockImpl(block, sample_block_with_columns_to_add, maps_vector); break; - case Strictness::Semi: - joinBlockImpl(block, sample_block_with_columns_to_add, maps_vector); + case JoinStrictness::Semi: + joinBlockImpl(block, sample_block_with_columns_to_add, maps_vector); break; - case Strictness::Anti: - joinBlockImpl(block, sample_block_with_columns_to_add, maps_vector); + case JoinStrictness::Anti: + joinBlockImpl(block, sample_block_with_columns_to_add, maps_vector); break; default: throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong JOIN combination: dictionary + {} {}", strictness, kind); } } - else if (kind == Kind::Inner && strictness == Strictness::All) - joinBlockImpl(block, sample_block_with_columns_to_add, maps_vector); + else if (kind == JoinKind::Inner && strictness == JoinStrictness::All) + joinBlockImpl(block, sample_block_with_columns_to_add, maps_vector); else throw Exception(ErrorCodes::LOGICAL_ERROR, "Wrong JOIN combination: {} {}", strictness, kind); @@ -1979,7 +1976,7 @@ private: return rows_added; } - template + template size_t fillColumnsFromMap(const Maps & maps, MutableColumns & columns_keys_and_right) { switch (parent.data->type) @@ -1996,7 +1993,7 @@ private: __builtin_unreachable(); } - template + template size_t fillColumns(const Map & map, MutableColumns & columns_keys_and_right) { size_t rows_added = 0; @@ -2089,8 +2086,8 @@ std::shared_ptr HashJoin::getNonJoinedBlocks(const Block & left const Block & result_sample_block, UInt64 max_block_size) const { - if (table_join->strictness() == ASTTableJoin::Strictness::Asof || - table_join->strictness() == ASTTableJoin::Strictness::Semi || + if (table_join->strictness() == JoinStrictness::Asof || + table_join->strictness() == JoinStrictness::Semi || !isRightOrFull(table_join->kind())) { return {}; diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h index 00bf92059bd..54c641627c0 100644 --- a/src/Interpreters/HashJoin.h +++ b/src/Interpreters/HashJoin.h @@ -54,10 +54,10 @@ public: /// Update size for vector with flags. /// Calling this method invalidates existing flags. /// It can be called several times, but all of them should happen before using this structure. - template + template void reinit(size_t size_); - template + template void reinit(const Block * block_ptr); bool getUsedSafe(size_t i) const; @@ -199,10 +199,10 @@ public: bool alwaysReturnsEmptySet() const final; - ASTTableJoin::Kind getKind() const { return kind; } - ASTTableJoin::Strictness getStrictness() const { return strictness; } + JoinKind getKind() const { return kind; } + JoinStrictness getStrictness() const { return strictness; } const std::optional & getAsofType() const { return asof_type; } - ASOF::Inequality getAsofInequality() const { return asof_inequality; } + ASOFJoinInequality getAsofInequality() const { return asof_inequality; } bool anyTakeLastRow() const { return any_take_last_row; } const ColumnWithTypeAndName & rightAsofKeyColumn() const; @@ -367,15 +367,15 @@ private: friend class JoinSource; std::shared_ptr table_join; - ASTTableJoin::Kind kind; - ASTTableJoin::Strictness strictness; + JoinKind kind; + JoinStrictness strictness; /// This join was created from StorageJoin and it is already filled. bool from_storage_join = false; bool any_take_last_row; /// Overwrite existing values when encountering the same key again std::optional asof_type; - ASOF::Inequality asof_inequality; + ASOFJoinInequality asof_inequality; /// Right table data. StorageJoin shares it between many Join objects. /// Flags that indicate that particular row already used in join. @@ -412,7 +412,7 @@ private: Block structureRightBlock(const Block & stored_block) const; void initRightBlockStructure(Block & saved_block_sample); - template + template void joinBlockImpl( Block & block, const Block & block_with_columns_to_add, @@ -421,7 +421,7 @@ private: void joinBlockImplCross(Block & block, ExtraBlockPtr & not_processed) const; - static Type chooseMethod(ASTTableJoin::Kind kind, const ColumnRawPtrs & key_columns, Sizes & key_sizes); + static Type chooseMethod(JoinKind kind, const ColumnRawPtrs & key_columns, Sizes & key_sizes); bool empty() const; bool overDictionary() const; diff --git a/src/Interpreters/InJoinSubqueriesPreprocessor.cpp b/src/Interpreters/InJoinSubqueriesPreprocessor.cpp index 28e1757bc3e..66ee32750f3 100644 --- a/src/Interpreters/InJoinSubqueriesPreprocessor.cpp +++ b/src/Interpreters/InJoinSubqueriesPreprocessor.cpp @@ -105,7 +105,7 @@ private: throw Exception("Logical error: unexpected function name " + concrete->name, ErrorCodes::LOGICAL_ERROR); } else if (table_join) - table_join->locality = ASTTableJoin::Locality::Global; + table_join->locality = JoinLocality::Global; else throw Exception("Logical error: unexpected AST node", ErrorCodes::LOGICAL_ERROR); } @@ -189,7 +189,7 @@ private: return; ASTTableJoin * table_join = node.table_join->as(); - if (table_join->locality != ASTTableJoin::Locality::Global) + if (table_join->locality != JoinLocality::Global) { if (auto * table = node.table_expression->as()) { diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index e07430c0feb..06b97c3f75a 100644 --- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -254,7 +254,7 @@ bool needRewrite(ASTSelectQuery & select, std::vectortable_join->as(); - if (join.kind == ASTTableJoin::Kind::Comma) + if (join.kind == JoinKind::Comma) throw Exception("COMMA to CROSS JOIN rewriter is not enabled or cannot rewrite query", ErrorCodes::NOT_IMPLEMENTED); if (join.using_expression_list) diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index 8ac0bc45f06..fd47c3d747f 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -62,7 +62,7 @@ void replaceJoinedTable(const ASTSelectQuery & select_query) const auto & table_join = join->table_join->as(); /// TODO: Push down for CROSS JOIN is not OK [disabled] - if (table_join.kind == ASTTableJoin::Kind::Cross) + if (table_join.kind == JoinKind::Cross) return; /* Do not push down predicates for ASOF because it can lead to incorrect results diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index 711b71a2b3d..77091a0a6d7 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -468,9 +468,9 @@ MergeJoin::MergeJoin(std::shared_ptr table_join_, const Block & right : table_join(table_join_) , size_limits(table_join->sizeLimits()) , right_sample_block(right_sample_block_) - , is_any_join(table_join->strictness() == ASTTableJoin::Strictness::Any) - , is_all_join(table_join->strictness() == ASTTableJoin::Strictness::All) - , is_semi_join(table_join->strictness() == ASTTableJoin::Strictness::Semi) + , is_any_join(table_join->strictness() == JoinStrictness::Any) + , is_all_join(table_join->strictness() == JoinStrictness::All) + , is_semi_join(table_join->strictness() == JoinStrictness::Semi) , is_inner(isInner(table_join->kind())) , is_left(isLeft(table_join->kind())) , is_right(isRight(table_join->kind())) @@ -482,10 +482,10 @@ MergeJoin::MergeJoin(std::shared_ptr table_join_, const Block & right { switch (table_join->strictness()) { - case ASTTableJoin::Strictness::All: + case JoinStrictness::All: break; - case ASTTableJoin::Strictness::Any: - case ASTTableJoin::Strictness::Semi: + case JoinStrictness::Any: + case JoinStrictness::Semi: if (!is_left && !is_inner) throw Exception("Not supported. MergeJoin supports SEMI and ANY variants only for LEFT and INNER JOINs.", ErrorCodes::NOT_IMPLEMENTED); @@ -1109,7 +1109,7 @@ private: std::shared_ptr MergeJoin::getNonJoinedBlocks( const Block & left_sample_block, const Block & result_sample_block, UInt64 max_block_size) const { - if (table_join->strictness() == ASTTableJoin::Strictness::All && (is_right || is_full)) + if (table_join->strictness() == JoinStrictness::All && (is_right || is_full)) { size_t left_columns_count = left_sample_block.columns(); assert(left_columns_count == result_sample_block.columns() - right_columns_to_add.columns()); diff --git a/src/Interpreters/RowRefs.cpp b/src/Interpreters/RowRefs.cpp index 2a18c2c700a..09af04bc7e5 100644 --- a/src/Interpreters/RowRefs.cpp +++ b/src/Interpreters/RowRefs.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -39,7 +40,7 @@ void callWithType(TypeIndex type, F && f) __builtin_unreachable(); } -template +template class SortedLookupVector : public SortedLookupVectorBase { struct Entry @@ -77,8 +78,8 @@ public: using Entries = PaddedPODArray; using RowRefs = PaddedPODArray; - static constexpr bool is_descending = (inequality == ASOF::Inequality::Greater || inequality == ASOF::Inequality::GreaterOrEquals); - static constexpr bool is_strict = (inequality == ASOF::Inequality::Less) || (inequality == ASOF::Inequality::Greater); + static constexpr bool is_descending = (inequality == ASOFJoinInequality::Greater || inequality == ASOFJoinInequality::GreaterOrEquals); + static constexpr bool is_strict = (inequality == ASOFJoinInequality::Less) || (inequality == ASOFJoinInequality::Greater); void insert(const IColumn & asof_column, const Block * block, size_t row_num) override { @@ -216,7 +217,7 @@ private: }; } -AsofRowRefs createAsofRowRef(TypeIndex type, ASOF::Inequality inequality) +AsofRowRefs createAsofRowRef(TypeIndex type, ASOFJoinInequality inequality) { AsofRowRefs result; auto call = [&](const auto & t) @@ -224,17 +225,17 @@ AsofRowRefs createAsofRowRef(TypeIndex type, ASOF::Inequality inequality) using T = std::decay_t; switch (inequality) { - case ASOF::Inequality::LessOrEquals: - result = std::make_unique>(); + case ASOFJoinInequality::LessOrEquals: + result = std::make_unique>(); break; - case ASOF::Inequality::Less: - result = std::make_unique>(); + case ASOFJoinInequality::Less: + result = std::make_unique>(); break; - case ASOF::Inequality::GreaterOrEquals: - result = std::make_unique>(); + case ASOFJoinInequality::GreaterOrEquals: + result = std::make_unique>(); break; - case ASOF::Inequality::Greater: - result = std::make_unique>(); + case ASOFJoinInequality::Greater: + result = std::make_unique>(); break; default: throw Exception("Invalid ASOF Join order", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Interpreters/RowRefs.h b/src/Interpreters/RowRefs.h index fa5ce867613..2c9f2062a82 100644 --- a/src/Interpreters/RowRefs.h +++ b/src/Interpreters/RowRefs.h @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include @@ -161,5 +161,5 @@ struct SortedLookupVectorBase // It only contains a std::unique_ptr which is memmovable. // Source: https://github.com/ClickHouse/ClickHouse/issues/4906 using AsofRowRefs = std::unique_ptr; -AsofRowRefs createAsofRowRef(TypeIndex type, ASOF::Inequality inequality); +AsofRowRefs createAsofRowRef(TypeIndex type, ASOFJoinInequality inequality); } diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp index 029038357c1..5d402ca4127 100644 --- a/src/Interpreters/TableJoin.cpp +++ b/src/Interpreters/TableJoin.cpp @@ -388,17 +388,17 @@ void TableJoin::addJoinedColumnsAndCorrectTypesImpl(TColumns & left_columns, boo left_columns.emplace_back(col.name, col.type); } -bool TableJoin::sameStrictnessAndKind(ASTTableJoin::Strictness strictness_, ASTTableJoin::Kind kind_) const +bool TableJoin::sameStrictnessAndKind(JoinStrictness strictness_, JoinKind kind_) const { if (strictness_ == strictness() && kind_ == kind()) return true; /// Compatibility: old ANY INNER == new SEMI LEFT - if (strictness_ == ASTTableJoin::Strictness::Semi && isLeft(kind_) && - strictness() == ASTTableJoin::Strictness::RightAny && isInner(kind())) + if (strictness_ == JoinStrictness::Semi && isLeft(kind_) && + strictness() == JoinStrictness::RightAny && isInner(kind())) return true; - if (strictness() == ASTTableJoin::Strictness::Semi && isLeft(kind()) && - strictness_ == ASTTableJoin::Strictness::RightAny && isInner(kind_)) + if (strictness() == JoinStrictness::Semi && isLeft(kind()) && + strictness_ == JoinStrictness::RightAny && isInner(kind_)) return true; return false; @@ -411,8 +411,8 @@ bool TableJoin::oneDisjunct() const bool TableJoin::needStreamWithNonJoinedRows() const { - if (strictness() == ASTTableJoin::Strictness::Asof || - strictness() == ASTTableJoin::Strictness::Semi) + if (strictness() == JoinStrictness::Asof || + strictness() == JoinStrictness::Semi) return false; return isRightOrFull(kind()); } @@ -430,13 +430,11 @@ static std::optional getDictKeyName(const String & dict_name , ContextPt bool TableJoin::tryInitDictJoin(const Block & sample_block, ContextPtr context) { - using Strictness = ASTTableJoin::Strictness; - - bool allowed_inner = isInner(kind()) && strictness() == Strictness::All; - bool allowed_left = isLeft(kind()) && (strictness() == Strictness::Any || - strictness() == Strictness::All || - strictness() == Strictness::Semi || - strictness() == Strictness::Anti); + bool allowed_inner = isInner(kind()) && strictness() == JoinStrictness::All; + bool allowed_left = isLeft(kind()) && (strictness() == JoinStrictness::Any || + strictness() == JoinStrictness::All || + strictness() == JoinStrictness::Semi || + strictness() == JoinStrictness::Anti); /// Support ALL INNER, [ANY | ALL | SEMI | ANTI] LEFT if (!allowed_inner && !allowed_left) @@ -569,7 +567,7 @@ void TableJoin::inferJoinKeyCommonType(const LeftNamesAndTypes & left, const Rig for (const auto & col : right) right_types[renamedRightColumnName(col.name)] = col.type; - if (strictness() == ASTTableJoin::Strictness::Asof) + if (strictness() == JoinStrictness::Asof) { if (clauses.size() != 1) throw DB::Exception("ASOF join over multiple keys is not supported", ErrorCodes::NOT_IMPLEMENTED); @@ -815,16 +813,16 @@ void TableJoin::assertHasOneOnExpr() const void TableJoin::resetToCross() { this->resetKeys(); - this->table_join.kind = ASTTableJoin::Kind::Cross; + this->table_join.kind = JoinKind::Cross; } bool TableJoin::allowParallelHashJoin() const { if (dictionary_reader || !join_algorithm.isSet(JoinAlgorithm::PARALLEL_HASH)) return false; - if (table_join.kind != ASTTableJoin::Kind::Left && table_join.kind != ASTTableJoin::Kind::Inner) + if (table_join.kind != JoinKind::Left && table_join.kind != JoinKind::Inner) return false; - if (table_join.strictness == ASTTableJoin::Strictness::Asof) + if (table_join.strictness == JoinStrictness::Asof) return false; if (isSpecialStorage() || !oneDisjunct()) return false; diff --git a/src/Interpreters/TableJoin.h b/src/Interpreters/TableJoin.h index 57895d6d1c1..dea82550f77 100644 --- a/src/Interpreters/TableJoin.h +++ b/src/Interpreters/TableJoin.h @@ -125,7 +125,7 @@ private: ASTTableJoin table_join; - ASOF::Inequality asof_inequality = ASOF::Inequality::GreaterOrEquals; + ASOFJoinInequality asof_inequality = ASOFJoinInequality::GreaterOrEquals; /// All columns which can be read from joined table. Duplicating names are qualified. NamesAndTypesList columns_from_joined_table; @@ -176,7 +176,7 @@ public: TableJoin(const Settings & settings, VolumePtr tmp_volume_); /// for StorageJoin - TableJoin(SizeLimits limits, bool use_nulls, ASTTableJoin::Kind kind, ASTTableJoin::Strictness strictness, + TableJoin(SizeLimits limits, bool use_nulls, JoinKind kind, JoinStrictness strictness, const Names & key_names_right) : size_limits(limits) , default_max_bytes(0) @@ -188,9 +188,9 @@ public: table_join.strictness = strictness; } - ASTTableJoin::Kind kind() const { return table_join.kind; } - ASTTableJoin::Strictness strictness() const { return table_join.strictness; } - bool sameStrictnessAndKind(ASTTableJoin::Strictness, ASTTableJoin::Kind) const; + JoinKind kind() const { return table_join.kind; } + JoinStrictness strictness() const { return table_join.strictness; } + bool sameStrictnessAndKind(JoinStrictness, JoinKind) const; const SizeLimits & sizeLimits() const { return size_limits; } VolumePtr getTemporaryVolume() { return tmp_volume; } @@ -280,8 +280,8 @@ public: const ColumnsWithTypeAndName & left_sample_columns, const ColumnsWithTypeAndName & right_sample_columns); - void setAsofInequality(ASOF::Inequality inequality) { asof_inequality = inequality; } - ASOF::Inequality getAsofInequality() { return asof_inequality; } + void setAsofInequality(ASOFJoinInequality inequality) { asof_inequality = inequality; } + ASOFJoinInequality getAsofInequality() { return asof_inequality; } ASTPtr leftKeysList() const; ASTPtr rightKeysList() const; /// For ON syntax only diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index bd18984faed..3385534ba9a 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -612,13 +612,13 @@ void setJoinStrictness(ASTSelectQuery & select_query, JoinStrictness join_defaul auto & table_join = const_cast(node)->table_join->as(); - if (table_join.strictness == ASTTableJoin::Strictness::Unspecified && - table_join.kind != ASTTableJoin::Kind::Cross) + if (table_join.strictness == JoinStrictness::Unspecified && + table_join.kind != JoinKind::Cross) { - if (join_default_strictness == JoinStrictness::ANY) - table_join.strictness = ASTTableJoin::Strictness::Any; - else if (join_default_strictness == JoinStrictness::ALL) - table_join.strictness = ASTTableJoin::Strictness::All; + if (join_default_strictness == JoinStrictness::Any) + table_join.strictness = JoinStrictness::Any; + else if (join_default_strictness == JoinStrictness::All) + table_join.strictness = JoinStrictness::All; else throw Exception("Expected ANY or ALL in JOIN section, because setting (join_default_strictness) is empty", DB::ErrorCodes::EXPECTED_ALL_OR_ANY); @@ -626,19 +626,19 @@ void setJoinStrictness(ASTSelectQuery & select_query, JoinStrictness join_defaul if (old_any) { - if (table_join.strictness == ASTTableJoin::Strictness::Any && - table_join.kind == ASTTableJoin::Kind::Inner) + if (table_join.strictness == JoinStrictness::Any && + table_join.kind == JoinKind::Inner) { - table_join.strictness = ASTTableJoin::Strictness::Semi; - table_join.kind = ASTTableJoin::Kind::Left; + table_join.strictness = JoinStrictness::Semi; + table_join.kind = JoinKind::Left; } - if (table_join.strictness == ASTTableJoin::Strictness::Any) - table_join.strictness = ASTTableJoin::Strictness::RightAny; + if (table_join.strictness == JoinStrictness::Any) + table_join.strictness = JoinStrictness::RightAny; } else { - if (table_join.strictness == ASTTableJoin::Strictness::Any && table_join.kind == ASTTableJoin::Kind::Full) + if (table_join.strictness == JoinStrictness::Any && table_join.kind == JoinKind::Full) throw Exception("ANY FULL JOINs are not implemented", ErrorCodes::NOT_IMPLEMENTED); } @@ -720,7 +720,7 @@ void collectJoinedColumns(TableJoin & analyzed_join, ASTTableJoin & table_join, } else if (table_join.on_expression) { - bool is_asof = (table_join.strictness == ASTTableJoin::Strictness::Asof); + bool is_asof = (table_join.strictness == JoinStrictness::Asof); CollectJoinOnKeysVisitor::Data data{analyzed_join, tables[0], tables[1], aliases, is_asof}; if (auto * or_func = table_join.on_expression->as(); or_func && or_func->name == "or") diff --git a/src/Interpreters/asof.h b/src/Interpreters/asof.h deleted file mode 100644 index 439bf4cc58c..00000000000 --- a/src/Interpreters/asof.h +++ /dev/null @@ -1,46 +0,0 @@ -#pragma once -#include - -namespace DB -{ -namespace ASOF -{ - -enum class Inequality -{ - None = 0, - Less, - Greater, - LessOrEquals, - GreaterOrEquals, -}; - -inline Inequality getInequality(const std::string & func_name) -{ - Inequality inequality{Inequality::None}; - if (func_name == "less") - inequality = Inequality::Less; - else if (func_name == "greater") - inequality = Inequality::Greater; - else if (func_name == "lessOrEquals") - inequality = Inequality::LessOrEquals; - else if (func_name == "greaterOrEquals") - inequality = Inequality::GreaterOrEquals; - return inequality; -} - -inline Inequality reverseInequality(Inequality inequality) -{ - if (inequality == Inequality::Less) - return Inequality::Greater; - else if (inequality == Inequality::Greater) - return Inequality::Less; - else if (inequality == Inequality::LessOrEquals) - return Inequality::GreaterOrEquals; - else if (inequality == Inequality::GreaterOrEquals) - return Inequality::LessOrEquals; - return Inequality::None; -} - -} -} diff --git a/src/Interpreters/joinDispatch.h b/src/Interpreters/joinDispatch.h index a9de8206b65..ca2fcd27cda 100644 --- a/src/Interpreters/joinDispatch.h +++ b/src/Interpreters/joinDispatch.h @@ -12,58 +12,57 @@ namespace DB { -template +template struct MapGetter; -template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = false; }; -template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = false; }; -template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = true; }; -template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = true; }; +template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = false; }; +template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = false; }; +template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = true; }; +template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = true; }; -template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = false; }; -template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = true; }; -template <> struct MapGetter { using Map = HashJoin::MapsAll; static constexpr bool flagged = true; }; -template <> struct MapGetter { using Map = HashJoin::MapsAll; static constexpr bool flagged = true; }; +template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = false; }; +template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = true; }; +template <> struct MapGetter { using Map = HashJoin::MapsAll; static constexpr bool flagged = true; }; +template <> struct MapGetter { using Map = HashJoin::MapsAll; static constexpr bool flagged = true; }; -template <> struct MapGetter { using Map = HashJoin::MapsAll; static constexpr bool flagged = false; }; -template <> struct MapGetter { using Map = HashJoin::MapsAll; static constexpr bool flagged = false; }; -template <> struct MapGetter { using Map = HashJoin::MapsAll; static constexpr bool flagged = true; }; -template <> struct MapGetter { using Map = HashJoin::MapsAll; static constexpr bool flagged = true; }; +template <> struct MapGetter { using Map = HashJoin::MapsAll; static constexpr bool flagged = false; }; +template <> struct MapGetter { using Map = HashJoin::MapsAll; static constexpr bool flagged = false; }; +template <> struct MapGetter { using Map = HashJoin::MapsAll; static constexpr bool flagged = true; }; +template <> struct MapGetter { using Map = HashJoin::MapsAll; static constexpr bool flagged = true; }; /// Only SEMI LEFT and SEMI RIGHT are valid. INNER and FULL are here for templates instantiation. -template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = false; }; -template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = false; }; -template <> struct MapGetter { using Map = HashJoin::MapsAll; static constexpr bool flagged = true; }; -template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = false; }; +template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = false; }; +template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = false; }; +template <> struct MapGetter { using Map = HashJoin::MapsAll; static constexpr bool flagged = true; }; +template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = false; }; /// Only SEMI LEFT and SEMI RIGHT are valid. INNER and FULL are here for templates instantiation. -template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = false; }; -template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = false; }; -template <> struct MapGetter { using Map = HashJoin::MapsAll; static constexpr bool flagged = true; }; -template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = false; }; +template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = false; }; +template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = false; }; +template <> struct MapGetter { using Map = HashJoin::MapsAll; static constexpr bool flagged = true; }; +template <> struct MapGetter { using Map = HashJoin::MapsOne; static constexpr bool flagged = false; }; -template -struct MapGetter { using Map = HashJoin::MapsAsof; static constexpr bool flagged = false; }; +template +struct MapGetter { using Map = HashJoin::MapsAsof; static constexpr bool flagged = false; }; - -static constexpr std::array STRICTNESSES = { - ASTTableJoin::Strictness::RightAny, - ASTTableJoin::Strictness::Any, - ASTTableJoin::Strictness::All, - ASTTableJoin::Strictness::Asof, - ASTTableJoin::Strictness::Semi, - ASTTableJoin::Strictness::Anti, +static constexpr std::array STRICTNESSES = { + JoinStrictness::RightAny, + JoinStrictness::Any, + JoinStrictness::All, + JoinStrictness::Asof, + JoinStrictness::Semi, + JoinStrictness::Anti, }; -static constexpr std::array KINDS = { - ASTTableJoin::Kind::Left, - ASTTableJoin::Kind::Inner, - ASTTableJoin::Kind::Full, - ASTTableJoin::Kind::Right +static constexpr std::array KINDS = { + JoinKind::Left, + JoinKind::Inner, + JoinKind::Full, + JoinKind::Right }; /// Init specified join map -inline bool joinDispatchInit(ASTTableJoin::Kind kind, ASTTableJoin::Strictness strictness, HashJoin::MapsVariant & maps) +inline bool joinDispatchInit(JoinKind kind, JoinStrictness strictness, HashJoin::MapsVariant & maps) { return static_for<0, KINDS.size() * STRICTNESSES.size()>([&](auto ij) { @@ -80,7 +79,7 @@ inline bool joinDispatchInit(ASTTableJoin::Kind kind, ASTTableJoin::Strictness s /// Call function on specified join map template -inline bool joinDispatch(ASTTableJoin::Kind kind, ASTTableJoin::Strictness strictness, MapsVariant & maps, Func && func) +inline bool joinDispatch(JoinKind kind, JoinStrictness strictness, MapsVariant & maps, Func && func) { return static_for<0, KINDS.size() * STRICTNESSES.size()>([&](auto ij) { @@ -91,8 +90,8 @@ inline bool joinDispatch(ASTTableJoin::Kind kind, ASTTableJoin::Strictness stric if (kind == KINDS[i] && strictness == STRICTNESSES[j]) { func( - std::integral_constant(), - std::integral_constant(), + std::integral_constant(), + std::integral_constant(), std::get::Map>(maps)); return true; } @@ -102,7 +101,7 @@ inline bool joinDispatch(ASTTableJoin::Kind kind, ASTTableJoin::Strictness stric /// Call function on specified join map template -inline bool joinDispatch(ASTTableJoin::Kind kind, ASTTableJoin::Strictness strictness, std::vector & mapsv, Func && func) +inline bool joinDispatch(JoinKind kind, JoinStrictness strictness, std::vector & mapsv, Func && func) { return static_for<0, KINDS.size() * STRICTNESSES.size()>([&](auto ij) { @@ -120,8 +119,8 @@ inline bool joinDispatch(ASTTableJoin::Kind kind, ASTTableJoin::Strictness stric } func( - std::integral_constant(), - std::integral_constant(), + std::integral_constant(), + std::integral_constant(), v /*std::get::Map>(maps)*/); return true; diff --git a/src/Parsers/ASTTablesInSelectQuery.cpp b/src/Parsers/ASTTablesInSelectQuery.cpp index 32d7f166564..3b7a3a342e6 100644 --- a/src/Parsers/ASTTablesInSelectQuery.cpp +++ b/src/Parsers/ASTTablesInSelectQuery.cpp @@ -150,41 +150,41 @@ void ASTTableJoin::formatImplBeforeTable(const FormatSettings & settings, Format settings.ostr << (settings.hilite ? hilite_keyword : ""); std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); - if (kind != Kind::Comma) + if (kind != JoinKind::Comma) { settings.ostr << settings.nl_or_ws << indent_str; } switch (locality) { - case Locality::Unspecified: - case Locality::Local: + case JoinLocality::Unspecified: + case JoinLocality::Local: break; - case Locality::Global: + case JoinLocality::Global: settings.ostr << "GLOBAL "; break; } - if (kind != Kind::Cross && kind != Kind::Comma) + if (kind != JoinKind::Cross && kind != JoinKind::Comma) { switch (strictness) { - case Strictness::Unspecified: + case JoinStrictness::Unspecified: break; - case Strictness::RightAny: - case Strictness::Any: + case JoinStrictness::RightAny: + case JoinStrictness::Any: settings.ostr << "ANY "; break; - case Strictness::All: + case JoinStrictness::All: settings.ostr << "ALL "; break; - case Strictness::Asof: + case JoinStrictness::Asof: settings.ostr << "ASOF "; break; - case Strictness::Semi: + case JoinStrictness::Semi: settings.ostr << "SEMI "; break; - case Strictness::Anti: + case JoinStrictness::Anti: settings.ostr << "ANTI "; break; } @@ -192,22 +192,22 @@ void ASTTableJoin::formatImplBeforeTable(const FormatSettings & settings, Format switch (kind) { - case Kind::Inner: + case JoinKind::Inner: settings.ostr << "INNER JOIN"; break; - case Kind::Left: + case JoinKind::Left: settings.ostr << "LEFT JOIN"; break; - case Kind::Right: + case JoinKind::Right: settings.ostr << "RIGHT JOIN"; break; - case Kind::Full: + case JoinKind::Full: settings.ostr << "FULL OUTER JOIN"; break; - case Kind::Cross: + case JoinKind::Cross: settings.ostr << "CROSS JOIN"; break; - case Kind::Comma: + case JoinKind::Comma: settings.ostr << ","; break; } diff --git a/src/Parsers/ASTTablesInSelectQuery.h b/src/Parsers/ASTTablesInSelectQuery.h index 9eddaf3137d..a004cbf9847 100644 --- a/src/Parsers/ASTTablesInSelectQuery.h +++ b/src/Parsers/ASTTablesInSelectQuery.h @@ -1,8 +1,10 @@ #pragma once -#include #include +#include + +#include namespace DB { @@ -64,40 +66,9 @@ struct ASTTableExpression : public IAST /// How to JOIN another table. struct ASTTableJoin : public IAST { - /// Algorithm for distributed query processing. - enum class Locality - { - Unspecified, - Local, /// Perform JOIN, using only data available on same servers (co-located data). - Global /// Collect and merge data from remote servers, and broadcast it to each server. - }; - - /// Allows more optimal JOIN for typical cases. - enum class Strictness - { - Unspecified, - RightAny, /// Old ANY JOIN. If there are many suitable rows in right table, use any from them to join. - Any, /// Semi Join with any value from filtering table. For LEFT JOIN with Any and RightAny are the same. - All, /// If there are many suitable rows to join, use all of them and replicate rows of "left" table (usual semantic of JOIN). - Asof, /// For the last JOIN column, pick the latest value - Semi, /// LEFT or RIGHT. SEMI LEFT JOIN filters left table by values exists in right table. SEMI RIGHT - otherwise. - Anti, /// LEFT or RIGHT. Same as SEMI JOIN but filter values that are NOT exists in other table. - }; - - /// Join method. - enum class Kind - { - Inner, /// Leave only rows that was JOINed. - Left, /// If in "right" table there is no corresponding rows, use default values instead. - Right, - Full, - Cross, /// Direct product. Strictness and condition doesn't matter. - Comma /// Same as direct product. Intended to be converted to INNER JOIN with conditions from WHERE. - }; - - Locality locality = Locality::Unspecified; - Strictness strictness = Strictness::Unspecified; - Kind kind = Kind::Inner; + JoinLocality locality = JoinLocality::Unspecified; + JoinStrictness strictness = JoinStrictness::Unspecified; + JoinKind kind = JoinKind::Inner; /// Condition. One of fields is non-nullptr. ASTPtr using_expression_list; @@ -113,16 +84,6 @@ struct ASTTableJoin : public IAST void updateTreeHashImpl(SipHash & hash_state) const override; }; -inline constexpr bool isLeft(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Left; } -inline constexpr bool isRight(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Right; } -inline constexpr bool isInner(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Inner; } -inline constexpr bool isFull(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Full; } -inline constexpr bool isCrossOrComma(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Comma || kind == ASTTableJoin::Kind::Cross; } -inline constexpr bool isRightOrFull(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Right || kind == ASTTableJoin::Kind::Full; } -inline constexpr bool isLeftOrFull(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Left || kind == ASTTableJoin::Kind::Full; } -inline constexpr bool isInnerOrRight(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Inner || kind == ASTTableJoin::Kind::Right; } -inline constexpr bool isInnerOrLeft(ASTTableJoin::Kind kind) { return kind == ASTTableJoin::Kind::Inner || kind == ASTTableJoin::Kind::Left; } - /// Specification of ARRAY JOIN. struct ASTArrayJoin : public IAST { diff --git a/src/Parsers/ParserTablesInSelectQuery.cpp b/src/Parsers/ParserTablesInSelectQuery.cpp index 88251dbd3c8..8137093b990 100644 --- a/src/Parsers/ParserTablesInSelectQuery.cpp +++ b/src/Parsers/ParserTablesInSelectQuery.cpp @@ -109,15 +109,15 @@ bool ParserArrayJoin::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) void ParserTablesInSelectQueryElement::parseJoinStrictness(Pos & pos, ASTTableJoin & table_join) { if (ParserKeyword("ANY").ignore(pos)) - table_join.strictness = ASTTableJoin::Strictness::Any; + table_join.strictness = JoinStrictness::Any; else if (ParserKeyword("ALL").ignore(pos)) - table_join.strictness = ASTTableJoin::Strictness::All; + table_join.strictness = JoinStrictness::All; else if (ParserKeyword("ASOF").ignore(pos)) - table_join.strictness = ASTTableJoin::Strictness::Asof; + table_join.strictness = JoinStrictness::Asof; else if (ParserKeyword("SEMI").ignore(pos)) - table_join.strictness = ASTTableJoin::Strictness::Semi; + table_join.strictness = JoinStrictness::Semi; else if (ParserKeyword("ANTI").ignore(pos) || ParserKeyword("ONLY").ignore(pos)) - table_join.strictness = ASTTableJoin::Strictness::Anti; + table_join.strictness = JoinStrictness::Anti; } bool ParserTablesInSelectQueryElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) @@ -139,31 +139,31 @@ bool ParserTablesInSelectQueryElement::parseImpl(Pos & pos, ASTPtr & node, Expec if (pos->type == TokenType::Comma) { ++pos; - table_join->kind = ASTTableJoin::Kind::Comma; + table_join->kind = JoinKind::Comma; } else { if (ParserKeyword("GLOBAL").ignore(pos)) - table_join->locality = ASTTableJoin::Locality::Global; + table_join->locality = JoinLocality::Global; else if (ParserKeyword("LOCAL").ignore(pos)) - table_join->locality = ASTTableJoin::Locality::Local; + table_join->locality = JoinLocality::Local; - table_join->strictness = ASTTableJoin::Strictness::Unspecified; + table_join->strictness = JoinStrictness::Unspecified; /// Legacy: allow JOIN type before JOIN kind parseJoinStrictness(pos, *table_join); bool no_kind = false; if (ParserKeyword("INNER").ignore(pos)) - table_join->kind = ASTTableJoin::Kind::Inner; + table_join->kind = JoinKind::Inner; else if (ParserKeyword("LEFT").ignore(pos)) - table_join->kind = ASTTableJoin::Kind::Left; + table_join->kind = JoinKind::Left; else if (ParserKeyword("RIGHT").ignore(pos)) - table_join->kind = ASTTableJoin::Kind::Right; + table_join->kind = JoinKind::Right; else if (ParserKeyword("FULL").ignore(pos)) - table_join->kind = ASTTableJoin::Kind::Full; + table_join->kind = JoinKind::Full; else if (ParserKeyword("CROSS").ignore(pos)) - table_join->kind = ASTTableJoin::Kind::Cross; + table_join->kind = JoinKind::Cross; else no_kind = true; @@ -171,9 +171,9 @@ bool ParserTablesInSelectQueryElement::parseImpl(Pos & pos, ASTPtr & node, Expec parseJoinStrictness(pos, *table_join); /// Optional OUTER keyword for outer joins. - if (table_join->kind == ASTTableJoin::Kind::Left - || table_join->kind == ASTTableJoin::Kind::Right - || table_join->kind == ASTTableJoin::Kind::Full) + if (table_join->kind == JoinKind::Left + || table_join->kind == JoinKind::Right + || table_join->kind == JoinKind::Full) { ParserKeyword("OUTER").ignore(pos); } @@ -181,19 +181,19 @@ bool ParserTablesInSelectQueryElement::parseImpl(Pos & pos, ASTPtr & node, Expec if (no_kind) { /// Use INNER by default as in another DBMS. - if (table_join->strictness == ASTTableJoin::Strictness::Semi || - table_join->strictness == ASTTableJoin::Strictness::Anti) - table_join->kind = ASTTableJoin::Kind::Left; + if (table_join->strictness == JoinStrictness::Semi || + table_join->strictness == JoinStrictness::Anti) + table_join->kind = JoinKind::Left; else - table_join->kind = ASTTableJoin::Kind::Inner; + table_join->kind = JoinKind::Inner; } - if (table_join->strictness != ASTTableJoin::Strictness::Unspecified - && table_join->kind == ASTTableJoin::Kind::Cross) + if (table_join->strictness != JoinStrictness::Unspecified + && table_join->kind == JoinKind::Cross) throw Exception("You must not specify ANY or ALL for CROSS JOIN.", ErrorCodes::SYNTAX_ERROR); - if ((table_join->strictness == ASTTableJoin::Strictness::Semi || table_join->strictness == ASTTableJoin::Strictness::Anti) && - (table_join->kind != ASTTableJoin::Kind::Left && table_join->kind != ASTTableJoin::Kind::Right)) + if ((table_join->strictness == JoinStrictness::Semi || table_join->strictness == JoinStrictness::Anti) && + (table_join->kind != JoinKind::Left && table_join->kind != JoinKind::Right)) throw Exception("SEMI|ANTI JOIN should be LEFT or RIGHT.", ErrorCodes::SYNTAX_ERROR); if (!ParserKeyword("JOIN").ignore(pos, expected)) @@ -203,8 +203,8 @@ bool ParserTablesInSelectQueryElement::parseImpl(Pos & pos, ASTPtr & node, Expec if (!ParserTableExpression().parse(pos, res->table_expression, expected)) return false; - if (table_join->kind != ASTTableJoin::Kind::Comma - && table_join->kind != ASTTableJoin::Kind::Cross) + if (table_join->kind != JoinKind::Comma + && table_join->kind != JoinKind::Cross) { if (ParserKeyword("USING").ignore(pos, expected)) { diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index 680d158ecaf..1c4760504b4 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -220,16 +220,16 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes if (auto * join = typeid_cast(child.get())) { - auto join_push_down = [&](ASTTableJoin::Kind kind) -> size_t + auto join_push_down = [&](JoinKind kind) -> size_t { const auto & table_join = join->getJoin()->getTableJoin(); /// Only inner and left(/right) join are supported. Other types may generate default values for left table keys. /// So, if we push down a condition like `key != 0`, not all rows may be filtered. - if (table_join.kind() != ASTTableJoin::Kind::Inner && table_join.kind() != kind) + if (table_join.kind() != JoinKind::Inner && table_join.kind() != kind) return 0; - bool is_left = kind == ASTTableJoin::Kind::Left; + bool is_left = kind == JoinKind::Left; const auto & input_header = is_left ? join->getInputStreams().front().header : join->getInputStreams().back().header; const auto & res_header = join->getOutputStream().header; Names allowed_keys; @@ -258,13 +258,13 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes return updated_steps; }; - if (size_t updated_steps = join_push_down(ASTTableJoin::Kind::Left)) + if (size_t updated_steps = join_push_down(JoinKind::Left)) return updated_steps; /// For full sorting merge join we push down both to the left and right tables, because left and right streams are not independent. if (join->allowPushDownToRight()) { - if (size_t updated_steps = join_push_down(ASTTableJoin::Kind::Right)) + if (size_t updated_steps = join_push_down(JoinKind::Right)) return updated_steps; } } diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp index c7b7afab541..6f842bec939 100644 --- a/src/Processors/Transforms/MergeJoinTransform.cpp +++ b/src/Processors/Transforms/MergeJoinTransform.cpp @@ -30,8 +30,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -using JoinKind = ASTTableJoin::Kind; - namespace { @@ -281,7 +279,7 @@ MergeJoinAlgorithm::MergeJoinAlgorithm( throw Exception("MergeJoinAlgorithm requires exactly two inputs", ErrorCodes::LOGICAL_ERROR); auto strictness = table_join->getTableJoin().strictness(); - if (strictness != ASTTableJoin::Strictness::Any && strictness != ASTTableJoin::Strictness::All) + if (strictness != JoinStrictness::Any && strictness != JoinStrictness::All) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm is not implemented for strictness {}", strictness); auto kind = table_join->getTableJoin().kind(); @@ -826,10 +824,10 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge() auto strictness = table_join->getTableJoin().strictness(); - if (strictness == ASTTableJoin::Strictness::Any) + if (strictness == JoinStrictness::Any) return anyJoin(kind); - if (strictness == ASTTableJoin::Strictness::All) + if (strictness == JoinStrictness::All) return allJoin(kind); throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported strictness '{}'", strictness); diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h index 9f60eafb455..58ac652cb18 100644 --- a/src/Processors/Transforms/MergeJoinTransform.h +++ b/src/Processors/Transforms/MergeJoinTransform.h @@ -245,10 +245,10 @@ public: private: std::optional handleAnyJoinState(); - Status anyJoin(ASTTableJoin::Kind kind); + Status anyJoin(JoinKind kind); std::optional handleAllJoinState(); - Status allJoin(ASTTableJoin::Kind kind); + Status allJoin(JoinKind kind); Chunk createBlockWithDefaults(size_t source_num); Chunk createBlockWithDefaults(size_t source_num, size_t start, size_t num_rows) const; diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index 5e161fc2e6a..c94fd3b2256 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -45,8 +45,8 @@ StorageJoin::StorageJoin( const Names & key_names_, bool use_nulls_, SizeLimits limits_, - ASTTableJoin::Kind kind_, - ASTTableJoin::Strictness strictness_, + JoinKind kind_, + JoinStrictness strictness_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, const String & comment, @@ -279,8 +279,8 @@ void registerStorageJoin(StorageFactory & factory) "Storage Join requires at least 3 parameters: Join(ANY|ALL|SEMI|ANTI, LEFT|INNER|RIGHT, keys...).", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - ASTTableJoin::Strictness strictness = ASTTableJoin::Strictness::Unspecified; - ASTTableJoin::Kind kind = ASTTableJoin::Kind::Comma; + JoinStrictness strictness = JoinStrictness::Unspecified; + JoinKind kind = JoinKind::Comma; if (auto opt_strictness_id = tryGetIdentifierName(engine_args[0])) { @@ -289,19 +289,19 @@ void registerStorageJoin(StorageFactory & factory) if (strictness_str == "any") { if (old_any_join) - strictness = ASTTableJoin::Strictness::RightAny; + strictness = JoinStrictness::RightAny; else - strictness = ASTTableJoin::Strictness::Any; + strictness = JoinStrictness::Any; } else if (strictness_str == "all") - strictness = ASTTableJoin::Strictness::All; + strictness = JoinStrictness::All; else if (strictness_str == "semi") - strictness = ASTTableJoin::Strictness::Semi; + strictness = JoinStrictness::Semi; else if (strictness_str == "anti") - strictness = ASTTableJoin::Strictness::Anti; + strictness = JoinStrictness::Anti; } - if (strictness == ASTTableJoin::Strictness::Unspecified) + if (strictness == JoinStrictness::Unspecified) throw Exception("First parameter of storage Join must be ANY or ALL or SEMI or ANTI (without quotes).", ErrorCodes::BAD_ARGUMENTS); @@ -310,20 +310,20 @@ void registerStorageJoin(StorageFactory & factory) const String kind_str = Poco::toLower(*opt_kind_id); if (kind_str == "left") - kind = ASTTableJoin::Kind::Left; + kind = JoinKind::Left; else if (kind_str == "inner") - kind = ASTTableJoin::Kind::Inner; + kind = JoinKind::Inner; else if (kind_str == "right") - kind = ASTTableJoin::Kind::Right; + kind = JoinKind::Right; else if (kind_str == "full") { - if (strictness == ASTTableJoin::Strictness::Any) - strictness = ASTTableJoin::Strictness::RightAny; - kind = ASTTableJoin::Kind::Full; + if (strictness == JoinStrictness::Any) + strictness = JoinStrictness::RightAny; + kind = JoinKind::Full; } } - if (kind == ASTTableJoin::Kind::Comma) + if (kind == JoinKind::Comma) throw Exception("Second parameter of storage Join must be LEFT or INNER or RIGHT or FULL (without quotes).", ErrorCodes::BAD_ARGUMENTS); @@ -444,7 +444,7 @@ private: std::unique_ptr> position; /// type erasure - template + template Chunk createChunk(const Maps & maps) { MutableColumns mut_columns = restored_block.cloneEmpty().mutateColumns(); @@ -491,7 +491,7 @@ private: return Chunk(std::move(columns), num_rows); } - template + template size_t fillColumns(const Map & map, MutableColumns & columns) { size_t rows_added = 0; @@ -506,33 +506,33 @@ private: for (; it != end; ++it) { - if constexpr (STRICTNESS == ASTTableJoin::Strictness::RightAny) + if constexpr (STRICTNESS == JoinStrictness::RightAny) { fillOne(columns, column_indices, it, key_pos, rows_added); } - else if constexpr (STRICTNESS == ASTTableJoin::Strictness::All) + else if constexpr (STRICTNESS == JoinStrictness::All) { fillAll(columns, column_indices, it, key_pos, rows_added); } - else if constexpr (STRICTNESS == ASTTableJoin::Strictness::Any) + else if constexpr (STRICTNESS == JoinStrictness::Any) { - if constexpr (KIND == ASTTableJoin::Kind::Left || KIND == ASTTableJoin::Kind::Inner) + if constexpr (KIND == JoinKind::Left || KIND == JoinKind::Inner) fillOne(columns, column_indices, it, key_pos, rows_added); - else if constexpr (KIND == ASTTableJoin::Kind::Right) + else if constexpr (KIND == JoinKind::Right) fillAll(columns, column_indices, it, key_pos, rows_added); } - else if constexpr (STRICTNESS == ASTTableJoin::Strictness::Semi) + else if constexpr (STRICTNESS == JoinStrictness::Semi) { - if constexpr (KIND == ASTTableJoin::Kind::Left) + if constexpr (KIND == JoinKind::Left) fillOne(columns, column_indices, it, key_pos, rows_added); - else if constexpr (KIND == ASTTableJoin::Kind::Right) + else if constexpr (KIND == JoinKind::Right) fillAll(columns, column_indices, it, key_pos, rows_added); } - else if constexpr (STRICTNESS == ASTTableJoin::Strictness::Anti) + else if constexpr (STRICTNESS == JoinStrictness::Anti) { - if constexpr (KIND == ASTTableJoin::Kind::Left) + if constexpr (KIND == JoinKind::Left) fillOne(columns, column_indices, it, key_pos, rows_added); - else if constexpr (KIND == ASTTableJoin::Kind::Right) + else if constexpr (KIND == JoinKind::Right) fillAll(columns, column_indices, it, key_pos, rows_added); } else diff --git a/src/Storages/StorageJoin.h b/src/Storages/StorageJoin.h index acb127c5a2c..a75fe944a34 100644 --- a/src/Storages/StorageJoin.h +++ b/src/Storages/StorageJoin.h @@ -31,8 +31,8 @@ public: const Names & key_names_, bool use_nulls_, SizeLimits limits_, - ASTTableJoin::Kind kind_, - ASTTableJoin::Strictness strictness_, + JoinKind kind_, + JoinStrictness strictness_, const ColumnsDescription & columns_, const ConstraintsDescription & constraints_, const String & comment, @@ -94,8 +94,8 @@ private: const Names key_names; bool use_nulls; SizeLimits limits; - ASTTableJoin::Kind kind; /// LEFT | INNER ... - ASTTableJoin::Strictness strictness; /// ANY | ALL + JoinKind kind; /// LEFT | INNER ... + JoinStrictness strictness; /// ANY | ALL bool overwrite; std::shared_ptr table_join; From 453e59aae8db54447de2c3bda74ef75d419effcf Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 29 Jul 2022 16:35:43 +0000 Subject: [PATCH 096/139] fix flaky test --- .../0_stateless/02360_send_logs_level_colors.reference | 1 - tests/queries/0_stateless/02360_send_logs_level_colors.sh | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02360_send_logs_level_colors.reference b/tests/queries/0_stateless/02360_send_logs_level_colors.reference index fe2824243c4..1c30d50f5c0 100644 --- a/tests/queries/0_stateless/02360_send_logs_level_colors.reference +++ b/tests/queries/0_stateless/02360_send_logs_level_colors.reference @@ -1,3 +1,2 @@ ASCII text ASCII text -ASCII text diff --git a/tests/queries/0_stateless/02360_send_logs_level_colors.sh b/tests/queries/0_stateless/02360_send_logs_level_colors.sh index 4e5ce057702..eaa294cebe4 100755 --- a/tests/queries/0_stateless/02360_send_logs_level_colors.sh +++ b/tests/queries/0_stateless/02360_send_logs_level_colors.sh @@ -26,6 +26,8 @@ EOF run "$CLICKHOUSE_CLIENT -q 'SELECT 1' 2>$file_name" run "$CLICKHOUSE_CLIENT -q 'SELECT 1' --server_logs_file=$file_name" -run "$CLICKHOUSE_CLIENT -q 'SELECT 1' --server_logs_file=- >$file_name" + +# This query may fail due to bug in clickhouse-client. +# run "$CLICKHOUSE_CLIENT -q 'SELECT 1' --server_logs_file=- >$file_name" rm -f "$file_name" From 117fda2d83b525dad971239a03e2f5731142d1d6 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 29 Jul 2022 19:10:43 +0200 Subject: [PATCH 097/139] Update gtest_ast_deleter.cpp --- src/Parsers/tests/gtest_ast_deleter.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Parsers/tests/gtest_ast_deleter.cpp b/src/Parsers/tests/gtest_ast_deleter.cpp index 2a94d6a3a20..ddbbb960929 100644 --- a/src/Parsers/tests/gtest_ast_deleter.cpp +++ b/src/Parsers/tests/gtest_ast_deleter.cpp @@ -2,11 +2,6 @@ #include #include -namespace DB::ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - struct StackDecrementer { size_t & depth; From 8ba42876fc02a1e626755e7ad8c3c4c7e3467451 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 29 Jul 2022 19:35:15 +0200 Subject: [PATCH 098/139] Fixed build --- src/Core/Joins.cpp | 24 ++++++++++++++++++++++++ src/Core/Joins.h | 22 ++++++++++++++++++++++ src/Core/SettingsEnums.h | 12 ------------ src/Interpreters/JoinedTables.cpp | 2 +- src/Interpreters/MergeJoin.cpp | 6 +++--- src/Interpreters/TableJoin.h | 7 ------- 6 files changed, 50 insertions(+), 23 deletions(-) diff --git a/src/Core/Joins.cpp b/src/Core/Joins.cpp index 8c395d4d5cc..1cd7215335f 100644 --- a/src/Core/Joins.cpp +++ b/src/Core/Joins.cpp @@ -52,4 +52,28 @@ const char * toString(ASOFJoinInequality asof_join_inequality) } } +const char * toString(JoinAlgorithm join_algorithm) +{ + switch (join_algorithm) + { + case JoinAlgorithm::DEFAULT: return "DEFAULT"; + case JoinAlgorithm::AUTO: return "AUTO"; + case JoinAlgorithm::HASH: return "HASH"; + case JoinAlgorithm::PARTIAL_MERGE: return "PARTIAL_MERGE"; + case JoinAlgorithm::PREFER_PARTIAL_MERGE: return "PREFER_PARTIAL_MERGE"; + case JoinAlgorithm::PARALLEL_HASH: return "PARALLEL_HASH"; + case JoinAlgorithm::DIRECT: return "DIRECT"; + case JoinAlgorithm::FULL_SORTING_MERGE: return "FULL_SORTING_MERGE"; + } +} + +const char * toString(JoinTableSide join_table_side) +{ + switch (join_table_side) + { + case JoinTableSide::Left: return "LEFT"; + case JoinTableSide::Right: return "RIGHT"; + } +} + } diff --git a/src/Core/Joins.h b/src/Core/Joins.h index 41529e9888f..9a3a1684910 100644 --- a/src/Core/Joins.h +++ b/src/Core/Joins.h @@ -92,4 +92,26 @@ inline constexpr ASOFJoinInequality reverseASOFJoinInequality(ASOFJoinInequality return ASOFJoinInequality::None; } +enum class JoinAlgorithm +{ + DEFAULT = 0, + AUTO, + HASH, + PARTIAL_MERGE, + PREFER_PARTIAL_MERGE, + PARALLEL_HASH, + DIRECT, + FULL_SORTING_MERGE, +}; + +const char * toString(JoinAlgorithm join_algorithm); + +enum class JoinTableSide +{ + Left, + Right +}; + +const char * toString(JoinTableSide join_table_side); + } diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 18b5323dea5..308d53ff690 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -30,18 +30,6 @@ DECLARE_SETTING_ENUM(LoadBalancing) DECLARE_SETTING_ENUM(JoinStrictness) -enum class JoinAlgorithm -{ - DEFAULT = 0, - AUTO, - HASH, - PARTIAL_MERGE, - PREFER_PARTIAL_MERGE, - PARALLEL_HASH, - DIRECT, - FULL_SORTING_MERGE, -}; - DECLARE_SETTING_MULTI_ENUM(JoinAlgorithm) diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index fd47c3d747f..74b54ef537f 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -70,7 +70,7 @@ void replaceJoinedTable(const ASTSelectQuery & select_query) * ANY join behavior can also be different with this optimization, * but it's ok because we don't guarantee which row to choose for ANY, unlike ASOF, where we have to pick the closest one. */ - if (table_join.strictness == ASTTableJoin::Strictness::Asof) + if (table_join.strictness == JoinStrictness::Asof) return; auto & table_expr = join->table_expression->as(); diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index 77091a0a6d7..655d50355f9 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -1140,9 +1140,9 @@ bool MergeJoin::isSupported(const std::shared_ptr & table_join) auto kind = table_join->kind(); auto strictness = table_join->strictness(); - bool is_any = (strictness == ASTTableJoin::Strictness::Any); - bool is_all = (strictness == ASTTableJoin::Strictness::All); - bool is_semi = (strictness == ASTTableJoin::Strictness::Semi); + bool is_any = (strictness == JoinStrictness::Any); + bool is_all = (strictness == JoinStrictness::All); + bool is_semi = (strictness == JoinStrictness::Semi); bool all_join = is_all && (isInner(kind) || isLeft(kind) || isRight(kind) || isFull(kind)); bool special_left = isInnerOrLeft(kind) && (is_any || is_semi); diff --git a/src/Interpreters/TableJoin.h b/src/Interpreters/TableJoin.h index dea82550f77..3bb3b00416c 100644 --- a/src/Interpreters/TableJoin.h +++ b/src/Interpreters/TableJoin.h @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -42,12 +41,6 @@ struct Settings; class IVolume; using VolumePtr = std::shared_ptr; -enum class JoinTableSide -{ - Left, - Right -}; - class TableJoin { public: From be656f9dfa3c8dfdb7b125454d4717efb660740b Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Fri, 29 Jul 2022 19:38:00 +0200 Subject: [PATCH 099/139] Update 02376_arrow_dict_with_string.sql --- tests/queries/0_stateless/02376_arrow_dict_with_string.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02376_arrow_dict_with_string.sql b/tests/queries/0_stateless/02376_arrow_dict_with_string.sql index c1be023e8e6..307498f7b92 100644 --- a/tests/queries/0_stateless/02376_arrow_dict_with_string.sql +++ b/tests/queries/0_stateless/02376_arrow_dict_with_string.sql @@ -1,4 +1,4 @@ --- Tags: no-parallel +-- Tags: no-parallel, no-fasttest insert into function file(02376_data.arrow) select toLowCardinality(toString(number)) as x from numbers(10) settings output_format_arrow_string_as_string=1, output_format_arrow_low_cardinality_as_dictionary=1, engine_file_truncate_on_insert=1; desc file (02376_data.arrow); select * from file(02376_data.arrow); From 6099f66fd613214f2a8b90c96ddce42625bb709e Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Fri, 29 Jul 2022 19:55:50 +0200 Subject: [PATCH 100/139] fixed type --- src/Parsers/tests/gtest_ast_deleter.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Parsers/tests/gtest_ast_deleter.cpp b/src/Parsers/tests/gtest_ast_deleter.cpp index ddbbb960929..c6913f8ef8f 100644 --- a/src/Parsers/tests/gtest_ast_deleter.cpp +++ b/src/Parsers/tests/gtest_ast_deleter.cpp @@ -58,12 +58,12 @@ TEST(ASTDeleter, SimpleChain) { size_t depth = 0; size_t max_depth = 5; - size_t chain_lenght = 10; + size_t chain_length = 10; { DB::ASTPtr ast = std::make_shared(DB::ASTs{}, depth, max_depth); - for (size_t i = 0; i < chain_lenght; ++i) + for (size_t i = 0; i < chain_length; ++i) ast = std::make_shared(DB::ASTs{std::move(ast)}, depth, max_depth); } } @@ -72,12 +72,12 @@ TEST(ASTDeleter, SimpleChainLong) { size_t depth = 0; size_t max_depth = 5; - size_t chain_lenght = 100000; + size_t chain_length = 100000; { DB::ASTPtr ast = std::make_shared(DB::ASTs{}, depth, max_depth); - for (size_t i = 0; i < chain_lenght; ++i) + for (size_t i = 0; i < chain_length; ++i) ast = std::make_shared(DB::ASTs{std::move(ast)}, depth, max_depth); } } @@ -87,12 +87,12 @@ TEST(ASTDeleter, ChainWithExtraMember) size_t depth = 0; size_t max_depth = 5; size_t member_depth = 10; - size_t chain_lenght = 100; + size_t chain_length = 100; { DB::ASTPtr ast = std::make_shared(DB::ASTs{}, depth, max_depth); - for (size_t i = 0; i < chain_lenght; ++i) + for (size_t i = 0; i < chain_length; ++i) { ast = std::make_shared(DB::ASTs{std::move(ast)}, depth, max_depth); if (i > member_depth) @@ -110,13 +110,13 @@ TEST(ASTDeleter, DoubleChain) { size_t depth = 0; size_t max_depth = 5; - size_t chain_lenght = 10; + size_t chain_length = 10; { DB::ASTPtr ast1 = std::make_shared(DB::ASTs{}, depth, max_depth); DB::ASTPtr ast2 = std::make_shared(DB::ASTs{}, depth, max_depth); - for (size_t i = 0; i < chain_lenght; ++i) + for (size_t i = 0; i < chain_length; ++i) { ast1 = std::make_shared(DB::ASTs{std::move(ast1)}, depth, max_depth); ast2 = std::make_shared(DB::ASTs{std::move(ast2)}, depth, max_depth); @@ -130,13 +130,13 @@ TEST(ASTDeleter, DoubleChainLong) { size_t depth = 0; size_t max_depth = 5; - size_t chain_lenght = 100000; + size_t chain_length = 100000; { DB::ASTPtr ast1 = std::make_shared(DB::ASTs{}, depth, max_depth); DB::ASTPtr ast2 = std::make_shared(DB::ASTs{}, depth, max_depth); - for (size_t i = 0; i < chain_lenght; ++i) + for (size_t i = 0; i < chain_length; ++i) { ast1 = std::make_shared(DB::ASTs{std::move(ast1)}, depth, max_depth); ast2 = std::make_shared(DB::ASTs{std::move(ast2)}, depth, max_depth); From 942f056ce57068d0397da55cb894ce095a92565e Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Fri, 29 Jul 2022 19:10:23 +0000 Subject: [PATCH 101/139] fix redirecting of logs to stdout in client --- src/Client/ClientBase.cpp | 9 +++++++-- .../0_stateless/02360_send_logs_level_colors.reference | 1 + .../queries/0_stateless/02360_send_logs_level_colors.sh | 4 +--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index faef19fe1a3..977d2bca01f 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -583,9 +583,14 @@ try if (has_vertical_output_suffix) current_format = "Vertical"; - /// It is not clear how to write progress intermixed with data with parallel formatting. + bool logs_into_stdout = server_logs_file == "-"; + bool extras_into_stdout = need_render_progress || logs_into_stdout; + bool select_only_into_file = select_into_file && !select_into_file_and_stdout; + + /// It is not clear how to write progress and logs + /// intermixed with data with parallel formatting. /// It may increase code complexity significantly. - if (!need_render_progress || (select_into_file && !select_into_file_and_stdout)) + if (!extras_into_stdout || select_only_into_file) output_format = global_context->getOutputFormatParallelIfPossible( current_format, out_file_buf ? *out_file_buf : *out_buf, block); else diff --git a/tests/queries/0_stateless/02360_send_logs_level_colors.reference b/tests/queries/0_stateless/02360_send_logs_level_colors.reference index 1c30d50f5c0..fe2824243c4 100644 --- a/tests/queries/0_stateless/02360_send_logs_level_colors.reference +++ b/tests/queries/0_stateless/02360_send_logs_level_colors.reference @@ -1,2 +1,3 @@ ASCII text ASCII text +ASCII text diff --git a/tests/queries/0_stateless/02360_send_logs_level_colors.sh b/tests/queries/0_stateless/02360_send_logs_level_colors.sh index eaa294cebe4..4e5ce057702 100755 --- a/tests/queries/0_stateless/02360_send_logs_level_colors.sh +++ b/tests/queries/0_stateless/02360_send_logs_level_colors.sh @@ -26,8 +26,6 @@ EOF run "$CLICKHOUSE_CLIENT -q 'SELECT 1' 2>$file_name" run "$CLICKHOUSE_CLIENT -q 'SELECT 1' --server_logs_file=$file_name" - -# This query may fail due to bug in clickhouse-client. -# run "$CLICKHOUSE_CLIENT -q 'SELECT 1' --server_logs_file=- >$file_name" +run "$CLICKHOUSE_CLIENT -q 'SELECT 1' --server_logs_file=- >$file_name" rm -f "$file_name" From 8653562f52349e13a33ec1af4b57cf428aa69bb2 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 29 Jul 2022 23:34:25 +0200 Subject: [PATCH 102/139] Fixed build --- src/Core/Joins.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Core/Joins.h b/src/Core/Joins.h index 9a3a1684910..7c91c5a5c16 100644 --- a/src/Core/Joins.h +++ b/src/Core/Joins.h @@ -1,5 +1,7 @@ #pragma once +#include + namespace DB { From c0d7b6efc388f10388b4cb6fcbbf5dc40c70f980 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 30 Jul 2022 01:45:06 +0200 Subject: [PATCH 103/139] Suggestions from @tavplubix --- programs/server/config.xml | 5 ++++- src/Common/StackTrace.cpp | 30 ++++++++++++++++++++---------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index c49d2a83499..1060cb3db0a 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -1389,7 +1389,10 @@ - true + false diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp index 01d8bccc1f5..70f80b62868 100644 --- a/src/Common/StackTrace.cpp +++ b/src/Common/StackTrace.cpp @@ -25,6 +25,18 @@ namespace /// Currently this variable is set up once on server startup. /// But we use atomic just in case, so it is possible to be modified at runtime. std::atomic show_addresses = true; + + bool shouldShowAddress(const void * addr) + { + /// If the address is less than 4096, most likely it is a nullptr dereference with offset, + /// and showing this offset is secure nevertheless. + /// NOTE: 4096 is the page size on x86 and it can be different on other systems, + /// but for the purpose of this branch, it does not matter. + if (reinterpret_cast(addr) < 4096) + return true; + + return show_addresses.load(std::memory_order_relaxed); + } } void StackTrace::setShowAddresses(bool show) @@ -44,7 +56,7 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, [[maybe_unused /// Print info about address and reason. if (nullptr == info.si_addr) error << "Address: NULL pointer."; - else if (show_addresses.load(std::memory_order_relaxed)) + else if (shouldShowAddress(info.si_addr)) error << "Address: " << info.si_addr; #if defined(__x86_64__) && !defined(OS_FREEBSD) && !defined(OS_DARWIN) && !defined(__arm__) && !defined(__powerpc__) @@ -386,7 +398,7 @@ static void toStringEveryLineImpl( else out << "?"; - if (show_addresses.load(std::memory_order_relaxed)) + if (shouldShowAddress(physical_addr)) out << " @ " << physical_addr; out << " in " << (object ? object->name : "?"); @@ -403,22 +415,20 @@ static void toStringEveryLineImpl( out.str({}); } #else - if (show_addresses.load(std::memory_order_relaxed)) - { - std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - out.exceptions(std::ios::failbit); + std::stringstream out; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + out.exceptions(std::ios::failbit); - for (size_t i = offset; i < size; ++i) + for (size_t i = offset; i < size; ++i) + { + const void * addr = frame_pointers[i]; + if (shouldShowAddress(addr)) { - const void * addr = frame_pointers[i]; out << i << ". " << addr; callback(out.str()); out.str({}); } } - else - callback("Addresses are hidden for security reasons."); #endif } From 9adfd259d05bfbae03f934d531895a60e99ba6da Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 30 Jul 2022 01:50:45 +0200 Subject: [PATCH 104/139] Support in clickhouse-local --- programs/local/LocalServer.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 4f3b92bbcf0..1a93e43c730 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -352,6 +352,9 @@ try { UseSSL use_ssl; ThreadStatus thread_status; + + StackTrace::setShowAddresses(config().getBool("show_addresses_in_stack_traces", true)); + setupSignalHandler(); std::cout << std::fixed << std::setprecision(3); From 87d513f7990033302af968a484961074310033c2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 30 Jul 2022 01:57:29 +0200 Subject: [PATCH 105/139] Add a test --- .../02375_stack_trace_no_addresses.reference | 2 + .../02375_stack_trace_no_addresses.sh | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 tests/queries/0_stateless/02375_stack_trace_no_addresses.reference create mode 100755 tests/queries/0_stateless/02375_stack_trace_no_addresses.sh diff --git a/tests/queries/0_stateless/02375_stack_trace_no_addresses.reference b/tests/queries/0_stateless/02375_stack_trace_no_addresses.reference new file mode 100644 index 00000000000..0d66ea1aee9 --- /dev/null +++ b/tests/queries/0_stateless/02375_stack_trace_no_addresses.reference @@ -0,0 +1,2 @@ +0 +1 diff --git a/tests/queries/0_stateless/02375_stack_trace_no_addresses.sh b/tests/queries/0_stateless/02375_stack_trace_no_addresses.sh new file mode 100755 index 00000000000..8faa698cf01 --- /dev/null +++ b/tests/queries/0_stateless/02375_stack_trace_no_addresses.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +CONFIG="${CLICKHOUSE_TMP}/config.xml" + +echo " + + false + + + + + + + + ::/0 + + default + default + + + + + + +" > "${CONFIG}" + +${CLICKHOUSE_LOCAL} --query "SELECT throwIf(1)" --stacktrace --config-file "${CONFIG}" 2>&1 | grep -c -F '@ 0x' + +sed -i 's/false/true/' "${CONFIG}" + +${CLICKHOUSE_LOCAL} --query "SELECT throwIf(1)" --stacktrace --config-file "${CONFIG}" 2>&1 | grep -c -F '@ 0x' | grep -c -v '^0$' + +rm "${CONFIG}" From 628503c92a226703b81e7a42a8294e5429389698 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Sat, 30 Jul 2022 12:15:55 +0000 Subject: [PATCH 106/139] Use different root path for total-queue test --- .../src/jepsen/clickhouse_keeper/queue.clj | 11 +++++++---- .../src/jepsen/clickhouse_keeper/utils.clj | 12 ++++++++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/queue.clj b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/queue.clj index 3aeaa328606..1c306a0ff04 100644 --- a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/queue.clj +++ b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/queue.clj @@ -11,6 +11,7 @@ [zookeeper :as zk]) (:import (org.apache.zookeeper ZooKeeper KeeperException KeeperException$BadVersionException))) +(def root-path "/queue") (defn enqueue [val _ _] {:type :invoke, :f :enqueue :value val}) (defn dequeue [_ _] {:type :invoke, :f :dequeue}) @@ -22,18 +23,20 @@ :conn (zk-connect node 9181 30000)) :nodename node)) - (setup! [this test]) + (setup! [this test] + (exec-with-retries 30 (fn [] + (zk-create-if-not-exists conn root-path "")))) (invoke! [this test op] (case (:f op) :enqueue (try (do - (zk-create-if-not-exists conn (str "/" (:value op)) "") + (zk-create conn (concat-path root-path (:value op)) "") (assoc op :type :ok)) (catch Exception _ (assoc op :type :info, :error :connect-error))) :dequeue (try - (let [result (zk-multi-delete-first-child conn "/")] + (let [result (zk-multi-delete-first-child conn root-path)] (if (not (nil? result)) (assoc op :type :ok :value result) (assoc op :type :fail :value result))) @@ -42,7 +45,7 @@ ; drain via delete is to long, just list all nodes (exec-with-retries 30 (fn [] (zk-sync conn) - (assoc op :type :ok :value (into #{} (map #(str %1) (zk-list conn "/")))))))) + (assoc op :type :ok :value (into #{} (map #(str %1) (zk-list conn root-path)))))))) (teardown! [_ test]) diff --git a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/utils.clj b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/utils.clj index f377d91f5fe..596458a8f3e 100644 --- a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/utils.clj +++ b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/utils.clj @@ -88,10 +88,14 @@ new-set (conj current-set elem)] (zk-set conn path (pr-str new-set) (:version (:stat current-value))))) -(defn zk-create-if-not-exists +(defn zk-create [conn path data] (zk/create conn path :data (data/to-bytes (str data)) :persistent? true)) +(defn zk-create-if-not-exists + [conn path data] + (if-not (zk/exists conn path) (zk-create conn path data))) + (defn zk-create-sequential [conn path-prefix data] (zk/create conn path-prefix :data (data/to-bytes (str data)) :persistent? true :sequential? true)) @@ -119,6 +123,10 @@ (subs path 0 rslash_pos) "/"))) +(defn concat-path + [parent child] + (str parent (if (= parent "/") "" "/") child)) + (defn zk-multi-delete-first-child [conn path] (let [{children :children stat :stat} (zk-list-with-stat conn path) @@ -128,7 +136,7 @@ (try (do (.check txn path (:version stat)) (.setData txn path (data/to-bytes "") -1) ; I'm just checking multitransactions - (.delete txn (str path first-child) -1) + (.delete txn (concat-path path first-child) -1) (.commit txn) first-child) (catch KeeperException$BadVersionException _ nil) From d66c108a0435e06c34f91971a10cb74f5fe5d53e Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 30 Jul 2022 15:36:01 +0300 Subject: [PATCH 107/139] Fix --- src/Interpreters/TreeRewriter.cpp | 29 ++++++++++--------- .../replaceForPositionalArguments.cpp | 5 ++++ .../replaceForPositionalArguments.h | 1 + ..._shard_no_aggregates_and_constant_keys.sql | 2 ++ .../01119_optimize_trivial_insert_select.sql | 1 + .../0_stateless/01414_optimize_any_bug.sql | 1 + .../0_stateless/01798_having_push_down.sql | 1 + 7 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index c93045a0fc1..5b28af6479e 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -1239,6 +1239,21 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( source_columns_set, right_table.table.getQualifiedNamePrefix()); } + translateQualifiedNames(query, *select_query, source_columns_set, tables_with_columns); + + /// Optimizes logical expressions. + LogicalExpressionsOptimizer(select_query, settings.optimize_min_equality_disjunction_chain_length.value).perform(); + + NameSet all_source_columns_set = source_columns_set; + if (table_join) + { + for (const auto & [name, _] : table_join->columns_from_joined_table) + all_source_columns_set.insert(name); + } + + normalize(query, result.aliases, all_source_columns_set, select_options.ignore_alias, settings, /* allow_self_aliases = */ true, getContext()); + + if (getContext()->getSettingsRef().enable_positional_arguments) { if (select_query->groupBy()) @@ -1258,20 +1273,6 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( } } - translateQualifiedNames(query, *select_query, source_columns_set, tables_with_columns); - - /// Optimizes logical expressions. - LogicalExpressionsOptimizer(select_query, settings.optimize_min_equality_disjunction_chain_length.value).perform(); - - NameSet all_source_columns_set = source_columns_set; - if (table_join) - { - for (const auto & [name, _] : table_join->columns_from_joined_table) - all_source_columns_set.insert(name); - } - - normalize(query, result.aliases, all_source_columns_set, select_options.ignore_alias, settings, /* allow_self_aliases = */ true, getContext()); - /// Remove unneeded columns according to 'required_result_columns'. /// Leave all selected columns in case of DISTINCT; columns that contain arrayJoin function inside. /// Must be after 'normalizeTree' (after expanding aliases, for aliases not get lost) diff --git a/src/Interpreters/replaceForPositionalArguments.cpp b/src/Interpreters/replaceForPositionalArguments.cpp index deac5801d7a..410cc5bb10c 100644 --- a/src/Interpreters/replaceForPositionalArguments.cpp +++ b/src/Interpreters/replaceForPositionalArguments.cpp @@ -8,6 +8,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + bool replaceForPositionalArguments(ASTPtr & argument, const ASTSelectQuery * select_query, ASTSelectQuery::Expression expression) { auto columns = select_query->select()->children; diff --git a/src/Interpreters/replaceForPositionalArguments.h b/src/Interpreters/replaceForPositionalArguments.h index 9906978c561..9e238ba32d3 100644 --- a/src/Interpreters/replaceForPositionalArguments.h +++ b/src/Interpreters/replaceForPositionalArguments.h @@ -1,3 +1,4 @@ +#pragma once #include namespace DB diff --git a/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.sql b/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.sql index 48336faf382..93fa1f440b1 100644 --- a/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.sql +++ b/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.sql @@ -1,5 +1,7 @@ -- Tags: shard +set enable_positional_arguments=0; + select 40 as z from (select * from system.numbers limit 3) group by z; select 41 as z from remote('127.0.0.{2,3}', system.one) group by z; select count(), 42 AS z from remote('127.0.0.{2,3}', system.one) group by z; diff --git a/tests/queries/0_stateless/01119_optimize_trivial_insert_select.sql b/tests/queries/0_stateless/01119_optimize_trivial_insert_select.sql index 001e758284f..a53b60a5ad3 100644 --- a/tests/queries/0_stateless/01119_optimize_trivial_insert_select.sql +++ b/tests/queries/0_stateless/01119_optimize_trivial_insert_select.sql @@ -1,6 +1,7 @@ drop table if exists t; create table t(n int, a Int64, s String) engine = MergeTree() order by a; +set enable_positional_arguments=0; set optimize_trivial_insert_select=1; -- due to aggregate functions, optimize_trivial_insert_select will not be applied diff --git a/tests/queries/0_stateless/01414_optimize_any_bug.sql b/tests/queries/0_stateless/01414_optimize_any_bug.sql index 6f6f291c504..ec24a09fc11 100644 --- a/tests/queries/0_stateless/01414_optimize_any_bug.sql +++ b/tests/queries/0_stateless/01414_optimize_any_bug.sql @@ -8,6 +8,7 @@ CREATE TABLE test ENGINE = MergeTree() ORDER BY tuple(); +SET enable_positional_arguments=0; SET optimize_move_functions_out_of_any = 1; SELECT any(arrayFilter((c, d) -> (4 = d), `Source.C1`, `Source.C2`)[1]) AS x diff --git a/tests/queries/0_stateless/01798_having_push_down.sql b/tests/queries/0_stateless/01798_having_push_down.sql index 28b97b7f065..b3a77c8f5b5 100644 --- a/tests/queries/0_stateless/01798_having_push_down.sql +++ b/tests/queries/0_stateless/01798_having_push_down.sql @@ -12,6 +12,7 @@ SELECT c0 + -1, sum(intDivOrZero(intDivOrZero(NULL, NULL), '2'), intDivOrZero(10 SELECT sum(c0 + 257) FROM t_having GROUP BY c0 = -9223372036854775808, NULL, -2147483649, c0 HAVING c0 = -9223372036854775808 SETTINGS enable_optimize_predicate_expression = 0; +SET enable_positional_arguments=0; SELECT c0 + -2, c0 + -9223372036854775807, c0 = NULL FROM t_having GROUP BY c0 = 0.9998999834060669, 1023, c0 HAVING c0 = 0.9998999834060669 SETTINGS enable_optimize_predicate_expression = 0; DROP TABLE t_having; From b9d7cd6a5d7c5b22be2cf2e5e6055313dfd25f14 Mon Sep 17 00:00:00 2001 From: Constantine Peresypkin Date: Thu, 28 Jul 2022 19:54:46 +0200 Subject: [PATCH 108/139] add settings for executable table func SELECT * FROM executable('