thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-27 01:51:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #51481 from ClickHouse/fix-the-same-names-for-lift-up-functions-before-sorting

Browse Source 
Fix optimization to move functions before sorting.
This commit is contained in:
Nikolai Kochetov 2023-07-18 12:34:51 +02:00 committed by GitHub
commit 83e9ec117c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 261 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/Processors/QueryPlan/Optimizations/liftUpFunctions.cpp
View File

@ -4,6 +4,7 @@
#include <Processors/QueryPlan/Optimizations/Optimizations.h>
#include <Processors/QueryPlan/SortingStep.h>
#include <Common/Exception.h>
#include <DataTypes/IDataType.h>
namespace DB
{
@ -28,6 +29,20 @@ const DB::DataStream & getChildOutputStream(DB::QueryPlan::Node & node)
namespace DB::QueryPlanOptimizations
{
/// This is a check that output columns does not have the same name
/// This is ok for DAG, but may introduce a bug in a SotringStep cause columns are selected by name.
static bool areOutputsConvertableToBlock(const ActionsDAG::NodeRawConstPtrs & outputs)
{
std::unordered_set<std::string_view> names;
for (const auto & output : outputs)
{
if (!names.emplace(output->result_name).second)
return false;
}
return true;
}
size_t tryExecuteFunctionsAfterSorting(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes)
{
if (parent_node->children.size() != 1)
@ -57,6 +72,9 @@ size_t tryExecuteFunctionsAfterSorting(QueryPlan::Node * parent_node, QueryPlan:
if (unneeded_for_sorting->trivial())
return 0;
if (!areOutputsConvertableToBlock(needed_for_sorting->getOutputs()))
return 0;
// Sorting (parent_node) -> Expression (child_node)
auto & node_with_needed = nodes.emplace_back();
std::swap(node_with_needed.children, child_node->children);

0
tests/queries/0_stateless/02789_functions_after_sorting_and_columns_with_same_names_bug.reference Normal file
View File

133
tests/queries/0_stateless/02789_functions_after_sorting_and_columns_with_same_names_bug.sql Normal file
View File

@ -0,0 +1,133 @@
drop table if exists test;
drop table if exists test1;
CREATE TABLE test
(
`pt` String,
`count_distinct_exposure_uv` AggregateFunction(uniqHLL12, Int64)
)
ENGINE = AggregatingMergeTree
ORDER BY pt;
SELECT *
FROM
(
SELECT m0.pt AS pt
,m0.`exposure_uv` AS exposure_uv
,round(m2.exposure_uv,4) AS exposure_uv_hb_last_value
,if(m2.exposure_uv IS NULL OR m2.exposure_uv = 0,NULL,round((m0.exposure_uv - m2.exposure_uv) * 1.0 / m2.exposure_uv,4)) AS exposure_uv_hb_diff_percent
,round(m1.exposure_uv,4) AS exposure_uv_tb_last_value
,if(m1.exposure_uv IS NULL OR m1.exposure_uv = 0,NULL,round((m0.exposure_uv - m1.exposure_uv) * 1.0 / m1.exposure_uv,4)) AS exposure_uv_tb_diff_percent
FROM
(
SELECT m0.pt AS pt
,`exposure_uv` AS `exposure_uv`
FROM
(
SELECT pt AS pt
,CASE WHEN COUNT(`exposure_uv`) > 0 THEN AVG(`exposure_uv`) ELSE 0 END AS `exposure_uv`
FROM
(
SELECT pt AS pt
,uniqHLL12Merge(count_distinct_exposure_uv) AS `exposure_uv`
FROM test
GROUP BY pt
) m
GROUP BY pt
) m0
) m0
LEFT JOIN
(
SELECT m0.pt AS pt
,`exposure_uv` AS `exposure_uv`
FROM
(
SELECT formatDateTime(addYears(parseDateTimeBestEffort(pt),1),'%Y%m%d') AS pt
,CASE WHEN COUNT(`exposure_uv`) > 0 THEN AVG(`exposure_uv`) ELSE 0 END AS `exposure_uv`
FROM
(
SELECT pt AS pt
,uniqHLL12Merge(count_distinct_exposure_uv) AS `exposure_uv`
FROM test
GROUP BY pt
) m
GROUP BY pt
) m0
) m1
ON m0.pt = m1.pt
LEFT JOIN
(
SELECT m0.pt AS pt
,`exposure_uv` AS `exposure_uv`
FROM
(
SELECT formatDateTime(addDays(toDate(parseDateTimeBestEffort(pt)),1),'%Y%m%d') AS pt
,CASE WHEN COUNT(`exposure_uv`) > 0 THEN AVG(`exposure_uv`) ELSE 0 END AS `exposure_uv`
FROM
(
SELECT pt AS pt
,uniqHLL12Merge(count_distinct_exposure_uv) AS `exposure_uv`
FROM test
GROUP BY pt
) m
GROUP BY pt
) m0
) m2
ON m0.pt = m2.pt
) c0
ORDER BY pt ASC, exposure_uv DESC
settings join_use_nulls = 1;
CREATE TABLE test1
(
`pt` String,
`exposure_uv` Float64
)
ENGINE = Memory;
SELECT *
FROM
(
SELECT m0.pt
,m0.exposure_uv AS exposure_uv
,round(m2.exposure_uv,4)
FROM
(
SELECT pt
,exposure_uv
FROM test1
) m0
LEFT JOIN
(
SELECT pt
,exposure_uv
FROM test1
) m1
ON m0.pt = m1.pt
LEFT JOIN
(
SELECT pt
,exposure_uv
FROM test1
) m2
ON m0.pt = m2.pt
) c0
ORDER BY exposure_uv
settings join_use_nulls = 1;
SELECT
pt AS pt,
exposure_uv AS exposure_uv
FROM
(
SELECT
pt
FROM test1
) AS m0
FULL OUTER JOIN
(
SELECT
pt,
exposure_uv
FROM test1
) AS m1 ON m0.pt = m1.pt;

3
tests/queries/0_stateless/02789_functions_after_sorting_and_columns_with_same_names_bug_2.reference Normal file
View File

@ -0,0 +1,3 @@
20230626 0.3156979034107179 \N \N
20230626 0.2624629016490004 \N \N
20230626 0.19390556368960468 \N \N

107
tests/queries/0_stateless/02789_functions_after_sorting_and_columns_with_same_names_bug_2.sql Normal file
View File

@ -0,0 +1,107 @@
create table test1 (
`pt` String,
`brand_name` String,
`total_indirect_order_cnt` Float64,
`total_indirect_gmv` Float64
) ENGINE = Memory;
create table test2 (
`pt` String,
`brand_name` String,
`exposure_uv` Float64,
`click_uv` Float64
) ENGINE = Memory;
INSERT INTO test1 (`pt`, `brand_name`, `total_indirect_order_cnt`, `total_indirect_gmv`) VALUES ('20230625', 'LINING', 2232, 1008710), ('20230625', 'adidas', 125, 58820), ('20230625', 'Nike', 1291, 1033020), ('20230626', 'Nike', 1145, 938926), ('20230626', 'LINING', 1904, 853336), ('20230626', 'adidas', 133, 62546), ('20220626', 'LINING', 3747, 1855203), ('20220626', 'Nike', 2295, 1742665), ('20220626', 'adidas', 302, 122388);
INSERT INTO test2 (`pt`, `brand_name`, `exposure_uv`, `click_uv`) VALUES ('20230625', 'Nike', 2012913, 612831), ('20230625', 'adidas', 480277, 96176), ('20230625', 'LINING', 2474234, 627814), ('20230626', 'Nike', 1934666, 610770), ('20230626', 'adidas', 469904, 91117), ('20230626', 'LINING', 2285142, 599765), ('20220626', 'Nike', 2979656, 937166), ('20220626', 'adidas', 704751, 124250), ('20220626', 'LINING', 3163884, 1010221);
SELECT * FROM (
SELECT m0.pt AS pt
,m0.`uvctr` AS uvctr
,round(m1.uvctr,4) AS uvctr_hb_last_value
,round(m2.uvctr,4) AS uvctr_tb_last_value
FROM
(
SELECT m0.pt AS pt
,COALESCE(m0.brand_name,m1.brand_name) AS brand_name
,if(isNaN(`click_uv` / `exposure_uv`) OR isInfinite(`click_uv` / `exposure_uv`),NULL,`click_uv` / `exposure_uv`) AS `uvctr`
FROM
(
SELECT pt AS pt
,brand_name AS `brand_name`
,exposure_uv AS `exposure_uv`
,click_uv AS `click_uv`
FROM test2
WHERE pt = '20230626'
) m0
FULL JOIN
(
SELECT pt AS pt
,brand_name AS `brand_name`
,total_indirect_order_cnt AS `total_indirect_order_cnt`
,total_indirect_gmv AS `total_indirect_gmv`
FROM test1
WHERE pt = '20230626'
) m1
ON m0.brand_name = m1.brand_name AND m0.pt = m1.pt
) m0
LEFT JOIN
(
SELECT m0.pt AS pt
,if(isNaN(`click_uv` / `exposure_uv`) OR isInfinite(`click_uv` / `exposure_uv`),NULL,`click_uv` / `exposure_uv`) AS `uvctr`
,COALESCE(m0.brand_name,m1.brand_name) AS brand_name
,`exposure_uv` AS `exposure_uv`
,`click_uv`
FROM
(
SELECT pt AS pt
,brand_name AS `brand_name`
,exposure_uv AS `exposure_uv`
,click_uv AS `click_uv`
FROM test2
WHERE pt = '20230625'
) m0
FULL JOIN
(
SELECT pt AS pt
,brand_name AS `brand_name`
,total_indirect_order_cnt AS `total_indirect_order_cnt`
,total_indirect_gmv AS `total_indirect_gmv`
FROM test1
WHERE pt = '20230625'
) m1
ON m0.brand_name = m1.brand_name AND m0.pt = m1.pt
) m1
ON m0.brand_name = m1.brand_name AND m0.pt = m1.pt
LEFT JOIN
(
SELECT m0.pt AS pt
,if(isNaN(`click_uv` / `exposure_uv`) OR isInfinite(`click_uv` / `exposure_uv`),NULL,`click_uv` / `exposure_uv`) AS `uvctr`
,COALESCE(m0.brand_name,m1.brand_name) AS brand_name
,`exposure_uv` AS `exposure_uv`
,`click_uv`
FROM
(
SELECT pt AS pt
,brand_name AS `brand_name`
,exposure_uv AS `exposure_uv`
,click_uv AS `click_uv`
FROM test2
WHERE pt = '20220626'
) m0
FULL JOIN
(
SELECT pt AS pt
,brand_name AS `brand_name`
,total_indirect_order_cnt AS `total_indirect_order_cnt`
,total_indirect_gmv AS `total_indirect_gmv`
FROM test1
WHERE pt = '20220626'
) m1
ON m0.brand_name = m1.brand_name AND m0.pt = m1.pt
) m2
ON m0.brand_name = m2.brand_name AND m0.pt = m2.pt
) c0
ORDER BY pt ASC, uvctr DESC;