fix low card types in merge join (#12035)

This commit is contained in:
Artem Zuikov 2020-06-30 14:13:43 +03:00 committed by GitHub
parent 094334de0b
commit 29178e26da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 195 additions and 3 deletions

View File

@ -396,7 +396,6 @@ MergeJoin::MergeJoin(std::shared_ptr<TableJoin> table_join_, const Block & right
if (required_right_keys.count(column.name))
right_columns_to_add.insert(ColumnWithTypeAndName{nullptr, column.type, column.name});
JoinCommon::removeLowCardinalityInplace(right_columns_to_add);
JoinCommon::createMissedColumns(right_columns_to_add);
if (nullable_right_side)
@ -513,7 +512,7 @@ bool MergeJoin::saveRightBlock(Block && block)
bool MergeJoin::addJoinedBlock(const Block & src_block, bool)
{
Block block = materializeBlock(src_block);
JoinCommon::removeLowCardinalityInplace(block);
JoinCommon::removeLowCardinalityInplace(block, table_join->keyNamesRight());
sortBlock(block, right_sort_description);
return saveRightBlock(std::move(block));
@ -525,7 +524,7 @@ void MergeJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed)
{
JoinCommon::checkTypesOfKeys(block, table_join->keyNamesLeft(), right_table_keys, table_join->keyNamesRight());
materializeBlockInplace(block);
JoinCommon::removeLowCardinalityInplace(block);
JoinCommon::removeLowCardinalityInplace(block, table_join->keyNamesLeft());
sortBlock(block, left_sort_description);
}

View File

@ -104,6 +104,16 @@ void removeLowCardinalityInplace(Block & block)
}
}
void removeLowCardinalityInplace(Block & block, const Names & names)
{
for (const String & column_name : names)
{
auto & col = block.getByName(column_name);
col.column = recursiveRemoveLowCardinality(col.column);
col.type = recursiveRemoveLowCardinality(col.type);
}
}
void splitAdditionalColumns(const Block & sample_block, const Names & key_names, Block & block_keys, Block & block_others)
{
block_others = materializeBlock(sample_block);

View File

@ -20,6 +20,7 @@ Columns materializeColumns(const Block & block, const Names & names);
ColumnRawPtrs materializeColumnsInplace(Block & block, const Names & names);
ColumnRawPtrs getRawPointers(const Columns & columns);
void removeLowCardinalityInplace(Block & block);
void removeLowCardinalityInplace(Block & block, const Names & names);
/// Split key and other columns by keys name list
void splitAdditionalColumns(const Block & sample_block, const Names & key_names, Block & block_keys, Block & block_others);

View File

@ -0,0 +1,41 @@
0
0
0
0
0
0
0
0
0
-
0 0
0 0
0 0
0 0
0 0
0 0
0 0
0 0
0 0
-
0 1
1 2
2 0
0 1
1 2
2 0
0 1
1 2
2 0
0 1
1 2
2 0
0 1
1 2
2 0
0 1
1 2
2 \N
0 1
1 2
2 \N

View File

@ -0,0 +1,30 @@
set join_algorithm = 'partial_merge';
select * from (select dummy as val from system.one) s1 any left join (select dummy as val from system.one) s2 using val;
select * from (select toLowCardinality(dummy) as val from system.one) s1 any left join (select dummy as val from system.one) s2 using val;
select * from (select dummy as val from system.one) s1 any left join (select toLowCardinality(dummy) as val from system.one) s2 using val;
select * from (select toLowCardinality(dummy) as val from system.one) s1 any left join (select toLowCardinality(dummy) as val from system.one) s2 using val;
select * from (select toLowCardinality(toNullable(dummy)) as val from system.one) s1 any left join (select dummy as val from system.one) s2 using val;
select * from (select dummy as val from system.one) s1 any left join (select toLowCardinality(toNullable(dummy)) as val from system.one) s2 using val;
select * from (select toLowCardinality(toNullable(dummy)) as val from system.one) s1 any left join (select toLowCardinality(dummy) as val from system.one) s2 using val;
select * from (select toLowCardinality(dummy) as val from system.one) s1 any left join (select toLowCardinality(toNullable(dummy)) as val from system.one) s2 using val;
select * from (select toLowCardinality(toNullable(dummy)) as val from system.one) s1 any left join (select toLowCardinality(toNullable(dummy)) as val from system.one) s2 using val;
select '-';
select * from (select dummy as val from system.one) s1 any left join (select dummy as val from system.one) s2 on val + 0 = val * 1; -- { serverError 352 }
select * from (select dummy as val from system.one) s1 any left join (select dummy as rval from system.one) s2 on val + 0 = rval * 1;
select * from (select toLowCardinality(dummy) as val from system.one) s1 any left join (select dummy as rval from system.one) s2 on val + 0 = rval * 1;
select * from (select dummy as val from system.one) s1 any left join (select toLowCardinality(dummy) as rval from system.one) s2 on val + 0 = rval * 1;
select * from (select toLowCardinality(dummy) as val from system.one) s1 any left join (select toLowCardinality(dummy) as rval from system.one) s2 on val + 0 = rval * 1;
select * from (select toLowCardinality(toNullable(dummy)) as val from system.one) s1 any left join (select dummy as rval from system.one) s2 on val + 0 = rval * 1;
select * from (select dummy as val from system.one) s1 any left join (select toLowCardinality(toNullable(dummy)) as rval from system.one) s2 on val + 0 = rval * 1;
select * from (select toLowCardinality(toNullable(dummy)) as val from system.one) s1 any left join (select toLowCardinality(dummy) as rval from system.one) s2 on val + 0 = rval * 1;
select * from (select toLowCardinality(dummy) as val from system.one) s1 any left join (select toLowCardinality(toNullable(dummy)) as rval from system.one) s2 on val + 0 = rval * 1;
select * from (select toLowCardinality(toNullable(dummy)) as val from system.one) s1 any left join (select toLowCardinality(toNullable(dummy)) as rval from system.one) s2 on val + 0 = rval * 1;
select '-';
select * from (select number as l from system.numbers limit 3) s1 any left join (select number as r from system.numbers limit 3) s2 on l + 1 = r * 1;
select * from (select toLowCardinality(number) as l from system.numbers limit 3) s1 any left join (select number as r from system.numbers limit 3) s2 on l + 1 = r * 1;
select * from (select number as l from system.numbers limit 3) s1 any left join (select toLowCardinality(number) as r from system.numbers limit 3) s2 on l + 1 = r * 1;
select * from (select toLowCardinality(number) as l from system.numbers limit 3) s1 any left join (select toLowCardinality(number) as r from system.numbers limit 3) s2 on l + 1 = r * 1;
select * from (select toLowCardinality(toNullable(number)) as l from system.numbers limit 3) s1 any left join (select toLowCardinality(number) as r from system.numbers limit 3) s2 on l + 1 = r * 1;
select * from (select toLowCardinality(number) as l from system.numbers limit 3) s1 any left join (select toLowCardinality(toNullable(number)) as r from system.numbers limit 3) s2 on l + 1 = r * 1;
select * from (select toLowCardinality(toNullable(number)) as l from system.numbers limit 3) s1 any left join (select toLowCardinality(toNullable(number)) as r from system.numbers limit 3) s2 on l + 1 = r * 1;

View File

@ -0,0 +1,36 @@
-
LowCardinality(UInt64) UInt64 String LowCardinality(String)
-
UInt64 LowCardinality(UInt64) LowCardinality(String) String
-
LowCardinality(UInt64) LowCardinality(UInt64) LowCardinality(String) LowCardinality(String)
-
LowCardinality(UInt64) UInt64 String LowCardinality(String)
LowCardinality(UInt64) UInt64 String LowCardinality(String)
LowCardinality(UInt64) UInt64 String LowCardinality(String)
-
UInt64 LowCardinality(UInt64) LowCardinality(String) String
UInt64 LowCardinality(UInt64) LowCardinality(String) String
UInt64 LowCardinality(UInt64) LowCardinality(String) String
-
LowCardinality(UInt64) LowCardinality(UInt64) LowCardinality(String) LowCardinality(String)
LowCardinality(UInt64) LowCardinality(UInt64) LowCardinality(String) LowCardinality(String)
LowCardinality(UInt64) LowCardinality(UInt64) LowCardinality(String) LowCardinality(String)
-
LowCardinality(UInt64) UInt64 String LowCardinality(String)
-
UInt64 LowCardinality(UInt64) LowCardinality(String) String
-
LowCardinality(UInt64) LowCardinality(UInt64) LowCardinality(String) LowCardinality(String)
-
LowCardinality(UInt64) UInt64 String LowCardinality(String)
LowCardinality(UInt64) UInt64 String LowCardinality(String)
LowCardinality(UInt64) UInt64 String LowCardinality(String)
-
UInt64 LowCardinality(UInt64) LowCardinality(String) String
UInt64 LowCardinality(UInt64) LowCardinality(String) String
UInt64 LowCardinality(UInt64) LowCardinality(String) String
-
LowCardinality(UInt64) LowCardinality(UInt64) LowCardinality(String) LowCardinality(String)
LowCardinality(UInt64) LowCardinality(UInt64) LowCardinality(String) LowCardinality(String)
LowCardinality(UInt64) LowCardinality(UInt64) LowCardinality(String) LowCardinality(String)

View File

@ -0,0 +1,75 @@
set join_algorithm = 'hash';
select '-';
select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s))
from (select toLowCardinality(number) k, toString(number) s from numbers(2)) as js1
join (select number+1 k, toLowCardinality(toString(number+1)) s from numbers(2)) as js2
using k order by js1.k, js2.k;
select '-';
select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s))
from (select number k, toLowCardinality(toString(number)) s from numbers(2)) as js1
join (select toLowCardinality(number+1) k, toString(number+1) s from numbers(2)) as js2
using k order by js1.k, js2.k;
select '-';
select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s))
from (select toLowCardinality(number) k, toLowCardinality(toString(number)) s from numbers(2)) as js1
join (select toLowCardinality(number+1) k, toLowCardinality(toString(number+1)) s from numbers(2)) as js2
using k order by js1.k, js2.k;
select '-';
select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s))
from (select toLowCardinality(number) k, toString(number) s from numbers(2)) as js1
full join (select number+1 k, toLowCardinality(toString(number+1)) s from numbers(2)) as js2
using k order by js1.k, js2.k;
select '-';
select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s))
from (select number k, toLowCardinality(toString(number)) s from numbers(2)) as js1
full join (select toLowCardinality(number+1) k, toString(number+1) s from numbers(2)) as js2
using k order by js1.k, js2.k;
select '-';
select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s))
from (select toLowCardinality(number) k, toLowCardinality(toString(number)) s from numbers(2)) as js1
full join (select toLowCardinality(number+1) k, toLowCardinality(toString(number+1)) s from numbers(2)) as js2
using k order by js1.k, js2.k;
set join_algorithm = 'prefer_partial_merge';
select '-';
select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s))
from (select toLowCardinality(number) k, toString(number) s from numbers(2)) as js1
join (select number+1 k, toLowCardinality(toString(number+1)) s from numbers(2)) as js2
using k order by js1.k, js2.k;
select '-';
select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s))
from (select number k, toLowCardinality(toString(number)) s from numbers(2)) as js1
join (select toLowCardinality(number+1) k, toString(number+1) s from numbers(2)) as js2
using k order by js1.k, js2.k;
select '-';
select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s))
from (select toLowCardinality(number) k, toLowCardinality(toString(number)) s from numbers(2)) as js1
join (select toLowCardinality(number+1) k, toLowCardinality(toString(number+1)) s from numbers(2)) as js2
using k order by js1.k, js2.k;
select '-';
select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s))
from (select toLowCardinality(number) k, toString(number) s from numbers(2)) as js1
full join (select number+1 k, toLowCardinality(toString(number+1)) s from numbers(2)) as js2
using k order by js1.k, js2.k;
select '-';
select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s))
from (select number k, toLowCardinality(toString(number)) s from numbers(2)) as js1
full join (select toLowCardinality(number+1) k, toString(number+1) s from numbers(2)) as js2
using k order by js1.k, js2.k;
select '-';
select toTypeName(materialize(js1.k)), toTypeName(materialize(js2.k)), toTypeName(materialize(js1.s)), toTypeName(materialize(js2.s))
from (select toLowCardinality(number) k, toLowCardinality(toString(number)) s from numbers(2)) as js1
full join (select toLowCardinality(number+1) k, toLowCardinality(toString(number+1)) s from numbers(2)) as js2
using k order by js1.k, js2.k;