Merge pull request #67359 from Avogar/disable-dynamic-tests-in-sanitizers

Disable some Dynamic tests under sanitizers, rewrite 03202_dynamic_null_map_subcolumn to sql

tests/queries/0_stateless/03036_dynamic_read_subcolumns_compact_merge_tree.sql

@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 
 set allow_experimental_variant_type = 1;
 set use_variant_as_common_type = 1;

tests/queries/0_stateless/03036_dynamic_read_subcolumns_memory.sql

@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 
 set allow_experimental_variant_type = 1;
 set use_variant_as_common_type = 1;

tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.sql.j2

@@ -0,0 +1,43 @@
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+set allow_experimental_dynamic_type = 1;
+
+drop table if exists test;
+
+{% for engine in ['Memory', 'MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000', 'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1'] -%}
+
+create table test (id UInt64, d Dynamic) engine={{ engine }};
+
+insert into test select number, number from numbers(10);
+insert into test select number, 'str_' || toString(number) from numbers(10, 10);
+insert into test select number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1)) from numbers(20, 10);
+insert into test select number, NULL from numbers(30, 10);
+insert into test select number, multiIf(number % 4 == 3, 'str_' || toString(number), number % 4 == 2, NULL, number % 4 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1))) from numbers(40, 40);
+insert into test select number, [range((number % 10 + 1)::UInt64)]::Array(Array(Dynamic)) from numbers(10, 10);
+
+select distinct dynamicType(d) as type from test order by type;
+select count() from test where dynamicType(d) == 'UInt64';
+select count() from test where d.UInt64 is not NULL;
+select count() from test where dynamicType(d) == 'String';
+select count() from test where d.String is not NULL;
+select count() from test where dynamicType(d) == 'Date';
+select count() from test where d.Date is not NULL;
+select count() from test where dynamicType(d) == 'Array(Variant(String, UInt64))';
+select count() from test where not empty(d.`Array(Variant(String, UInt64))`);
+select count() from test where dynamicType(d) == 'Array(Array(Dynamic))';
+select count() from test where not empty(d.`Array(Array(Dynamic))`);
+select count() from test where d is NULL;
+select count() from test where not empty(d.`Tuple(a Array(Dynamic))`.a.String);
+
+select d, d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test order by id, d;
+select d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test order by id, d;
+select d.Int8, d.Date, d.`Array(String)` from test order by id, d;
+select d, d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test order by id, d;
+select d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64, d.`Array(Variant(String, UInt64))`.String from test order by id, d;
+select d, d.`Tuple(a UInt64, b String)`.a, d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Variant(String, UInt64))`.UInt64 from test order by id, d;
+select d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Dynamic)`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test order by id, d;
+select d.`Array(Array(Dynamic))`.size1, d.`Array(Array(Dynamic))`.UInt64, d.`Array(Array(Dynamic))`.`Map(String, Tuple(a UInt64))`.values.a from test order by id, d;
+
+drop table test;
+
+{% endfor -%}

tests/queries/0_stateless/03036_dynamic_read_subcolumns_wide_merge_tree.sql

@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 
 set allow_experimental_variant_type = 1;
 set use_variant_as_common_type = 1;

tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.sql

@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 set allow_experimental_dynamic_type=1;
 
 drop table if exists test;
@@ -30,4 +30,4 @@ system start merges test;
 optimize table test final;
 select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
 
-drop table test;
+drop table test;

tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.sql

@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 set allow_experimental_dynamic_type=1;
 
 drop table if exists test;
@@ -30,4 +30,4 @@ system start merges test;
 optimize table test final;
 select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
 
-drop table test;
+drop table test;

tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.sql

@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 set allow_experimental_dynamic_type=1;
 
 drop table if exists test;
@@ -31,4 +31,4 @@ system start merges test;
 optimize table test final;
 select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
 
-drop table test;
+drop table test;

tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.sql

@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 set allow_experimental_dynamic_type=1;
 
 drop table if exists test;
@@ -30,4 +30,4 @@ system start merges test;
 optimize table test final;
 select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
 
-drop table test;
+drop table test;

tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.sql

@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 
 set allow_experimental_dynamic_type = 1;
 

tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.sql

@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 
 set allow_experimental_dynamic_type = 1;
 

tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.sql

@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 
 set allow_experimental_dynamic_type = 1;
 

tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.sql

@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 
 set allow_experimental_dynamic_type = 1;
 

tests/queries/0_stateless/03037_dynamic_merges_small.reference.j2

@@ -0,0 +1,112 @@
+5	DateTime
+6	Date
+7	Array(UInt16)
+8	String
+10	None
+10	UInt64
+7	Array(UInt16)
+10	None
+10	UInt64
+19	String
+7	Array(UInt16)
+10	None
+10	UInt64
+19	String
+20	Map(UInt64, UInt64)
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+26	String
+1	Tuple(UInt64, UInt64)
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+26	String
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+27	String
+5	DateTime
+6	Date
+7	Array(UInt16)
+8	String
+10	None
+10	UInt64
+7	Array(UInt16)
+10	None
+10	UInt64
+19	String
+7	Array(UInt16)
+10	None
+10	UInt64
+19	String
+20	Map(UInt64, UInt64)
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+26	String
+1	Tuple(UInt64, UInt64)
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+26	String
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+27	String
+5	DateTime
+6	Date
+7	Array(UInt16)
+8	String
+10	None
+10	UInt64
+7	Array(UInt16)
+10	None
+10	UInt64
+19	String
+7	Array(UInt16)
+10	None
+10	UInt64
+19	String
+20	Map(UInt64, UInt64)
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+26	String
+1	Tuple(UInt64, UInt64)
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+26	String
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+27	String
+5	DateTime
+6	Date
+7	Array(UInt16)
+8	String
+10	None
+10	UInt64
+7	Array(UInt16)
+10	None
+10	UInt64
+19	String
+7	Array(UInt16)
+10	None
+10	UInt64
+19	String
+20	Map(UInt64, UInt64)
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+26	String
+1	Tuple(UInt64, UInt64)
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+26	String
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+27	String

tests/queries/0_stateless/03037_dynamic_merges_small.sql.j2

@@ -0,0 +1,42 @@
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+set allow_experimental_dynamic_type = 1;
+
+drop table if exists test;
+
+{% for engine in ['MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000',
+                  'MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1',
+                  'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1',
+                  'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1'] -%}
+
+create table test (id UInt64, d Dynamic(max_types=3)) engine={{ engine }};
+
+system stop merges test;
+insert into test select number, number from numbers(10);
+insert into test select number, 'str_' || toString(number) from numbers(8);
+insert into test select number, range(number % 10 + 1) from numbers(7);
+insert into test select number, toDate(number) from numbers(6);
+insert into test select number, toDateTime(number) from numbers(5);
+insert into test select number, NULL from numbers(10);
+
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+system start merges test; optimize table test final;;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+
+system stop merges test;
+insert into test select number, map(number, number) from numbers(20);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+
+system stop merges test;
+insert into test select number, tuple(number, number) from numbers(1);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+
+drop table test;
+
+{% endfor -%}

tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.reference.j2

@@ -1,4 +1,4 @@
-Memory
+--- Memory ---
 test
 Array(Array(Dynamic))
 Array(Variant(String, UInt64))
@@ -17,7 +17,7 @@ UInt64
 10
 20
 0
-MergeTree compact
+--- MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000 ---
 test
 Array(Array(Dynamic))
 Array(Variant(String, UInt64))
@@ -36,7 +36,7 @@ UInt64
 10
 20
 0
-MergeTree wide
+--- MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1 ---
 test
 Array(Array(Dynamic))
 Array(Variant(String, UInt64))

tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh

@@ -1,60 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 --optimize_functions_to_subcolumns=0"
-
-
-function test()
-{
-    echo "test"
-    $CH_CLIENT -q "insert into test select number, number from numbers(10) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, 'str_' || toString(number) from numbers(10, 10) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1)) from numbers(20, 10) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, NULL from numbers(30, 10) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, multiIf(number % 4 == 3, 'str_' || toString(number), number % 4 == 2, NULL, number % 4 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1))) from numbers(40, 40) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, [range((number % 10 + 1)::UInt64)]::Array(Array(Dynamic)) from numbers(10, 10) settings min_insert_block_size_rows=50000"
-
-    $CH_CLIENT -q "select distinct dynamicType(d) as type from test order by type"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'UInt64'"
-    $CH_CLIENT -q "select count() from test where d.UInt64 is not NULL"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'String'"
-    $CH_CLIENT -q "select count() from test where d.String is not NULL"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'Date'"
-    $CH_CLIENT -q "select count() from test where d.Date is not NULL"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'Array(Variant(String, UInt64))'"
-    $CH_CLIENT -q "select count() from test where not empty(d.\`Array(Variant(String, UInt64))\`)"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'Array(Array(Dynamic))'"
-    $CH_CLIENT -q "select count() from test where not empty(d.\`Array(Array(Dynamic))\`)"
-    $CH_CLIENT -q "select count() from test where d is NULL"
-    $CH_CLIENT -q "select count() from test where not empty(d.\`Tuple(a Array(Dynamic))\`.a.String)"
-
-    $CH_CLIENT -q "select d, d.UInt64.null, d.String.null, d.\`Array(Variant(String, UInt64))\`.null from test format Null"
-    $CH_CLIENT -q "select d.UInt64.null, d.String.null, d.\`Array(Variant(String, UInt64))\`.null from test format Null"
-    $CH_CLIENT -q "select d.Int8.null, d.Date.null, d.\`Array(String)\`.null from test format Null"
-    $CH_CLIENT -q "select d, d.UInt64.null, d.Date.null, d.\`Array(Variant(String, UInt64))\`.null, d.\`Array(Variant(String, UInt64))\`.size0, d.\`Array(Variant(String, UInt64))\`.UInt64.null from test format Null"
-    $CH_CLIENT -q "select d.UInt64.null, d.Date.null, d.\`Array(Variant(String, UInt64))\`.null, d.\`Array(Variant(String, UInt64))\`.size0, d.\`Array(Variant(String, UInt64))\`.UInt64.null, d.\`Array(Variant(String, UInt64))\`.String.null from test format Null"
-    $CH_CLIENT -q "select d, d.\`Tuple(a UInt64, b String)\`.a, d.\`Array(Dynamic)\`.\`Variant(String, UInt64)\`.UInt64.null, d.\`Array(Variant(String, UInt64))\`.UInt64.null from test format Null"
-    $CH_CLIENT -q "select d.\`Array(Dynamic)\`.\`Variant(String, UInt64)\`.UInt64.null, d.\`Array(Dynamic)\`.size0, d.\`Array(Variant(String, UInt64))\`.UInt64.null from test format Null"
-    $CH_CLIENT -q "select d.\`Array(Array(Dynamic))\`.size1, d.\`Array(Array(Dynamic))\`.UInt64.null, d.\`Array(Array(Dynamic))\`.\`Map(String, Tuple(a UInt64))\`.values.a from test format Null"
-}
-
-$CH_CLIENT -q "drop table if exists test;"
-
-echo "Memory"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=Memory"
-test
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree compact"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;"
-test
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree wide"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
-test
-$CH_CLIENT -q "drop table test;"

tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sql.j2

@@ -0,0 +1,49 @@
+# Tags: long
+
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+set allow_experimental_dynamic_type = 1;
+set optimize_functions_to_subcolumns = 0;
+
+drop table if exists test;
+
+{% for engine in ['Memory', 'MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000', 'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1'] -%}
+
+SELECT '--- {{ engine }} ---';
+
+create table test (id UInt64, d Dynamic) engine={{ engine }};
+
+select 'test';
+insert into test select number, number from numbers(10) settings min_insert_block_size_rows=50000;
+insert into test select number, 'str_' || toString(number) from numbers(10, 10) settings min_insert_block_size_rows=50000;
+insert into test select number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1)) from numbers(20, 10) settings min_insert_block_size_rows=50000;
+insert into test select number, NULL from numbers(30, 10) settings min_insert_block_size_rows=50000;
+insert into test select number, multiIf(number % 4 == 3, 'str_' || toString(number), number % 4 == 2, NULL, number % 4 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1))) from numbers(40, 40) settings min_insert_block_size_rows=50000;
+insert into test select number, [range((number % 10 + 1)::UInt64)]::Array(Array(Dynamic)) from numbers(10, 10) settings min_insert_block_size_rows=50000;
+
+select distinct dynamicType(d) as type from test order by type;
+select count() from test where dynamicType(d) == 'UInt64';
+select count() from test where d.UInt64 is not NULL;
+select count() from test where dynamicType(d) == 'String';
+select count() from test where d.String is not NULL;
+select count() from test where dynamicType(d) == 'Date';
+select count() from test where d.Date is not NULL;
+select count() from test where dynamicType(d) == 'Array(Variant(String, UInt64))';
+select count() from test where not empty(d.`Array(Variant(String, UInt64))`);
+select count() from test where dynamicType(d) == 'Array(Array(Dynamic))';
+select count() from test where not empty(d.`Array(Array(Dynamic))`);
+select count() from test where d is NULL;
+select count() from test where not empty(d.`Tuple(a Array(Dynamic))`.a.String);
+
+select d, d.UInt64.null, d.String.null, d.`Array(Variant(String, UInt64))`.null from test format Null;
+select d.UInt64.null, d.String.null, d.`Array(Variant(String, UInt64))`.null from test format Null;
+select d.Int8.null, d.Date.null, d.`Array(String)`.null from test format Null;
+select d, d.UInt64.null, d.Date.null, d.`Array(Variant(String, UInt64))`.null, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64.null from test format Null;
+select d.UInt64.null, d.Date.null, d.`Array(Variant(String, UInt64))`.null, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64.null, d.`Array(Variant(String, UInt64))`.String.null from test format Null;
+select d, d.`Tuple(a UInt64, b String)`.a, d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64.null, d.`Array(Variant(String, UInt64))`.UInt64.null from test format Null;
+select d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64.null, d.`Array(Dynamic)`.size0, d.`Array(Variant(String, UInt64))`.UInt64.null from test format Null;
+select d.`Array(Array(Dynamic))`.size1, d.`Array(Array(Dynamic))`.UInt64.null, d.`Array(Array(Dynamic))`.`Map(String, Tuple(a UInt64))`.values.a from test format Null;
+
+drop table test;
+
+{% endfor -%}