thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-25 17:12:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #66983 from ClickHouse/split_01508_partition_pruning_long

Browse Source 
Split 01508_partition_pruning_long
This commit is contained in:
Nikita Taranov 2024-07-25 10:24:53 +00:00 committed by GitHub
commit 8988f0518f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 228 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
tests/queries/0_stateless/01508_partition_pruning_long.sh
View File

@ -1,30 +0,0 @@
#!/usr/bin/env bash
# Tags: long, no-polymorphic-parts, no-random-settings, no-random-merge-tree-settings, no-debug
# Description of test result:
# Test the correctness of the partition pruning
#
# Script executes queries from a file 01508_partition_pruning_long.queries (1 line = 1 query)
# Queries are started with 'select' (but NOT with 'SELECT') are executed with log_level=debug
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
queries="${CURDIR}/01508_partition_pruning_long.queries"
while IFS= read -r sql
do
[ -z "$sql" ] && continue
if [[ "$sql" == select* ]] ;
then
echo "$sql"
${CLICKHOUSE_CLIENT} --query "$sql"
CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=debug/g')
${CLICKHOUSE_CLIENT} --query "$sql" 2>&1 | grep -oh "Selected .* parts by partition key, *. parts by primary key, .* marks by primary key, .* marks to read from .* ranges.*$"
CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/--send_logs_level=debug/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/g')
echo ""
else
${CLICKHOUSE_CLIENT} --query "$sql"
fi
done < "$queries"

119
tests/queries/0_stateless/01508_partition_pruning_long.reference → tests/queries/0_stateless/01508_partition_pruning_long_1.reference
View File

@ -123,122 +123,3 @@ select uniqExact(_part), count() from tMM where toYYYYMM(d) between 202009 and 2
2 20000
Selected 2/3 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
--------- tDD ----------------------------
select uniqExact(_part), count() from tDD where toDate(d)=toDate('2020-09-24');
1 10000
Selected 1/4 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
select uniqExact(_part), count() FROM tDD WHERE toDate(d) = toDate('2020-09-24');
1 10000
Selected 1/4 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
select uniqExact(_part), count() FROM tDD WHERE toDate(d) = '2020-09-24';
1 10000
Selected 1/4 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
select uniqExact(_part), count() FROM tDD WHERE toDate(d) >= '2020-09-23' and toDate(d) <= '2020-09-26';
3 40000
Selected 3/4 parts by partition key, 3 parts by primary key, 4/4 marks by primary key, 4 marks to read from 3 ranges
select uniqExact(_part), count() FROM tDD WHERE toYYYYMMDD(d) >= 20200923 and toDate(d) <= '2020-09-26';
3 40000
Selected 3/4 parts by partition key, 3 parts by primary key, 4/4 marks by primary key, 4 marks to read from 3 ranges
--------- sDD ----------------------------
select uniqExact(_part), count() from sDD;
6 30000
Selected 6/6 parts by partition key, 6 parts by primary key, 6/6 marks by primary key, 6 marks to read from 6 ranges
select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC')-1)+1 = 202010;
3 9999
Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC')-1) = 202010;
2 9999
Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC')-1) = 202110;
0 0
Selected 0/6 parts by partition key, 0 parts by primary key, 0/0 marks by primary key, 0 marks to read from 0 ranges
select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC'))+1 > 202009 and toStartOfDay(toDateTime(intDiv(d,1000),'UTC')) < toDateTime('2020-10-02 00:00:00','UTC');
3 11440
Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC'))+1 > 202009 and toDateTime(intDiv(d,1000),'UTC') < toDateTime('2020-10-01 00:00:00','UTC');
2 10000
Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
select uniqExact(_part), count() from sDD where d >= 1598918400000;
4 20000
Selected 4/6 parts by partition key, 4 parts by primary key, 4/4 marks by primary key, 4 marks to read from 4 ranges
select uniqExact(_part), count() from sDD where d >= 1598918400000 and toYYYYMM(toDateTime(intDiv(d,1000),'UTC')-1) < 202010;
3 10001
Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
--------- xMM ----------------------------
select uniqExact(_part), count() from xMM where toStartOfDay(d) >= '2020-10-01 00:00:00';
2 10000
Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d <= '2020-10-01 00:00:00';
3 10001
Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d < '2020-10-01 00:00:00';
2 10000
Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d <= '2020-10-01 00:00:00' and a=1;
1 1
Selected 1/6 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d <= '2020-10-01 00:00:00' and a<>3;
2 5001
Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d < '2020-10-01 00:00:00' and a<>3;
1 5000
Selected 1/6 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d < '2020-11-01 00:00:00' and a = 1;
2 10000
Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
select uniqExact(_part), count() from xMM where a = 1;
3 15000
Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
select uniqExact(_part), count() from xMM where a = 66;
0 0
Selected 0/6 parts by partition key, 0 parts by primary key, 0/0 marks by primary key, 0 marks to read from 0 ranges
select uniqExact(_part), count() from xMM where a <> 66;
6 30000
Selected 6/6 parts by partition key, 6 parts by primary key, 6/6 marks by primary key, 6 marks to read from 6 ranges
select uniqExact(_part), count() from xMM where a = 2;
2 10000
Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
select uniqExact(_part), count() from xMM where a = 1;
2 15000
Selected 2/5 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
select uniqExact(_part), count() from xMM where toStartOfDay(d) >= '2020-10-01 00:00:00';
1 10000
Selected 1/5 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
select uniqExact(_part), count() from xMM where a <> 66;
5 30000
Selected 5/5 parts by partition key, 5 parts by primary key, 5/5 marks by primary key, 5 marks to read from 5 ranges
select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d <= '2020-10-01 00:00:00' and a<>3;
2 5001
Selected 2/5 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d < '2020-10-01 00:00:00' and a<>3;
1 5000
Selected 1/5 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges

79
tests/queries/0_stateless/01508_partition_pruning_long_1.sh Executable file
View File

@ -0,0 +1,79 @@
#!/usr/bin/env bash
# Tags: long, no-polymorphic-parts, no-random-settings, no-random-merge-tree-settings, no-debug
# Description of test result:
# Test the correctness of the partition pruning
#
# Script executes queries from a file 01508_partition_pruning_long.queries (1 line = 1 query)
# Queries are started with 'select' (but NOT with 'SELECT') are executed with log_level=debug
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
while IFS= read -r sql
do
[ -z "$sql" ] && continue
if [[ "$sql" == select* ]] ;
then
echo "$sql"
${CLICKHOUSE_CLIENT} --query "$sql"
CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=debug/g')
${CLICKHOUSE_CLIENT} --query "$sql" 2>&1 | grep -oh "Selected .* parts by partition key, *. parts by primary key, .* marks by primary key, .* marks to read from .* ranges.*$"
CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/--send_logs_level=debug/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/g')
echo ""
else
${CLICKHOUSE_CLIENT} --query "$sql"
fi
done <<< "
DROP TABLE IF EXISTS tMM;
CREATE TABLE tMM(d DateTime('Asia/Istanbul'), a Int64) ENGINE = MergeTree PARTITION BY toYYYYMM(d) ORDER BY tuple() SETTINGS index_granularity = 8192;
SYSTEM STOP MERGES tMM;
INSERT INTO tMM SELECT toDateTime('2020-08-16 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
INSERT INTO tMM SELECT toDateTime('2020-08-16 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
INSERT INTO tMM SELECT toDateTime('2020-09-01 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
INSERT INTO tMM SELECT toDateTime('2020-09-01 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
INSERT INTO tMM SELECT toDateTime('2020-10-01 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
INSERT INTO tMM SELECT toDateTime('2020-10-15 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
SELECT '--------- tMM ----------------------------';
select uniqExact(_part), count() from tMM where toDate(d)=toDate('2020-09-15');
select uniqExact(_part), count() from tMM where toDate(d)=toDate('2020-09-01');
select uniqExact(_part), count() from tMM where toDate(d)=toDate('2020-10-15');
select uniqExact(_part), count() from tMM where toDate(d)='2020-09-15';
select uniqExact(_part), count() from tMM where toYYYYMM(d)=202009;
select uniqExact(_part), count() from tMM where toYYYYMMDD(d)=20200816;
select uniqExact(_part), count() from tMM where toYYYYMMDD(d)=20201015;
select uniqExact(_part), count() from tMM where toDate(d)='2020-10-15';
select uniqExact(_part), count() from tMM where d >= '2020-09-01 00:00:00' and d<'2020-10-15 00:00:00';
select uniqExact(_part), count() from tMM where d >= '2020-01-16 00:00:00' and d < toDateTime('2021-08-17 00:00:00', 'Asia/Istanbul');
select uniqExact(_part), count() from tMM where d >= '2020-09-16 00:00:00' and d < toDateTime('2020-10-01 00:00:00', 'Asia/Istanbul');
select uniqExact(_part), count() from tMM where d >= '2020-09-12 00:00:00' and d < '2020-10-16 00:00:00';
select uniqExact(_part), count() from tMM where toStartOfDay(d) >= '2020-09-12 00:00:00';
select uniqExact(_part), count() from tMM where toStartOfDay(d) = '2020-09-01 00:00:00';
select uniqExact(_part), count() from tMM where toStartOfDay(d) = '2020-10-01 00:00:00';
select uniqExact(_part), count() from tMM where toStartOfDay(d) >= '2020-09-15 00:00:00' and d < '2020-10-16 00:00:00';
select uniqExact(_part), count() from tMM where toYYYYMM(d) between 202009 and 202010;
select uniqExact(_part), count() from tMM where toYYYYMM(d) between 202009 and 202009;
select uniqExact(_part), count() from tMM where toYYYYMM(d) between 202009 and 202010 and toStartOfDay(d) = '2020-10-01 00:00:00';
select uniqExact(_part), count() from tMM where toYYYYMM(d) >= 202009 and toStartOfDay(d) < '2020-10-02 00:00:00';
select uniqExact(_part), count() from tMM where toYYYYMM(d) > 202009 and toStartOfDay(d) < '2020-10-02 00:00:00';
select uniqExact(_part), count() from tMM where toYYYYMM(d)+1 > 202009 and toStartOfDay(d) < '2020-10-02 00:00:00';
select uniqExact(_part), count() from tMM where toYYYYMM(d)+1 > 202010 and toStartOfDay(d) < '2020-10-02 00:00:00';
select uniqExact(_part), count() from tMM where toYYYYMM(d)+1 > 202010;
select uniqExact(_part), count() from tMM where toYYYYMM(d-1)+1 = 202010;
select uniqExact(_part), count() from tMM where toStartOfMonth(d) >= '2020-09-15';
select uniqExact(_part), count() from tMM where toStartOfMonth(d) >= '2020-09-01';
select uniqExact(_part), count() from tMM where toStartOfMonth(d) >= '2020-09-01' and toStartOfMonth(d) < '2020-10-01';
SYSTEM START MERGES tMM;
OPTIMIZE TABLE tMM FINAL;
select uniqExact(_part), count() from tMM where toYYYYMM(d-1)+1 = 202010;
select uniqExact(_part), count() from tMM where toYYYYMM(d)+1 > 202010;
select uniqExact(_part), count() from tMM where toYYYYMM(d) between 202009 and 202010;
DROP TABLE tMM;
"

119
tests/queries/0_stateless/01508_partition_pruning_long_2.reference Normal file
View File

@ -0,0 +1,119 @@
--------- tDD ----------------------------
select uniqExact(_part), count() from tDD where toDate(d)=toDate('2020-09-24');
1 10000
Selected 1/4 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
select uniqExact(_part), count() FROM tDD WHERE toDate(d) = toDate('2020-09-24');
1 10000
Selected 1/4 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
select uniqExact(_part), count() FROM tDD WHERE toDate(d) = '2020-09-24';
1 10000
Selected 1/4 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
select uniqExact(_part), count() FROM tDD WHERE toDate(d) >= '2020-09-23' and toDate(d) <= '2020-09-26';
3 40000
Selected 3/4 parts by partition key, 3 parts by primary key, 4/4 marks by primary key, 4 marks to read from 3 ranges
select uniqExact(_part), count() FROM tDD WHERE toYYYYMMDD(d) >= 20200923 and toDate(d) <= '2020-09-26';
3 40000
Selected 3/4 parts by partition key, 3 parts by primary key, 4/4 marks by primary key, 4 marks to read from 3 ranges
--------- sDD ----------------------------
select uniqExact(_part), count() from sDD;
6 30000
Selected 6/6 parts by partition key, 6 parts by primary key, 6/6 marks by primary key, 6 marks to read from 6 ranges
select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC')-1)+1 = 202010;
3 9999
Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC')-1) = 202010;
2 9999
Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC')-1) = 202110;
0 0
Selected 0/6 parts by partition key, 0 parts by primary key, 0/0 marks by primary key, 0 marks to read from 0 ranges
select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC'))+1 > 202009 and toStartOfDay(toDateTime(intDiv(d,1000),'UTC')) < toDateTime('2020-10-02 00:00:00','UTC');
3 11440
Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC'))+1 > 202009 and toDateTime(intDiv(d,1000),'UTC') < toDateTime('2020-10-01 00:00:00','UTC');
2 10000
Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
select uniqExact(_part), count() from sDD where d >= 1598918400000;
4 20000
Selected 4/6 parts by partition key, 4 parts by primary key, 4/4 marks by primary key, 4 marks to read from 4 ranges
select uniqExact(_part), count() from sDD where d >= 1598918400000 and toYYYYMM(toDateTime(intDiv(d,1000),'UTC')-1) < 202010;
3 10001
Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
--------- xMM ----------------------------
select uniqExact(_part), count() from xMM where toStartOfDay(d) >= '2020-10-01 00:00:00';
2 10000
Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d <= '2020-10-01 00:00:00';
3 10001
Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d < '2020-10-01 00:00:00';
2 10000
Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d <= '2020-10-01 00:00:00' and a=1;
1 1
Selected 1/6 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d <= '2020-10-01 00:00:00' and a<>3;
2 5001
Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d < '2020-10-01 00:00:00' and a<>3;
1 5000
Selected 1/6 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d < '2020-11-01 00:00:00' and a = 1;
2 10000
Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
select uniqExact(_part), count() from xMM where a = 1;
3 15000
Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
select uniqExact(_part), count() from xMM where a = 66;
0 0
Selected 0/6 parts by partition key, 0 parts by primary key, 0/0 marks by primary key, 0 marks to read from 0 ranges
select uniqExact(_part), count() from xMM where a <> 66;
6 30000
Selected 6/6 parts by partition key, 6 parts by primary key, 6/6 marks by primary key, 6 marks to read from 6 ranges
select uniqExact(_part), count() from xMM where a = 2;
2 10000
Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
select uniqExact(_part), count() from xMM where a = 1;
2 15000
Selected 2/5 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
select uniqExact(_part), count() from xMM where toStartOfDay(d) >= '2020-10-01 00:00:00';
1 10000
Selected 1/5 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
select uniqExact(_part), count() from xMM where a <> 66;
5 30000
Selected 5/5 parts by partition key, 5 parts by primary key, 5/5 marks by primary key, 5 marks to read from 5 ranges
select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d <= '2020-10-01 00:00:00' and a<>3;
2 5001
Selected 2/5 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d < '2020-10-01 00:00:00' and a<>3;
1 5000
Selected 1/5 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges

80
tests/queries/0_stateless/01508_partition_pruning_long.queries → tests/queries/0_stateless/01508_partition_pruning_long_2.sh Normal file → Executable file
View File

@ -1,15 +1,35 @@
DROP TABLE IF EXISTS tMM;
#!/usr/bin/env bash
# Tags: long, no-polymorphic-parts, no-random-settings, no-random-merge-tree-settings, no-debug
# Description of test result:
# Test the correctness of the partition pruning
#
# Script executes queries from a file 01508_partition_pruning_long.queries (1 line = 1 query)
# Queries are started with 'select' (but NOT with 'SELECT') are executed with log_level=debug
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
while IFS= read -r sql
do
[ -z "$sql" ] && continue
if [[ "$sql" == select* ]] ;
then
echo "$sql"
${CLICKHOUSE_CLIENT} --query "$sql"
CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=debug/g')
${CLICKHOUSE_CLIENT} --query "$sql" 2>&1 | grep -oh "Selected .* parts by partition key, *. parts by primary key, .* marks by primary key, .* marks to read from .* ranges.*$"
CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/--send_logs_level=debug/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/g')
echo ""
else
${CLICKHOUSE_CLIENT} --query "$sql"
fi
done <<< "
DROP TABLE IF EXISTS tDD;
DROP TABLE IF EXISTS sDD;
DROP TABLE IF EXISTS xMM;
CREATE TABLE tMM(d DateTime('Asia/Istanbul'), a Int64) ENGINE = MergeTree PARTITION BY toYYYYMM(d) ORDER BY tuple() SETTINGS index_granularity = 8192;
SYSTEM STOP MERGES tMM;
INSERT INTO tMM SELECT toDateTime('2020-08-16 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
INSERT INTO tMM SELECT toDateTime('2020-08-16 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
INSERT INTO tMM SELECT toDateTime('2020-09-01 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
INSERT INTO tMM SELECT toDateTime('2020-09-01 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
INSERT INTO tMM SELECT toDateTime('2020-10-01 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
INSERT INTO tMM SELECT toDateTime('2020-10-15 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
CREATE TABLE tDD(d DateTime('Asia/Istanbul'),a Int) ENGINE = MergeTree PARTITION BY toYYYYMMDD(d) ORDER BY tuple() SETTINGS index_granularity = 8192;
SYSTEM STOP MERGES tDD;
@ -34,44 +54,6 @@ INSERT INTO xMM SELECT toDateTime('2020-10-01 00:00:00', 'Asia/Istanbul') + numb
INSERT INTO xMM SELECT toDateTime('2020-10-15 00:00:00', 'Asia/Istanbul') + number*60, 1, number FROM numbers(5000);
SELECT '--------- tMM ----------------------------';
select uniqExact(_part), count() from tMM where toDate(d)=toDate('2020-09-15');
select uniqExact(_part), count() from tMM where toDate(d)=toDate('2020-09-01');
select uniqExact(_part), count() from tMM where toDate(d)=toDate('2020-10-15');
select uniqExact(_part), count() from tMM where toDate(d)='2020-09-15';
select uniqExact(_part), count() from tMM where toYYYYMM(d)=202009;
select uniqExact(_part), count() from tMM where toYYYYMMDD(d)=20200816;
select uniqExact(_part), count() from tMM where toYYYYMMDD(d)=20201015;
select uniqExact(_part), count() from tMM where toDate(d)='2020-10-15';
select uniqExact(_part), count() from tMM where d >= '2020-09-01 00:00:00' and d<'2020-10-15 00:00:00';
select uniqExact(_part), count() from tMM where d >= '2020-01-16 00:00:00' and d < toDateTime('2021-08-17 00:00:00', 'Asia/Istanbul');
select uniqExact(_part), count() from tMM where d >= '2020-09-16 00:00:00' and d < toDateTime('2020-10-01 00:00:00', 'Asia/Istanbul');
select uniqExact(_part), count() from tMM where d >= '2020-09-12 00:00:00' and d < '2020-10-16 00:00:00';
select uniqExact(_part), count() from tMM where toStartOfDay(d) >= '2020-09-12 00:00:00';
select uniqExact(_part), count() from tMM where toStartOfDay(d) = '2020-09-01 00:00:00';
select uniqExact(_part), count() from tMM where toStartOfDay(d) = '2020-10-01 00:00:00';
select uniqExact(_part), count() from tMM where toStartOfDay(d) >= '2020-09-15 00:00:00' and d < '2020-10-16 00:00:00';
select uniqExact(_part), count() from tMM where toYYYYMM(d) between 202009 and 202010;
select uniqExact(_part), count() from tMM where toYYYYMM(d) between 202009 and 202009;
select uniqExact(_part), count() from tMM where toYYYYMM(d) between 202009 and 202010 and toStartOfDay(d) = '2020-10-01 00:00:00';
select uniqExact(_part), count() from tMM where toYYYYMM(d) >= 202009 and toStartOfDay(d) < '2020-10-02 00:00:00';
select uniqExact(_part), count() from tMM where toYYYYMM(d) > 202009 and toStartOfDay(d) < '2020-10-02 00:00:00';
select uniqExact(_part), count() from tMM where toYYYYMM(d)+1 > 202009 and toStartOfDay(d) < '2020-10-02 00:00:00';
select uniqExact(_part), count() from tMM where toYYYYMM(d)+1 > 202010 and toStartOfDay(d) < '2020-10-02 00:00:00';
select uniqExact(_part), count() from tMM where toYYYYMM(d)+1 > 202010;
select uniqExact(_part), count() from tMM where toYYYYMM(d-1)+1 = 202010;
select uniqExact(_part), count() from tMM where toStartOfMonth(d) >= '2020-09-15';
select uniqExact(_part), count() from tMM where toStartOfMonth(d) >= '2020-09-01';
select uniqExact(_part), count() from tMM where toStartOfMonth(d) >= '2020-09-01' and toStartOfMonth(d) < '2020-10-01';
SYSTEM START MERGES tMM;
OPTIMIZE TABLE tMM FINAL;
select uniqExact(_part), count() from tMM where toYYYYMM(d-1)+1 = 202010;
select uniqExact(_part), count() from tMM where toYYYYMM(d)+1 > 202010;
select uniqExact(_part), count() from tMM where toYYYYMM(d) between 202009 and 202010;
SELECT '--------- tDD ----------------------------';
SYSTEM START MERGES tDD;
OPTIMIZE TABLE tDD FINAL;
@ -116,9 +98,7 @@ select uniqExact(_part), count() from xMM where a <> 66;
select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d <= '2020-10-01 00:00:00' and a<>3;
select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d < '2020-10-01 00:00:00' and a<>3;
DROP TABLE tMM;
DROP TABLE tDD;
DROP TABLE sDD;
DROP TABLE xMM;
"