From 610798aa487ee1b2ef6007b9185a1c1b27a11660 Mon Sep 17 00:00:00 2001 From: keenwolf Date: Sat, 6 Feb 2021 15:32:49 +0800 Subject: [PATCH 1/4] fix the toMinute bug which will cause toDateTime or toString printing wrong time --- base/common/DateLUTImpl.h | 10 +++- src/Functions/ya.make | 1 + .../0_stateless/01698_fix_toMinute.reference | 24 ++++++++ .../0_stateless/01698_fix_toMinute.sql | 16 +++++ .../01699_timezoneOffset.reference | 58 +++++-------------- .../0_stateless/01699_timezoneOffset.sql | 3 +- 6 files changed, 65 insertions(+), 47 deletions(-) create mode 100644 tests/queries/0_stateless/01698_fix_toMinute.reference create mode 100644 tests/queries/0_stateless/01698_fix_toMinute.sql diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 23c78f6e7fc..8991f69d3f3 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -317,8 +317,14 @@ public: if (offset_is_whole_number_of_hours_everytime) return (t / 60) % 60; - UInt32 date = find(t).date; - return (UInt32(t) - date) / 60 % 60; + /// To consider the DST changing situation within this day. + /// also make the special timezones with no whole hour offset such as 'Australia/Lord_Howe' been taken into account + DayNum index = findIndex(t); + time_t res = t - lut[index].date; + if (lut[index].amount_of_offset_change != 0 && t >= lut[index].date + lut[index].time_at_offset_change) + res += lut[index].amount_of_offset_change; + + return res / 60 % 60; } inline time_t toStartOfMinute(time_t t) const { return t / 60 * 60; } diff --git a/src/Functions/ya.make b/src/Functions/ya.make index b97a4a187e9..647bbbb47cb 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -452,6 +452,7 @@ SRCS( timeSlot.cpp timeSlots.cpp timezone.cpp + timezoneOffset.cpp toColumnTypeName.cpp toCustomWeek.cpp toDayOfMonth.cpp diff --git a/tests/queries/0_stateless/01698_fix_toMinute.reference b/tests/queries/0_stateless/01698_fix_toMinute.reference new file mode 100644 index 00000000000..5df800c9fef --- /dev/null +++ b/tests/queries/0_stateless/01698_fix_toMinute.reference @@ -0,0 +1,24 @@ +Check the bug causing situation: the special Australia/Lord_Howe time zone. tooDateTime and toString functions are all tested at once +1554559200 2019-04-07 01:00:00 2019-04-07 01:00:00 +1554559800 2019-04-07 01:10:00 2019-04-07 01:10:00 +1554560400 2019-04-07 01:20:00 2019-04-07 01:20:00 +1554561000 2019-04-07 01:30:00 2019-04-07 01:30:00 +1554561600 2019-04-07 01:40:00 2019-04-07 01:40:00 +1554562200 2019-04-07 01:50:00 2019-04-07 01:50:00 +1554562800 2019-04-07 01:30:00 2019-04-07 01:30:00 +1554563400 2019-04-07 01:40:00 2019-04-07 01:40:00 +1554564000 2019-04-07 01:50:00 2019-04-07 01:50:00 +1554564600 2019-04-07 02:00:00 2019-04-07 02:00:00 +1554565200 2019-04-07 02:10:00 2019-04-07 02:10:00 +1554565800 2019-04-07 02:20:00 2019-04-07 02:20:00 +1554566400 2019-04-07 02:30:00 2019-04-07 02:30:00 +1554567000 2019-04-07 02:40:00 2019-04-07 02:40:00 +1554567600 2019-04-07 02:50:00 2019-04-07 02:50:00 +1554568200 2019-04-07 03:00:00 2019-04-07 03:00:00 +1554568800 2019-04-07 03:10:00 2019-04-07 03:10:00 +1554569400 2019-04-07 03:20:00 2019-04-07 03:20:00 +1554570000 2019-04-07 03:30:00 2019-04-07 03:30:00 +1554570600 2019-04-07 03:40:00 2019-04-07 03:40:00 +4 days test in batch comparing with manually computation result for Europe/Moscow whose timezone epoc is of whole hour: +4 days test in batch comparing with manually computation result for Asia/Tehran whose timezone epoc is of half hour: +4 days test in batch comparing with manually computation result for Australia/Lord_Howe whose timezone epoc is of half hour and also its DST offset is half hour: diff --git a/tests/queries/0_stateless/01698_fix_toMinute.sql b/tests/queries/0_stateless/01698_fix_toMinute.sql new file mode 100644 index 00000000000..293741b6957 --- /dev/null +++ b/tests/queries/0_stateless/01698_fix_toMinute.sql @@ -0,0 +1,16 @@ +/* toDateTime or toString or other functions which should call the toMinute() function will all meet this bug. tests below will verify the toDateTime and toString. */ +SELECT 'Check the bug causing situation: the special Australia/Lord_Howe time zone. tooDateTime and toString functions are all tested at once'; +SELECT toUnixTimestamp(x) as tt, (toDateTime('2019-04-07 01:00:00', 'Australia/Lord_Howe') + INTERVAL number * 600 SECOND) AS x, toString(x) as xx FROM numbers(20); + +/* The Batch Part. Test period is whole 4 days*/ +SELECT '4 days test in batch comparing with manually computation result for Europe/Moscow whose timezone epoc is of whole hour:'; +SELECT toUnixTimestamp(x) as tt, (toDateTime('1981-04-01 00:00:00', 'Europe/Moscow') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(576) where res != calc; +SELECT toUnixTimestamp(x) as tt, (toDateTime('1981-09-30 00:00:00', 'Europe/Moscow') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(576) where res != calc; + +SELECT '4 days test in batch comparing with manually computation result for Asia/Tehran whose timezone epoc is of half hour:'; +SELECT toUnixTimestamp(x) as tt, (toDateTime('2020-03-21 00:00:00', 'Asia/Tehran') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(576) where res != calc; +SELECT toUnixTimestamp(x) as tt, (toDateTime('2020-09-20 00:00:00', 'Asia/Tehran') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(576) where res != calc; + +SELECT '4 days test in batch comparing with manually computation result for Australia/Lord_Howe whose timezone epoc is of half hour and also its DST offset is half hour:'; +SELECT toUnixTimestamp(x) as tt, (toDateTime('2020-10-04 01:40:00', 'Australia/Lord_Howe') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(576) where res != calc; +SELECT toUnixTimestamp(x) as tt, (toDateTime('2019-04-07 01:00:00', 'Australia/Lord_Howe') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(576) where res != calc; diff --git a/tests/queries/0_stateless/01699_timezoneOffset.reference b/tests/queries/0_stateless/01699_timezoneOffset.reference index e70c5fa62ee..45f30314f5a 100644 --- a/tests/queries/0_stateless/01699_timezoneOffset.reference +++ b/tests/queries/0_stateless/01699_timezoneOffset.reference @@ -50,57 +50,29 @@ DST boundary test for Australia/Lord_Howe. This is a special timezone with DST o DST boundary test for Australia/Lord_Howe: 0 2020-10-04 01:40:00 37800 1601737800 1 2020-10-04 01:50:00 37800 1601738400 -2 2020-10-04 02:00:00 39600 1601739000 -3 2020-10-04 02:10:00 39600 1601739600 +2 2020-10-04 02:30:00 39600 1601739000 +3 2020-10-04 02:40:00 39600 1601739600 0 2019-04-07 01:00:00 39600 1554559200 1 2019-04-07 01:10:00 39600 1554559800 2 2019-04-07 01:20:00 39600 1554560400 3 2019-04-07 01:30:00 39600 1554561000 4 2019-04-07 01:40:00 39600 1554561600 5 2019-04-07 01:50:00 39600 1554562200 -6 2019-04-07 01:00:00 37800 1554562800 -7 2019-04-07 01:10:00 37800 1554563400 -8 2019-04-07 01:20:00 37800 1554564000 -9 2019-04-07 02:30:00 37800 1554564600 -10 2019-04-07 02:40:00 37800 1554565200 -11 2019-04-07 02:50:00 37800 1554565800 -12 2019-04-07 02:00:00 37800 1554566400 -13 2019-04-07 02:10:00 37800 1554567000 -14 2019-04-07 02:20:00 37800 1554567600 -15 2019-04-07 03:30:00 37800 1554568200 -16 2019-04-07 03:40:00 37800 1554568800 -17 2019-04-07 03:50:00 37800 1554569400 +6 2019-04-07 01:30:00 37800 1554562800 +7 2019-04-07 01:40:00 37800 1554563400 +8 2019-04-07 01:50:00 37800 1554564000 +9 2019-04-07 02:00:00 37800 1554564600 +10 2019-04-07 02:10:00 37800 1554565200 +11 2019-04-07 02:20:00 37800 1554565800 +12 2019-04-07 02:30:00 37800 1554566400 +13 2019-04-07 02:40:00 37800 1554567000 +14 2019-04-07 02:50:00 37800 1554567600 +15 2019-04-07 03:00:00 37800 1554568200 +16 2019-04-07 03:10:00 37800 1554568800 +17 2019-04-07 03:20:00 37800 1554569400 4 days test in batch comparing with manually computation result for Europe/Moscow: 4 days test in batch comparing with manually computation result for Asia/Tehran: -The result maybe wrong for toDateTime processing Australia/Lord_Howe -1601739000 2020-10-04 02:00:00 39600 37800 -1601739600 2020-10-04 02:10:00 39600 37800 -1601740200 2020-10-04 02:20:00 39600 37800 -1601740800 2020-10-04 03:30:00 39600 41400 -1601741400 2020-10-04 03:40:00 39600 41400 -1601742000 2020-10-04 03:50:00 39600 41400 -1601742600 2020-10-04 03:00:00 39600 37800 -1601743200 2020-10-04 03:10:00 39600 37800 -1601743800 2020-10-04 03:20:00 39600 37800 -1601744400 2020-10-04 04:30:00 39600 41400 -1601745000 2020-10-04 04:40:00 39600 41400 -1601745600 2020-10-04 04:50:00 39600 41400 -1601746200 2020-10-04 04:00:00 39600 37800 -1601746800 2020-10-04 04:10:00 39600 37800 -1601747400 2020-10-04 04:20:00 39600 37800 -1601748000 2020-10-04 05:30:00 39600 41400 -1554562800 2019-04-07 01:00:00 37800 36000 -1554563400 2019-04-07 01:10:00 37800 36000 -1554564000 2019-04-07 01:20:00 37800 36000 -1554564600 2019-04-07 02:30:00 37800 39600 -1554565200 2019-04-07 02:40:00 37800 39600 -1554565800 2019-04-07 02:50:00 37800 39600 -1554566400 2019-04-07 02:00:00 37800 36000 -1554567000 2019-04-07 02:10:00 37800 36000 -1554567600 2019-04-07 02:20:00 37800 36000 -1554568200 2019-04-07 03:30:00 37800 39600 -1554568800 2019-04-07 03:40:00 37800 39600 -1554569400 2019-04-07 03:50:00 37800 39600 +4 days test in batch comparing with manually computation result for Australia/Lord_Howe Moscow DST Years: 11 1981-06-01 00:00:00 14400 12 1982-06-01 00:00:00 14400 diff --git a/tests/queries/0_stateless/01699_timezoneOffset.sql b/tests/queries/0_stateless/01699_timezoneOffset.sql index 1b3f05ecdd7..8cabb23c4de 100644 --- a/tests/queries/0_stateless/01699_timezoneOffset.sql +++ b/tests/queries/0_stateless/01699_timezoneOffset.sql @@ -26,8 +26,7 @@ SELECT '4 days test in batch comparing with manually computation result for Asia SELECT toUnixTimestamp(x) as tt, (toDateTime('2020-03-21 00:00:00', 'Asia/Tehran') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(576) where res != calc; SELECT toUnixTimestamp(x) as tt, (toDateTime('2020-09-20 00:00:00', 'Asia/Tehran') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(576) where res != calc; -/* During this test we got unexpected result comes from the toDateTime() function when process the special time zone of 'Australia/Lord_Howe', which may be some kind of bugs. */ -SELECT 'The result maybe wrong for toDateTime processing Australia/Lord_Howe'; +SELECT '4 days test in batch comparing with manually computation result for Australia/Lord_Howe'; SELECT toUnixTimestamp(x) as tt, (toDateTime('2020-10-04 01:40:00', 'Australia/Lord_Howe') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(18) where res != calc; SELECT toUnixTimestamp(x) as tt, (toDateTime('2019-04-07 01:00:00', 'Australia/Lord_Howe') + INTERVAL number * 600 SECOND) AS x, timezoneOffset(x) as res,(toDateTime(toString(x), 'UTC') - x ) AS calc FROM numbers(18) where res != calc; From 740c1c72e6eed901e56d7256f1067304e265dcf9 Mon Sep 17 00:00:00 2001 From: keenwolf Date: Sat, 6 Feb 2021 16:55:46 +0800 Subject: [PATCH 2/4] little fix --- tests/queries/0_stateless/01698_fix_toMinute.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01698_fix_toMinute.sql b/tests/queries/0_stateless/01698_fix_toMinute.sql index 293741b6957..f582806719d 100644 --- a/tests/queries/0_stateless/01698_fix_toMinute.sql +++ b/tests/queries/0_stateless/01698_fix_toMinute.sql @@ -1,5 +1,5 @@ /* toDateTime or toString or other functions which should call the toMinute() function will all meet this bug. tests below will verify the toDateTime and toString. */ -SELECT 'Check the bug causing situation: the special Australia/Lord_Howe time zone. tooDateTime and toString functions are all tested at once'; +SELECT 'Check the bug causing situation: the special Australia/Lord_Howe time zone. toDateTime and toString functions are all tested at once'; SELECT toUnixTimestamp(x) as tt, (toDateTime('2019-04-07 01:00:00', 'Australia/Lord_Howe') + INTERVAL number * 600 SECOND) AS x, toString(x) as xx FROM numbers(20); /* The Batch Part. Test period is whole 4 days*/ From 34af94accfc03fb6335aae9b8ca27f6e6992d49d Mon Sep 17 00:00:00 2001 From: keenwolf Date: Sat, 6 Feb 2021 16:59:01 +0800 Subject: [PATCH 3/4] little fix --- tests/queries/0_stateless/01698_fix_toMinute.reference | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01698_fix_toMinute.reference b/tests/queries/0_stateless/01698_fix_toMinute.reference index 5df800c9fef..7675aad3a57 100644 --- a/tests/queries/0_stateless/01698_fix_toMinute.reference +++ b/tests/queries/0_stateless/01698_fix_toMinute.reference @@ -1,4 +1,4 @@ -Check the bug causing situation: the special Australia/Lord_Howe time zone. tooDateTime and toString functions are all tested at once +Check the bug causing situation: the special Australia/Lord_Howe time zone. toDateTime and toString functions are all tested at once 1554559200 2019-04-07 01:00:00 2019-04-07 01:00:00 1554559800 2019-04-07 01:10:00 2019-04-07 01:10:00 1554560400 2019-04-07 01:20:00 2019-04-07 01:20:00 From 303c389b368598a57ef85ccca1930533fd8a5c13 Mon Sep 17 00:00:00 2001 From: keenwolf Date: Sun, 28 Feb 2021 18:11:54 +0800 Subject: [PATCH 4/4] change the time_t to unsigned to handle properly the start date of 1970-01-01 --- base/common/DateLUTImpl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 8991f69d3f3..471169ebcec 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -320,7 +320,7 @@ public: /// To consider the DST changing situation within this day. /// also make the special timezones with no whole hour offset such as 'Australia/Lord_Howe' been taken into account DayNum index = findIndex(t); - time_t res = t - lut[index].date; + UInt32 res = t - lut[index].date; if (lut[index].amount_of_offset_change != 0 && t >= lut[index].date + lut[index].time_at_offset_change) res += lut[index].amount_of_offset_change;