From 2d03d330bcc400a0b61c8028b01587de072aa60e Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Fri, 17 Apr 2020 16:26:44 +0300 Subject: [PATCH] Extended range of DateTime64 to years 1925 - 2238 The Year 1925 is a starting point because most of the timezones switched to saner (mostly 15-minutes based) offsets somewhere during 1924 or before. And that significantly simplifies implementation. 2238 is to simplify arithmetics for sanitizing LUT index access; there are less than 0x1ffff days from 1925. * Extended DateLUTImpl internal LUT to 0x1ffff items, some of which represent negative (pre-1970) time values. As a collateral benefit, Date now correctly supports dates up to 2149 (instead of 2106). * Added a new strong typedef ExtendedDayNum, which represents dates pre-1970 and post 2149. * Functions that used to return DayNum now return ExtendedDayNum. * Refactored DateLUTImpl to untie DayNum from the dual role of being a value and an index (due to negative time). Index is now a different type LUTIndex with explicit conversion functions from DatNum, time_t, and ExtendedDayNum. * Updated DateLUTImpl to properly support values close to epoch start (1970-01-01 00:00), including negative ones. * Reduced resolution of DateLUTImpl::Values::time_at_offset_change to multiple of 15-minutes to allow storing 64-bits of time_t in DateLUTImpl::Value while keeping same size. * Minor performance updates to DateLUTImpl when building month LUT by skipping non-start-of-month days. * Fixed extractTimeZoneFromFunctionArguments to work correctly with DateTime64. * New unit-tests and stateless integration tests for both DateTime and DateTime64. --- base/common/DateLUT.h | 1 - base/common/DateLUTImpl.cpp | 42 +- base/common/DateLUTImpl.h | 683 +++++++++++------- base/common/DayNum.h | 5 + base/common/LocalDate.h | 3 +- base/common/strong_typedef.h | 1 + base/common/tests/CMakeLists.txt | 4 +- base/common/tests/gtest_DateLutImpl.cpp | 515 +++++++++++++ programs/client/Client.cpp | 2 +- src/Core/DecimalFunctions.h | 18 +- src/Core/MySQL/MySQLReplication.cpp | 8 +- src/Core/tests/gtest_DecimalFunctions.cpp | 2 +- src/DataStreams/MongoDBBlockInputStream.cpp | 4 +- src/DataTypes/DataTypeDateTime64.h | 61 -- src/Functions/CustomWeekTransforms.h | 24 +- src/Functions/DateTimeTransforms.h | 225 +++++- src/Functions/FunctionCustomWeekToSomething.h | 1 + .../FunctionDateOrDateTimeAddInterval.h | 143 ++-- .../FunctionDateOrDateTimeToSomething.h | 3 +- src/Functions/FunctionsConversion.h | 106 +-- src/Functions/TransformDateTime64.h | 92 +++ src/Functions/dateDiff.cpp | 4 +- .../extractTimeZoneFromFunctionArguments.cpp | 5 +- src/Functions/formatDateTime.cpp | 6 +- src/Functions/now64.cpp | 2 +- src/Functions/toStartOfInterval.cpp | 50 +- src/Functions/today.cpp | 2 +- src/IO/ReadHelpers.h | 10 +- src/IO/WriteHelpers.h | 16 +- src/IO/parseDateTimeBestEffort.cpp | 15 +- src/IO/parseDateTimeBestEffort.h | 1 + src/Interpreters/CrashLog.cpp | 2 +- src/Interpreters/MetricLog.cpp | 2 +- src/Interpreters/OpenTelemetrySpanLog.cpp | 2 +- src/Interpreters/PartLog.cpp | 2 +- src/Interpreters/QueryLog.cpp | 2 +- src/Interpreters/QueryThreadLog.cpp | 2 +- src/Interpreters/TextLog.cpp | 2 +- src/Interpreters/TraceLog.cpp | 2 +- src/Interpreters/convertFieldToType.cpp | 2 +- src/Storages/StorageReplicatedMergeTree.cpp | 2 +- src/Storages/tests/part_name.cpp | 2 +- .../00921_datetime64_compatibility.python | 5 +- .../00921_datetime64_compatibility.reference | 152 ---- .../00921_datetime64_compatibility.sh | 2 +- .../01252_weird_time_zone.reference | 14 +- .../0_stateless/01252_weird_time_zone.sql | 28 +- .../01440_to_date_monotonicity.reference | 2 +- .../01561_Date_and_DateTime64_comparision.sql | 6 +- ...1_date_overflow_as_partition_key.reference | 4 +- .../01631_date_overflow_as_partition_key.sql | 2 +- .../01691_DateTime64_clamp.reference | 23 + .../0_stateless/01691_DateTime64_clamp.sql | 7 + .../convert-month-partitioned-parts/main.cpp | 5 +- 54 files changed, 1585 insertions(+), 741 deletions(-) create mode 100644 base/common/tests/gtest_DateLutImpl.cpp create mode 100644 src/Functions/TransformDateTime64.h diff --git a/base/common/DateLUT.h b/base/common/DateLUT.h index 93c6cb403e2..378b4360f3b 100644 --- a/base/common/DateLUT.h +++ b/base/common/DateLUT.h @@ -32,7 +32,6 @@ public: return date_lut.getImplementation(time_zone); } - static void setDefaultTimezone(const std::string & time_zone) { auto & date_lut = getInstance(); diff --git a/base/common/DateLUTImpl.cpp b/base/common/DateLUTImpl.cpp index 50620e21b8f..906f88fa90f 100644 --- a/base/common/DateLUTImpl.cpp +++ b/base/common/DateLUTImpl.cpp @@ -46,19 +46,26 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) if (&inside_main) assert(inside_main); - size_t i = 0; - time_t start_of_day = 0; cctz::time_zone cctz_time_zone; if (!cctz::load_time_zone(time_zone, &cctz_time_zone)) throw Poco::Exception("Cannot load time zone " + time_zone_); - cctz::time_zone::absolute_lookup start_of_epoch_lookup = cctz_time_zone.lookup(std::chrono::system_clock::from_time_t(start_of_day)); - offset_at_start_of_epoch = start_of_epoch_lookup.offset; + const cctz::civil_day epoch{1970, 1, 1}; + const cctz::civil_day lut_start{DATE_LUT_MIN_YEAR, 1, 1}; + time_t start_of_day = std::chrono::system_clock::to_time_t(cctz_time_zone.lookup(lut_start).pre); + time_offset_epoch = cctz::convert(cctz::civil_second(lut_start), cctz_time_zone).time_since_epoch().count(); + + // Note validated this against all timezones in the system. + assert((epoch - lut_start) == daynum_offset_epoch); + + offset_at_start_of_epoch = cctz_time_zone.lookup(cctz_time_zone.lookup(epoch).pre).offset; + offset_at_start_of_lut = cctz_time_zone.lookup(cctz_time_zone.lookup(lut_start).pre).offset; offset_is_whole_number_of_hours_everytime = true; - cctz::civil_day date{1970, 1, 1}; + cctz::civil_day date = lut_start; + UInt32 i = 0; do { cctz::time_zone::civil_lookup lookup = cctz_time_zone.lookup(date); @@ -72,7 +79,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) values.day_of_week = getDayOfWeek(date); values.date = start_of_day; - assert(values.year >= DATE_LUT_MIN_YEAR && values.year <= DATE_LUT_MAX_YEAR); + assert(values.year >= DATE_LUT_MIN_YEAR && values.year <= DATE_LUT_MAX_YEAR + 1); assert(values.month >= 1 && values.month <= 12); assert(values.day_of_month >= 1 && values.day_of_month <= 31); assert(values.day_of_week >= 1 && values.day_of_week <= 7); @@ -85,10 +92,13 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) else values.days_in_month = i != 0 ? lut[i - 1].days_in_month : 31; - values.time_at_offset_change = 0; - values.amount_of_offset_change = 0; + values.time_at_offset_change_value = 0; + values.amount_of_offset_change_value = 0; - if (start_of_day % 3600) + // TODO: this partially ignores fractional pre-epoch offsets, which may cause incorrect toRelativeHourNum() results for some timezones, namelly Europe\Minsk + // when pre-May 2 1924 it had an offset of UTC+1:50, and after it was UTC+2h. + // https://www.timeanddate.com/time/zone/belarus/minsk?syear=1900 + if (start_of_day > 0 && start_of_day % 3600) offset_is_whole_number_of_hours_everytime = false; /// If UTC offset was changed in previous day. @@ -97,7 +107,7 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) auto amount_of_offset_change_at_prev_day = 86400 - (lut[i].date - lut[i - 1].date); if (amount_of_offset_change_at_prev_day) { - lut[i - 1].amount_of_offset_change = amount_of_offset_change_at_prev_day; + lut[i - 1].amount_of_offset_change_value = amount_of_offset_change_at_prev_day / Values::OffsetChangeFactor; const auto utc_offset_at_beginning_of_day = cctz_time_zone.lookup(std::chrono::system_clock::from_time_t(lut[i - 1].date)).offset; @@ -116,11 +126,11 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) time_at_offset_change += 900; } - lut[i - 1].time_at_offset_change = time_at_offset_change; + lut[i - 1].time_at_offset_change_value = time_at_offset_change / Values::OffsetChangeFactor; - /// We doesn't support cases when time change results in switching to previous day. - if (static_cast(lut[i - 1].time_at_offset_change) + static_cast(lut[i - 1].amount_of_offset_change) < 0) - lut[i - 1].time_at_offset_change = -lut[i - 1].amount_of_offset_change; + /// We don't support cases when time change results in switching to previous day. + if (static_cast(lut[i - 1].time_at_offset_change()) + static_cast(lut[i - 1].amount_of_offset_change()) < 0) + lut[i - 1].time_at_offset_change_value = -lut[i - 1].amount_of_offset_change_value; } } @@ -128,7 +138,9 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) ++date; ++i; } - while (start_of_day <= DATE_LUT_MAX && i <= DATE_LUT_MAX_DAY_NUM); + while (i < DATE_LUT_SIZE && lut[i - 1].year <= DATE_LUT_MAX_YEAR); + +// date_lut_max = start_of_day; /// Fill excessive part of lookup table. This is needed only to simplify handling of overflow cases. while (i < DATE_LUT_SIZE) diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h index 064787fb64e..adfffb04681 100644 --- a/base/common/DateLUTImpl.h +++ b/base/common/DateLUTImpl.h @@ -11,9 +11,9 @@ #define DATE_LUT_MAX (0xFFFFFFFFU - 86400) #define DATE_LUT_MAX_DAY_NUM (0xFFFFFFFFU / 86400) /// Table size is bigger than DATE_LUT_MAX_DAY_NUM to fill all indices within UInt16 range: this allows to remove extra check. -#define DATE_LUT_SIZE 0x10000 -#define DATE_LUT_MIN_YEAR 1970 -#define DATE_LUT_MAX_YEAR 2106 /// Last supported year (incomplete) +#define DATE_LUT_SIZE 0x20000 +#define DATE_LUT_MIN_YEAR 1925 /// 1925 since wast majority of timezones changed to 15-minute aligned offsets somewhere in 1924 or earlier. +#define DATE_LUT_MAX_YEAR 2283 /// Last supported year (complete) #define DATE_LUT_YEARS (1 + DATE_LUT_MAX_YEAR - DATE_LUT_MIN_YEAR) /// Number of years in lookup table #if defined(__PPC__) @@ -45,14 +45,68 @@ public: DateLUTImpl(const DateLUTImpl &&) = delete; DateLUTImpl & operator=(const DateLUTImpl &&) = delete; + // Normalized and bound-checked index of element in lut, + // has to be a separate type to support overloading + // TODO: make sure that any arithmetic on LUTIndex actually results in valid LUTIndex. + STRONG_TYPEDEF(UInt32, LUTIndex) + template + friend inline LUTIndex operator+(const LUTIndex & index, const T v) + { + return LUTIndex{(index.toUnderType() + v) & date_lut_mask}; + } + template + friend inline LUTIndex operator+(const T v, const LUTIndex & index) + { + return LUTIndex{(v + index.toUnderType()) & date_lut_mask}; + } + friend inline LUTIndex operator+(const LUTIndex & index, const LUTIndex & v) + { + return LUTIndex{(index.toUnderType() + v.toUnderType()) & date_lut_mask}; + } + + template + friend inline LUTIndex operator-(const LUTIndex & index, const T v) + { + return LUTIndex{(index.toUnderType() - v) & date_lut_mask}; + } + template + friend inline LUTIndex operator-(const T v, const LUTIndex & index) + { + return LUTIndex{(v - index.toUnderType()) & date_lut_mask}; + } + friend inline LUTIndex operator-(const LUTIndex & index, const LUTIndex & v) + { + return LUTIndex{(index.toUnderType() - v.toUnderType()) & date_lut_mask}; + } + + template + friend inline LUTIndex operator*(const LUTIndex & index, const T v) + { + return LUTIndex{(index.toUnderType() * v) & date_lut_mask}; + } + template + friend inline LUTIndex operator*(const T v, const LUTIndex & index) + { + return LUTIndex{(v * index.toUnderType()) & date_lut_mask}; + } + + template + friend inline LUTIndex operator/(const LUTIndex & index, const T v) + { + return LUTIndex{(index.toUnderType() / v) & date_lut_mask}; + } + template + friend inline LUTIndex operator/(const T v, const LUTIndex & index) + { + return LUTIndex{(v / index.toUnderType()) & date_lut_mask}; + } + public: /// The order of fields matters for alignment and sizeof. struct Values { - /// Least significat 32 bits from time_t at beginning of the day. - /// If the unix timestamp of beginning of the day is negative (example: 1970-01-01 MSK, where time_t == -10800), then value will overflow. - /// Change to time_t; change constants above; and recompile the sources if you need to support time after 2105 year. - UInt32 date; + /// Least significat 64 bits from time_t at beginning of the day. + Int64 date; /// Properties of the day. UInt16 year; @@ -65,107 +119,175 @@ public: UInt8 days_in_month; /// For days, when offset from UTC was changed due to daylight saving time or permanent change, following values could be non zero. - Int16 amount_of_offset_change; /// Usually -3600 or 3600, but look at Lord Howe Island. - UInt32 time_at_offset_change; /// In seconds from beginning of the day. + Int8 amount_of_offset_change_value; /// Usually -3600 or 3600, but look at Lord Howe Island. multiply by OffsetChangeFactor + UInt8 time_at_offset_change_value; /// In seconds from beginning of the day. multiply by OffsetChangeFactor + + inline Int32 amount_of_offset_change() const + { + return static_cast(amount_of_offset_change_value) * OffsetChangeFactor; + } + + inline UInt32 time_at_offset_change() const + { + return static_cast(time_at_offset_change_value) * OffsetChangeFactor; + } + + /// Since most of the modern timezones have a DST change aligned to 15 minutes, to save as much space as possible inside Value, + /// we are dividing any offset change related value by this factor before setting it to Value, + /// hence it has to be explicitly multiplied back by this factor before being used. + static const UInt16 OffsetChangeFactor = 900; }; static_assert(sizeof(Values) == 16); private: - /// Lookup table is indexed by DayNum. + + // Mask is all-ones to allow efficient protection against overflow. + static const UInt32 date_lut_mask = 0x1ffff; + static_assert(date_lut_mask == DATE_LUT_SIZE - 1); + + const UInt32 daynum_offset_epoch = 16436; // offset to epoch in days (ExtendedDayNum) of the first day in LUT. + + /// Lookup table is indexed by LUTIndex. /// Day nums are the same in all time zones. 1970-01-01 is 0 and so on. /// Table is relatively large, so better not to place the object on stack. /// In comparison to std::vector, plain array is cheaper by one indirection. - Values lut[DATE_LUT_SIZE]; + Values lut[DATE_LUT_SIZE + 1]; - /// Year number after DATE_LUT_MIN_YEAR -> day num for start of year. - DayNum years_lut[DATE_LUT_YEARS]; + /// Year number after DATE_LUT_MIN_YEAR -> LUTIndex in lut for start of year. + LUTIndex years_lut[DATE_LUT_YEARS]; /// Year number after DATE_LUT_MIN_YEAR * month number starting at zero -> day num for first day of month - DayNum years_months_lut[DATE_LUT_YEARS * 12]; + LUTIndex years_months_lut[DATE_LUT_YEARS * 12]; /// UTC offset at beginning of the Unix epoch. The same as unix timestamp of 1970-01-01 00:00:00 local time. time_t offset_at_start_of_epoch; + time_t offset_at_start_of_lut; bool offset_is_whole_number_of_hours_everytime; + time_t time_offset_epoch; /// Time zone name. std::string time_zone; - - /// We can correctly process only timestamps that less DATE_LUT_MAX (i.e. up to 2105 year inclusively) - /// We don't care about overflow. - inline DayNum findIndex(time_t t) const + inline LUTIndex findIndex(time_t t) const { /// First guess. - DayNum guess(t / 86400); + const UInt32 guess = ((t / 86400) + daynum_offset_epoch) & date_lut_mask; /// UTC offset is from -12 to +14 in all known time zones. This requires checking only three indices. - - if ((guess == 0 || t >= lut[guess].date) && t < lut[DayNum(guess + 1)].date) - return guess; + if ((guess == daynum_offset_epoch || t >= lut[guess].date) && t < lut[UInt32(guess + 1)].date) + return LUTIndex{guess}; /// Time zones that have offset 0 from UTC do daylight saving time change (if any) towards increasing UTC offset (example: British Standard Time). - if (t >= lut[DayNum(guess + 1)].date) - return DayNum(guess + 1); + if (t >= lut[UInt32(guess + 1)].date) + return LUTIndex(guess + 1); - return DayNum(guess - 1); + if (lut[guess - 1].date <= t) + return LUTIndex(guess - 1); + return LUTIndex(guess - 2); } - inline const Values & find(time_t t) const + inline LUTIndex toLUTIndex(DayNum d) const { - return lut[findIndex(t)]; + return LUTIndex{(d + daynum_offset_epoch) & date_lut_mask}; + } + + inline LUTIndex toLUTIndex(ExtendedDayNum d) const + { + return LUTIndex{static_cast(d + daynum_offset_epoch) & date_lut_mask}; + } + + inline LUTIndex toLUTIndex(time_t t) const + { + return findIndex(t); + } + + inline LUTIndex toLUTIndex(LUTIndex i) const + { + return i; + } + +// template +// inline LUTIndex toLUTIndex(T t) const +// { +// return LUTIndex{static_cast(t) & date_lut_mask}; +// } + + template + inline const Values & find(V v) const + { + return lut[toLUTIndex(v)]; } public: const std::string & getTimeZone() const { return time_zone; } + // Methods only for unit-testing, it makes very little sense to use it from user code. + auto getOffsetAtStartOfEpoch() const { return offset_at_start_of_epoch; } + auto getOffsetIsWholNumberOfHoursEveryWhere() const { return offset_is_whole_number_of_hours_everytime; } + auto getTimeOffsetEpoch() const { return time_offset_epoch; } + auto getTimeOffsetAtStartOfLUT() const { return offset_at_start_of_lut; } + /// All functions below are thread-safe; arguments are not checked. - inline time_t toDate(time_t t) const { return find(t).date; } - inline unsigned toMonth(time_t t) const { return find(t).month; } - inline unsigned toQuarter(time_t t) const { return (find(t).month - 1) / 3 + 1; } - inline unsigned toYear(time_t t) const { return find(t).year; } - inline unsigned toDayOfWeek(time_t t) const { return find(t).day_of_week; } - inline unsigned toDayOfMonth(time_t t) const { return find(t).day_of_month; } + inline ExtendedDayNum toDayNum(ExtendedDayNum d) const + { + return d; + } + + template + inline ExtendedDayNum toDayNum(V v) const + { + return ExtendedDayNum{static_cast(toLUTIndex(v).toUnderType()) - daynum_offset_epoch}; + } /// Round down to start of monday. - inline time_t toFirstDayOfWeek(time_t t) const + template + inline time_t toFirstDayOfWeek(V v) const { - DayNum index = findIndex(t); - return lut[DayNum(index - (lut[index].day_of_week - 1))].date; + const auto i = toLUTIndex(v); + return lut[i - (lut[i].day_of_week - 1)].date; } - inline DayNum toFirstDayNumOfWeek(DayNum d) const + template + inline ExtendedDayNum toFirstDayNumOfWeek(V v) const { - return DayNum(d - (lut[d].day_of_week - 1)); - } - - inline DayNum toFirstDayNumOfWeek(time_t t) const - { - return toFirstDayNumOfWeek(toDayNum(t)); + const auto i = toLUTIndex(v); + return toDayNum(i - (lut[i].day_of_week - 1)); } /// Round down to start of month. - inline time_t toFirstDayOfMonth(time_t t) const + template + inline time_t toFirstDayOfMonth(V v) const { - DayNum index = findIndex(t); - return lut[index - (lut[index].day_of_month - 1)].date; + const auto i = toLUTIndex(v); + return lut[i - (lut[i].day_of_month - 1)].date; } - inline DayNum toFirstDayNumOfMonth(DayNum d) const + template + inline ExtendedDayNum toFirstDayNumOfMonth(V v) const { - return DayNum(d - (lut[d].day_of_month - 1)); + const auto i = toLUTIndex(v); + return toDayNum(i - (lut[i].day_of_month - 1)); } - inline DayNum toFirstDayNumOfMonth(time_t t) const - { - return toFirstDayNumOfMonth(toDayNum(t)); - } +// inline DayNum toFirstDayNumOfMonth(time_t t) const +// { +// return toFirstDayNumOfMonth(toDayNum(t)); +// } /// Round down to start of quarter. - inline DayNum toFirstDayNumOfQuarter(DayNum d) const + template + inline ExtendedDayNum toFirstDayNumOfQuarter(V v) const { - DayNum index = d; + return toDayNum(toFirstDayOfQuarterIndex(v)); + } + + template + inline LUTIndex toFirstDayOfQuarterIndex(V v) const + { + //return fromDayNum(toFirstDayNumOfQuarter(v)); + auto index = toLUTIndex(v); size_t month_inside_quarter = (lut[index].month - 1) % 3; index -= lut[index].day_of_month; @@ -175,17 +297,13 @@ public: --month_inside_quarter; } - return DayNum(index + 1); + return index + 1; } - inline DayNum toFirstDayNumOfQuarter(time_t t) const + template + inline time_t toFirstDayOfQuarter(V v) const { - return toFirstDayNumOfQuarter(toDayNum(t)); - } - - inline time_t toFirstDayOfQuarter(time_t t) const - { - return fromDayNum(toFirstDayNumOfQuarter(t)); + return toDate(toFirstDayOfQuarterIndex(v)); } /// Round down to start of year. @@ -194,48 +312,47 @@ public: return lut[years_lut[lut[findIndex(t)].year - DATE_LUT_MIN_YEAR]].date; } - inline DayNum toFirstDayNumOfYear(DayNum d) const + template + inline LUTIndex toFirstDayNumOfYearIndex(V v) const { - return years_lut[lut[d].year - DATE_LUT_MIN_YEAR]; + return years_lut[lut[toLUTIndex(v)].year - DATE_LUT_MIN_YEAR]; } - inline DayNum toFirstDayNumOfYear(time_t t) const + template + inline ExtendedDayNum toFirstDayNumOfYear(V v) const { - return toFirstDayNumOfYear(toDayNum(t)); + return toDayNum(toFirstDayNumOfYearIndex(v)); } inline time_t toFirstDayOfNextMonth(time_t t) const { - DayNum index = findIndex(t); + auto index = findIndex(t); index += 32 - lut[index].day_of_month; return lut[index - (lut[index].day_of_month - 1)].date; } inline time_t toFirstDayOfPrevMonth(time_t t) const { - DayNum index = findIndex(t); + auto index = findIndex(t); index -= lut[index].day_of_month; return lut[index - (lut[index].day_of_month - 1)].date; } - inline UInt8 daysInMonth(DayNum d) const + template + inline UInt8 daysInMonth(V v) const { - return lut[d].days_in_month; + const auto i = toLUTIndex(v); + return lut[i].days_in_month; } - inline UInt8 daysInMonth(time_t t) const - { - return find(t).days_in_month; - } - - inline UInt8 daysInMonth(UInt16 year, UInt8 month) const + inline UInt8 daysInMonth(Int16 year, UInt8 month) const { UInt16 idx = year - DATE_LUT_MIN_YEAR; if (unlikely(idx >= DATE_LUT_YEARS)) return 31; /// Implementation specific behaviour on overflow. /// 32 makes arithmetic more simple. - DayNum any_day_of_month = DayNum(years_lut[idx] + 32 * (month - 1)); + const auto any_day_of_month = years_lut[year - DATE_LUT_MIN_YEAR] + 32 * (month - 1); return lut[any_day_of_month].days_in_month; } @@ -243,37 +360,38 @@ public: */ inline time_t toDateAndShift(time_t t, Int32 days) const { - return lut[DayNum(findIndex(t) + days)].date; + return lut[findIndex(t) + days].date; } inline time_t toTime(time_t t) const { - DayNum index = findIndex(t); + auto index = findIndex(t); - if (unlikely(index == 0 || index > DATE_LUT_MAX_DAY_NUM)) + if (unlikely(index == daynum_offset_epoch || index > DATE_LUT_MAX_DAY_NUM)) return t + offset_at_start_of_epoch; time_t res = t - lut[index].date; - if (res >= lut[index].time_at_offset_change) - res += lut[index].amount_of_offset_change; + if (res >= lut[index].time_at_offset_change()) + res += lut[index].amount_of_offset_change(); return res - offset_at_start_of_epoch; /// Starting at 1970-01-01 00:00:00 local time. } inline unsigned toHour(time_t t) const { - DayNum index = findIndex(t); + auto index = findIndex(t); /// If it is overflow case, - /// then limit number of hours to avoid insane results like 1970-01-01 89:28:15 - if (unlikely(index == 0 || index > DATE_LUT_MAX_DAY_NUM)) + /// than limit number of hours to avoid insane results like 1970-01-01 89:28:15 + if (unlikely(index == daynum_offset_epoch || index > DATE_LUT_MAX_DAY_NUM)) return static_cast((t + offset_at_start_of_epoch) / 3600) % 24; time_t time = t - lut[index].date; - if (time >= lut[index].time_at_offset_change) - time += lut[index].amount_of_offset_change; + /// Data is cleaned to avoid possibility of underflow. + if (time >= lut[index].time_at_offset_change()) + time += lut[index].amount_of_offset_change(); unsigned res = time / 3600; return res <= 23 ? res : 0; @@ -286,24 +404,32 @@ public: */ inline time_t timezoneOffset(time_t t) const { - DayNum index = findIndex(t); + const auto index = findIndex(t); /// Calculate daylight saving offset first. /// Because the "amount_of_offset_change" in LUT entry only exists in the change day, it's costly to scan it from the very begin. /// but we can figure out all the accumulated offsets from 1970-01-01 to that day just by get the whole difference between lut[].date, /// and then, we can directly subtract multiple 86400s to get the real DST offsets for the leap seconds is not considered now. - time_t res = (lut[index].date - lut[0].date) % 86400; + time_t res = (lut[index].date - lut[daynum_offset_epoch].date) % 86400; /// As so far to know, the maximal DST offset couldn't be more than 2 hours, so after the modulo operation the remainder /// will sits between [-offset --> 0 --> offset] which respectively corresponds to moving clock forward or backward. res = res > 43200 ? (86400 - res) : (0 - res); /// Check if has a offset change during this day. Add the change when cross the line - if (lut[index].amount_of_offset_change != 0 && t >= lut[index].date + lut[index].time_at_offset_change) - res += lut[index].amount_of_offset_change; + if (lut[index].amount_of_offset_change() != 0 && t >= lut[index].date + lut[index].time_at_offset_change()) + res += lut[index].amount_of_offset_change(); return res + offset_at_start_of_epoch; } + static inline time_t toSecondsSinceTheDayStart(time_t t) + { + t %= 86400; + t = (t < 0 ? t + 86400 : t); + + return t; + } + /** Only for time zones with/when offset from UTC is multiple of five minutes. * This is true for all time zones: right now, all time zones have an offset that is multiple of 15 minutes. * @@ -314,13 +440,15 @@ public: * Also please note, that unix timestamp doesn't count "leap seconds": * each minute, with added or subtracted leap second, spans exactly 60 unix timestamps. */ - - inline unsigned toSecond(time_t t) const { return UInt32(t) % 60; } + inline unsigned toSecond(time_t t) const + { + return toSecondsSinceTheDayStart(t) % 60; + } inline unsigned toMinute(time_t t) const { if (offset_is_whole_number_of_hours_everytime) - return (UInt32(t) / 60) % 60; + return (toSecondsSinceTheDayStart(t) / 60) % 60; UInt32 date = find(t).date; return (UInt32(t) - date) / 60 % 60; @@ -348,80 +476,85 @@ public: * because the same calendar day starts/ends at different timestamps in different time zones) */ - inline DayNum toDayNum(time_t t) const { return findIndex(t); } - inline time_t fromDayNum(DayNum d) const { return lut[d].date; } +// inline DayNum toDayNum(time_t t) const { return DayNum{findIndex(t) - daynum_offset_epoch}; } +// inline ExtendedDayNum toExtendedDayNum(time_t t) const { return ExtendedDayNum{findIndex(t) - daynum_offset_epoch}; } + inline time_t fromDayNum(DayNum d) const { return lut[toLUTIndex(d)].date; } + inline time_t fromDayNum(ExtendedDayNum d) const { return lut[toLUTIndex(d)].date; } - inline time_t toDate(DayNum d) const { return lut[d].date; } - inline unsigned toMonth(DayNum d) const { return lut[d].month; } - inline unsigned toQuarter(DayNum d) const { return (lut[d].month - 1) / 3 + 1; } - inline unsigned toYear(DayNum d) const { return lut[d].year; } - inline unsigned toDayOfWeek(DayNum d) const { return lut[d].day_of_week; } - inline unsigned toDayOfMonth(DayNum d) const { return lut[d].day_of_month; } - inline unsigned toDayOfYear(DayNum d) const { return d + 1 - toFirstDayNumOfYear(d); } - - inline unsigned toDayOfYear(time_t t) const { return toDayOfYear(toDayNum(t)); } + template + inline time_t toDate(V v) const { return lut[toLUTIndex(v)].date; } + template + inline unsigned toMonth(V v) const { return lut[toLUTIndex(v)].month; } + template + inline unsigned toQuarter(V v) const { return (lut[toLUTIndex(v)].month - 1) / 3 + 1; } + template + inline Int16 toYear(V v) const { return lut[toLUTIndex(v)].year; } + template + inline unsigned toDayOfWeek(V v) const { return lut[toLUTIndex(v)].day_of_week; } + template + inline unsigned toDayOfMonth(V v) const { return lut[toLUTIndex(v)].day_of_month; } + template + inline unsigned toDayOfYear(V v) const + { + // TODO: different overload for ExtendedDayNum + const auto i = toLUTIndex(v); + return i + 1 - toFirstDayNumOfYearIndex(i); + } /// Number of week from some fixed moment in the past. Week begins at monday. /// (round down to monday and divide DayNum by 7; we made an assumption, /// that in domain of the function there was no weeks with any other number of days than 7) - inline unsigned toRelativeWeekNum(DayNum d) const + template + inline unsigned toRelativeWeekNum(V v) const { + const auto i = toLUTIndex(v); /// We add 8 to avoid underflow at beginning of unix epoch. - return (d + 8 - toDayOfWeek(d)) / 7; - } - - inline unsigned toRelativeWeekNum(time_t t) const - { - return toRelativeWeekNum(toDayNum(t)); + return toDayNum(i + 8 - toDayOfWeek(i)) / 7; } /// Get year that contains most of the current week. Week begins at monday. - inline unsigned toISOYear(DayNum d) const + template + inline unsigned toISOYear(V v) const { + const auto i = toLUTIndex(v); /// That's effectively the year of thursday of current week. - return toYear(DayNum(d + 4 - toDayOfWeek(d))); - } - - inline unsigned toISOYear(time_t t) const - { - return toISOYear(toDayNum(t)); + return toYear(toLUTIndex(i + 4 - toDayOfWeek(i))); } /// ISO year begins with a monday of the week that is contained more than by half in the corresponding calendar year. /// Example: ISO year 2019 begins at 2018-12-31. And ISO year 2017 begins at 2017-01-02. /// https://en.wikipedia.org/wiki/ISO_week_date - inline DayNum toFirstDayNumOfISOYear(DayNum d) const + template + inline LUTIndex toFirstDayNumOfISOYearIndex(V v) const { - auto iso_year = toISOYear(d); + const auto i = toLUTIndex(v); + auto iso_year = toISOYear(i); - DayNum first_day_of_year = years_lut[iso_year - DATE_LUT_MIN_YEAR]; + const auto first_day_of_year = years_lut[iso_year - DATE_LUT_MIN_YEAR]; auto first_day_of_week_of_year = lut[first_day_of_year].day_of_week; - return DayNum(first_day_of_week_of_year <= 4 + return LUTIndex{first_day_of_week_of_year <= 4 ? first_day_of_year + 1 - first_day_of_week_of_year - : first_day_of_year + 8 - first_day_of_week_of_year); + : first_day_of_year + 8 - first_day_of_week_of_year}; } - inline DayNum toFirstDayNumOfISOYear(time_t t) const + template + inline ExtendedDayNum toFirstDayNumOfISOYear(V v) const { - return toFirstDayNumOfISOYear(toDayNum(t)); + return toDayNum(toFirstDayNumOfISOYearIndex(v)); } inline time_t toFirstDayOfISOYear(time_t t) const { - return fromDayNum(toFirstDayNumOfISOYear(t)); + return lut[toFirstDayNumOfISOYearIndex(t)].date; } /// ISO 8601 week number. Week begins at monday. /// The week number 1 is the first week in year that contains 4 or more days (that's more than half). - inline unsigned toISOWeek(DayNum d) const + template + inline unsigned toISOWeek(V v) const { - return 1 + DayNum(toFirstDayNumOfWeek(d) - toFirstDayNumOfISOYear(d)) / 7; - } - - inline unsigned toISOWeek(time_t t) const - { - return toISOWeek(toDayNum(t)); + return 1 + (toFirstDayNumOfWeek(v) - toFirstDayNumOfISOYear(v)) / 7; } /* @@ -457,30 +590,33 @@ public: Otherwise it is the last week of the previous year, and the next week is week 1. */ - inline YearWeek toYearWeek(DayNum d, UInt8 week_mode) const + template + inline YearWeek toYearWeek(V v, UInt8 week_mode) const { - bool newyear_day_mode = week_mode & static_cast(WeekModeFlag::NEWYEAR_DAY); + const bool newyear_day_mode = week_mode & static_cast(WeekModeFlag::NEWYEAR_DAY); week_mode = check_week_mode(week_mode); - bool monday_first_mode = week_mode & static_cast(WeekModeFlag::MONDAY_FIRST); + const bool monday_first_mode = week_mode & static_cast(WeekModeFlag::MONDAY_FIRST); bool week_year_mode = week_mode & static_cast(WeekModeFlag::YEAR); - bool first_weekday_mode = week_mode & static_cast(WeekModeFlag::FIRST_WEEKDAY); + const bool first_weekday_mode = week_mode & static_cast(WeekModeFlag::FIRST_WEEKDAY); + + const auto i = toLUTIndex(v); // Calculate week number of WeekModeFlag::NEWYEAR_DAY mode if (newyear_day_mode) { - return toYearWeekOfNewyearMode(d, monday_first_mode); + return toYearWeekOfNewyearMode(i, monday_first_mode); } - YearWeek yw(toYear(d), 0); + YearWeek yw(toYear(i), 0); UInt16 days = 0; - UInt16 daynr = makeDayNum(yw.first, toMonth(d), toDayOfMonth(d)); - UInt16 first_daynr = makeDayNum(yw.first, 1, 1); + const auto daynr = makeDayNum(yw.first, toMonth(i), toDayOfMonth(i)); + auto first_daynr = makeDayNum(yw.first, 1, 1); // 0 for monday, 1 for tuesday ... // get weekday from first day in year. - UInt16 weekday = calc_weekday(DayNum(first_daynr), !monday_first_mode); + UInt16 weekday = calc_weekday(first_daynr, !monday_first_mode); - if (toMonth(d) == 1 && toDayOfMonth(d) <= static_cast(7 - weekday)) + if (toMonth(i) == 1 && toDayOfMonth(i) <= static_cast(7 - weekday)) { if (!week_year_mode && ((first_weekday_mode && weekday != 0) || (!first_weekday_mode && weekday >= 4))) return yw; @@ -511,30 +647,34 @@ public: /// Calculate week number of WeekModeFlag::NEWYEAR_DAY mode /// The week number 1 is the first week in year that contains January 1, - inline YearWeek toYearWeekOfNewyearMode(DayNum d, bool monday_first_mode) const + template + inline YearWeek toYearWeekOfNewyearMode(V v, bool monday_first_mode) const { YearWeek yw(0, 0); UInt16 offset_day = monday_first_mode ? 0U : 1U; + const auto i = LUTIndex(v); + // Checking the week across the year - yw.first = toYear(DayNum(d + 7 - toDayOfWeek(DayNum(d + offset_day)))); + yw.first = toYear(i + 7 - toDayOfWeek(i + offset_day)); - DayNum first_day = makeDayNum(yw.first, 1, 1); - DayNum this_day = d; + auto first_day = makeLUTIndex(yw.first, 1, 1); + auto this_day = i; + //TODO: do not perform calculations in terms of DayNum, since that would under/overflow for extended range. if (monday_first_mode) { // Rounds down a date to the nearest Monday. first_day = toFirstDayNumOfWeek(first_day); - this_day = toFirstDayNumOfWeek(d); + this_day = toFirstDayNumOfWeek(i); } else { // Rounds down a date to the nearest Sunday. if (toDayOfWeek(first_day) != 7) - first_day = DayNum(first_day - toDayOfWeek(first_day)); - if (toDayOfWeek(d) != 7) - this_day = DayNum(d - toDayOfWeek(d)); + first_day = ExtendedDayNum(first_day - toDayOfWeek(first_day)); + if (toDayOfWeek(i) != 7) + this_day = ExtendedDayNum(i - toDayOfWeek(i)); } yw.second = (this_day - first_day) / 7 + 1; return yw; @@ -543,16 +683,17 @@ public: /** * get first day of week with week_mode, return Sunday or Monday */ - inline DayNum toFirstDayNumOfWeek(DayNum d, UInt8 week_mode) const + template + inline ExtendedDayNum toFirstDayNumOfWeek(V v, UInt8 week_mode) const { bool monday_first_mode = week_mode & static_cast(WeekModeFlag::MONDAY_FIRST); if (monday_first_mode) { - return toFirstDayNumOfWeek(d); + return toFirstDayNumOfWeek(v); } else { - return (toDayOfWeek(d) != 7) ? DayNum(d - toDayOfWeek(d)) : d; + return (toDayOfWeek(v) != 7) ? ExtendedDayNum(v - toDayOfWeek(v)) : toDayNum(v); } } @@ -568,39 +709,35 @@ public: /** Calculate weekday from d. * Returns 0 for monday, 1 for tuesday... */ - inline unsigned calc_weekday(DayNum d, bool sunday_first_day_of_week) const + template + inline unsigned calc_weekday(V v, bool sunday_first_day_of_week) const { + const auto i = toLUTIndex(v); if (!sunday_first_day_of_week) - return toDayOfWeek(d) - 1; + return toDayOfWeek(i) - 1; else - return toDayOfWeek(DayNum(d + 1)) - 1; + return toDayOfWeek(i + 1) - 1; } /// Calculate days in one year. - inline unsigned calc_days_in_year(UInt16 year) const + inline unsigned calc_days_in_year(Int32 year) const { return ((year & 3) == 0 && (year % 100 || (year % 400 == 0 && year)) ? 366 : 365); } /// Number of month from some fixed moment in the past (year * 12 + month) - inline unsigned toRelativeMonthNum(DayNum d) const + template + inline unsigned toRelativeMonthNum(V v) const { - return lut[d].year * 12 + lut[d].month; + const auto i = toLUTIndex(v); + return lut[i].year * 12 + lut[i].month; } - inline unsigned toRelativeMonthNum(time_t t) const + template + inline unsigned toRelativeQuarterNum(V v) const { - return toRelativeMonthNum(toDayNum(t)); - } - - inline unsigned toRelativeQuarterNum(DayNum d) const - { - return lut[d].year * 4 + (lut[d].month - 1) / 3; - } - - inline unsigned toRelativeQuarterNum(time_t t) const - { - return toRelativeQuarterNum(toDayNum(t)); + const auto i = toLUTIndex(v); + return lut[i].year * 4 + (lut[i].month - 1) / 3; } /// We count all hour-length intervals, unrelated to offset changes. @@ -614,9 +751,10 @@ public: return (t + 86400 - offset_at_start_of_epoch) / 3600; } - inline time_t toRelativeHourNum(DayNum d) const + template + inline time_t toRelativeHourNum(V v) const { - return toRelativeHourNum(lut[d].date); + return toRelativeHourNum(lut[toLUTIndex(v)].date); } inline time_t toRelativeMinuteNum(time_t t) const @@ -624,48 +762,52 @@ public: return t / 60; } - inline time_t toRelativeMinuteNum(DayNum d) const + template + inline time_t toRelativeMinuteNum(V v) const { - return toRelativeMinuteNum(lut[d].date); + return toRelativeMinuteNum(lut[toLUTIndex(v)].date); } - inline DayNum toStartOfYearInterval(DayNum d, UInt64 years) const + template + inline ExtendedDayNum toStartOfYearInterval(V v, UInt64 years) const { if (years == 1) - return toFirstDayNumOfYear(d); - return years_lut[(lut[d].year - DATE_LUT_MIN_YEAR) / years * years]; + return toFirstDayNumOfYear(v); + + const auto i = toLUTIndex(v); + return toDayNum(years_lut[lut[i].year / years * years - DATE_LUT_MIN_YEAR]); } - inline DayNum toStartOfQuarterInterval(DayNum d, UInt64 quarters) const + inline ExtendedDayNum toStartOfQuarterInterval(ExtendedDayNum d, UInt64 quarters) const { if (quarters == 1) return toFirstDayNumOfQuarter(d); return toStartOfMonthInterval(d, quarters * 3); } - inline DayNum toStartOfMonthInterval(DayNum d, UInt64 months) const + inline ExtendedDayNum toStartOfMonthInterval(ExtendedDayNum d, UInt64 months) const { if (months == 1) return toFirstDayNumOfMonth(d); - const auto & date = lut[d]; + const auto & date = lut[toLUTIndex(d)]; UInt32 month_total_index = (date.year - DATE_LUT_MIN_YEAR) * 12 + date.month - 1; - return years_months_lut[month_total_index / months * months]; + return toDayNum(years_months_lut[month_total_index / months * months]); } - inline DayNum toStartOfWeekInterval(DayNum d, UInt64 weeks) const + inline ExtendedDayNum toStartOfWeekInterval(ExtendedDayNum d, UInt64 weeks) const { if (weeks == 1) return toFirstDayNumOfWeek(d); UInt64 days = weeks * 7; // January 1st 1970 was Thursday so we need this 4-days offset to make weeks start on Monday. - return DayNum(4 + (d - 4) / days * days); + return ExtendedDayNum(4 + (d - 4) / days * days); } - inline time_t toStartOfDayInterval(DayNum d, UInt64 days) const + inline time_t toStartOfDayInterval(ExtendedDayNum d, UInt64 days) const { if (days == 1) return toDate(d); - return lut[d / days * days].date; + return lut[toLUTIndex(ExtendedDayNum(d / days * days))].date; } inline time_t toStartOfHourInterval(time_t t, UInt64 hours) const @@ -694,33 +836,41 @@ public: return t / seconds * seconds; } - /// Create DayNum from year, month, day of month. - inline DayNum makeDayNum(UInt16 year, UInt8 month, UInt8 day_of_month) const + inline LUTIndex makeLUTIndex(Int16 year, UInt8 month, UInt8 day_of_month) const { if (unlikely(year < DATE_LUT_MIN_YEAR || year > DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31)) - return DayNum(0); // TODO (nemkov, DateTime64 phase 2): implement creating real date for year outside of LUT range. + return LUTIndex(0); - // The day after 2106-02-07 will not stored fully as struct Values, so just overflow it as 0 - if (unlikely(year == DATE_LUT_MAX_YEAR && (month > 2 || (month == 2 && day_of_month > 7)))) - return DayNum(0); - - return DayNum(years_months_lut[(year - DATE_LUT_MIN_YEAR) * 12 + month - 1] + day_of_month - 1); + return LUTIndex{years_months_lut[(year - DATE_LUT_MIN_YEAR) * 12 + month - 1] + day_of_month - 1}; } - inline time_t makeDate(UInt16 year, UInt8 month, UInt8 day_of_month) const + /// Create DayNum from year, month, day of month. + inline ExtendedDayNum makeDayNum(Int16 year, UInt8 month, UInt8 day_of_month) const { - return lut[makeDayNum(year, month, day_of_month)].date; + if (unlikely(year < DATE_LUT_MIN_YEAR || year > DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31)) + return ExtendedDayNum(0); + + // The day after 2283 are not stored fully as struct Values, so just overflow it as 0 + if (unlikely(year > DATE_LUT_MAX_YEAR)) + return ExtendedDayNum(0); + + return toDayNum(makeLUTIndex(year, month, day_of_month)); + } + + inline time_t makeDate(Int16 year, UInt8 month, UInt8 day_of_month) const + { + return lut[makeLUTIndex(year, month, day_of_month)].date; } /** Does not accept daylight saving time as argument: in case of ambiguity, it choose greater timestamp. */ - inline time_t makeDateTime(UInt16 year, UInt8 month, UInt8 day_of_month, UInt8 hour, UInt8 minute, UInt8 second) const + inline time_t makeDateTime(Int16 year, UInt8 month, UInt8 day_of_month, UInt8 hour, UInt8 minute, UInt8 second) const { - size_t index = makeDayNum(year, month, day_of_month); + size_t index = makeLUTIndex(year, month, day_of_month); UInt32 time_offset = hour * 3600 + minute * 60 + second; - if (time_offset >= lut[index].time_at_offset_change) - time_offset -= lut[index].amount_of_offset_change; + if (time_offset >= lut[index].time_at_offset_change()) + time_offset -= lut[index].amount_of_offset_change(); UInt32 res = lut[index].date + time_offset; @@ -730,30 +880,20 @@ public: return res; } - inline const Values & getValues(DayNum d) const { return lut[d]; } - inline const Values & getValues(time_t t) const { return lut[findIndex(t)]; } + template + inline const Values & getValues(V v) const { return lut[toLUTIndex(v)]; } - inline UInt32 toNumYYYYMM(time_t t) const + template + inline UInt32 toNumYYYYMM(V v) const { - const Values & values = find(t); + const Values & values = getValues(v); return values.year * 100 + values.month; } - inline UInt32 toNumYYYYMM(DayNum d) const + template + inline UInt32 toNumYYYYMMDD(V v) const { - const Values & values = lut[d]; - return values.year * 100 + values.month; - } - - inline UInt32 toNumYYYYMMDD(time_t t) const - { - const Values & values = find(t); - return values.year * 10000 + values.month * 100 + values.day_of_month; - } - - inline UInt32 toNumYYYYMMDD(DayNum d) const - { - const Values & values = lut[d]; + const Values & values = getValues(v); return values.year * 10000 + values.month * 100 + values.day_of_month; } @@ -762,7 +902,7 @@ public: return makeDate(num / 10000, num / 100 % 100, num % 100); } - inline DayNum YYYYMMDDToDayNum(UInt32 num) const + inline ExtendedDayNum YYYYMMDDToDayNum(UInt32 num) const { return makeDayNum(num / 10000, num / 100 % 100, num % 100); } @@ -796,13 +936,14 @@ public: inline NO_SANITIZE_UNDEFINED time_t addDays(time_t t, Int64 delta) const { - DayNum index = findIndex(t); + auto index = findIndex(t); time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); index += delta; + index &= date_lut_mask; - if (time_offset >= lut[index].time_at_offset_change) - time_offset -= lut[index].amount_of_offset_change; + if (time_offset >= lut[index].time_at_offset_change()) + time_offset -= lut[index].amount_of_offset_change(); return lut[index].date + time_offset; } @@ -812,7 +953,7 @@ public: return addDays(t, delta * 7); } - inline UInt8 saturateDayOfMonth(UInt16 year, UInt8 month, UInt8 day_of_month) const + inline UInt8 saturateDayOfMonth(Int16 year, UInt8 month, UInt8 day_of_month) const { if (likely(day_of_month <= 28)) return day_of_month; @@ -825,23 +966,10 @@ public: return day_of_month; } - /// If resulting month has less deys than source month, then saturation can happen. - /// Example: 31 Aug + 1 month = 30 Sep. - inline time_t addMonths(time_t t, Int64 delta) const + template + inline LUTIndex addMonthsIndex(V v, Int64 delta) const { - DayNum result_day = addMonths(toDayNum(t), delta); - - time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); - - if (time_offset >= lut[result_day].time_at_offset_change) - time_offset -= lut[result_day].amount_of_offset_change; - - return lut[result_day].date + time_offset; - } - - inline NO_SANITIZE_UNDEFINED DayNum addMonths(DayNum d, Int64 delta) const - { - const Values & values = lut[d]; + const Values & values = lut[toLUTIndex(v)]; Int64 month = static_cast(values.month) + delta; @@ -851,7 +979,7 @@ public: month = ((month - 1) % 12) + 1; auto day_of_month = saturateDayOfMonth(year, month, values.day_of_month); - return makeDayNum(year, month, day_of_month); + return makeLUTIndex(year, month, day_of_month); } else { @@ -859,36 +987,43 @@ public: month = 12 - (-month % 12); auto day_of_month = saturateDayOfMonth(year, month, values.day_of_month); - return makeDayNum(year, month, day_of_month); + return makeLUTIndex(year, month, day_of_month); } } - inline NO_SANITIZE_UNDEFINED time_t addQuarters(time_t t, Int64 delta) const + /// If resulting month has less deys than source month, then saturation can happen. + /// Example: 31 Aug + 1 month = 30 Sep. + inline time_t NO_SANITIZE_UNDEFINED addMonths(time_t t, Int64 delta) const { - return addMonths(t, delta * 3); - } - - inline NO_SANITIZE_UNDEFINED DayNum addQuarters(DayNum d, Int64 delta) const - { - return addMonths(d, delta * 3); - } - - /// Saturation can occur if 29 Feb is mapped to non-leap year. - inline NO_SANITIZE_UNDEFINED time_t addYears(time_t t, Int64 delta) const - { - DayNum result_day = addYears(toDayNum(t), delta); + const auto result_day = addMonthsIndex(t, delta); time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); - if (time_offset >= lut[result_day].time_at_offset_change) - time_offset -= lut[result_day].amount_of_offset_change; + if (time_offset >= lut[result_day].time_at_offset_change()) + time_offset -= lut[result_day].amount_of_offset_change(); return lut[result_day].date + time_offset; } - inline NO_SANITIZE_UNDEFINED DayNum addYears(DayNum d, Int64 delta) const + inline ExtendedDayNum NO_SANITIZE_UNDEFINED addMonths(ExtendedDayNum d, Int64 delta) const { - const Values & values = lut[d]; + return toDayNum(addMonthsIndex(d, delta)); + } + + inline time_t NO_SANITIZE_UNDEFINED addQuarters(time_t t, Int64 delta) const + { + return addMonths(t, delta * 3); + } + + inline ExtendedDayNum addQuarters(ExtendedDayNum d, Int64 delta) const + { + return addMonths(d, delta * 3); + } + + template + inline LUTIndex NO_SANITIZE_UNDEFINED addYearsIndex(V v, Int64 delta) const + { + const Values & values = lut[toLUTIndex(v)]; auto year = values.year + delta; auto month = values.month; @@ -898,13 +1033,31 @@ public: if (unlikely(day_of_month == 29 && month == 2)) day_of_month = saturateDayOfMonth(year, month, day_of_month); - return makeDayNum(year, month, day_of_month); + return makeLUTIndex(year, month, day_of_month); + } + + /// Saturation can occur if 29 Feb is mapped to non-leap year. + inline time_t addYears(time_t t, Int64 delta) const + { + auto result_day = addYearsIndex(t, delta); + + time_t time_offset = toHour(t) * 3600 + toMinute(t) * 60 + toSecond(t); + + if (time_offset >= lut[result_day].time_at_offset_change()) + time_offset -= lut[result_day].amount_of_offset_change(); + + return lut[result_day].date + time_offset; + } + + inline ExtendedDayNum addYears(ExtendedDayNum d, Int64 delta) const + { + return toDayNum(addYearsIndex(d, delta)); } inline std::string timeToString(time_t t) const { - const Values & values = find(t); + const Values & values = getValues(t); std::string s {"0000-00-00 00:00:00"}; @@ -933,7 +1086,7 @@ public: inline std::string dateToString(time_t t) const { - const Values & values = find(t); + const Values & values = getValues(t); std::string s {"0000-00-00"}; @@ -949,9 +1102,9 @@ public: return s; } - inline std::string dateToString(DayNum d) const + inline std::string dateToString(ExtendedDayNum d) const { - const Values & values = lut[d]; + const Values & values = getValues(d); std::string s {"0000-00-00"}; diff --git a/base/common/DayNum.h b/base/common/DayNum.h index a4ef0c43b69..5cf4d4635c8 100644 --- a/base/common/DayNum.h +++ b/base/common/DayNum.h @@ -7,3 +7,8 @@ * See DateLUTImpl for usage examples. */ STRONG_TYPEDEF(UInt16, DayNum) + +/** Represent number of days since 1970-01-01 but in extended range, + * for dates before 1970-01-01 and after 2105 + */ +STRONG_TYPEDEF(Int32, ExtendedDayNum) diff --git a/base/common/LocalDate.h b/base/common/LocalDate.h index e5ebe877bc5..7e1260c1385 100644 --- a/base/common/LocalDate.h +++ b/base/common/LocalDate.h @@ -105,7 +105,8 @@ public: DayNum getDayNum() const { - return DateLUT::instance().makeDayNum(m_year, m_month, m_day); + const auto & lut = DateLUT::instance(); + return DayNum(lut.makeDayNum(m_year, m_month, m_day).toUnderType()); } operator DayNum() const diff --git a/base/common/strong_typedef.h b/base/common/strong_typedef.h index d9850a25c37..77b83bfa6e5 100644 --- a/base/common/strong_typedef.h +++ b/base/common/strong_typedef.h @@ -12,6 +12,7 @@ private: T t; public: + using UnderlyingType = T; template ::type> explicit StrongTypedef(const T & t_) : t(t_) {} template ::type> diff --git a/base/common/tests/CMakeLists.txt b/base/common/tests/CMakeLists.txt index b7082ee9900..b335b302cb0 100644 --- a/base/common/tests/CMakeLists.txt +++ b/base/common/tests/CMakeLists.txt @@ -16,7 +16,9 @@ target_link_libraries (realloc-perf PRIVATE common) add_check(local_date_time_comparison) if(USE_GTEST) - add_executable(unit_tests_libcommon gtest_json_test.cpp gtest_strong_typedef.cpp gtest_find_symbols.cpp) + add_executable(unit_tests_libcommon gtest_json_test.cpp gtest_strong_typedef.cpp gtest_find_symbols.cpp gtest_DateLutImpl.cpp + ${CMAKE_BINARY_DIR}/src/Storages/System/StorageSystemTimeZones.generated.cpp + ) target_link_libraries(unit_tests_libcommon PRIVATE common ${GTEST_MAIN_LIBRARIES} ${GTEST_LIBRARIES}) add_check(unit_tests_libcommon) endif() diff --git a/base/common/tests/gtest_DateLutImpl.cpp b/base/common/tests/gtest_DateLutImpl.cpp new file mode 100644 index 00000000000..395e2eddb00 --- /dev/null +++ b/base/common/tests/gtest_DateLutImpl.cpp @@ -0,0 +1,515 @@ +#include +#include + +#include + +#include +#include + +/// For the expansion of gtest macros. +#if defined(__clang__) + #pragma clang diagnostic ignored "-Wused-but-marked-unused" +#endif + +// All timezones present at build time and embedded into CH binary. +extern const char * auto_time_zones[]; + +namespace +{ + +cctz::civil_day YYYYMMDDToDay(unsigned value) +{ + return cctz::civil_day( + value / 10000, // year + (value % 10000) / 100, // month + value % 100); // day +} + +cctz::civil_second YYYYMMDDHMMSSToSecond(std::uint64_t value) +{ + return cctz::civil_second( + value / 10000000000, + value / 100000000 % 100, + value / 1000000 % 100, + value / 10000 % 100, + value / 100 % 100, + value % 100); +} + + +std::vector allTimezones() +{ + std::vector result; + + auto timezone_name = auto_time_zones; + while (*timezone_name) + { + result.push_back(*timezone_name); + ++timezone_name; + } + + return result; +} + +struct FailuresCount +{ + size_t non_fatal = 0; + size_t fatal = 0; + size_t total = 0; +}; + +FailuresCount countFailures(const ::testing::TestResult & test_result) +{ + FailuresCount failures{0, 0, 0}; + const size_t count = test_result.total_part_count(); + for (size_t i = 0; i < count; ++i) + { + const auto & part = test_result.GetTestPartResult(i); + if (part.nonfatally_failed()) + { + ++failures.non_fatal; + ++failures.total; + } + if (part.fatally_failed()) + { + ++failures.fatal; + ++failures.total; + } + } + + return failures; +} + +} + +TEST(YYYYMMDDToDay, Test) +{ + std::cerr << YYYYMMDDHMMSSToSecond(19700101'00'00'00) << std::endl; +} + +TEST(DateLUTTest, TimeValuesInMiddleOfRange) +{ + const DateLUTImpl lut("Europe/Minsk"); + const time_t time = 1568650811; // 2019-09-16 19:20:11 (Monday) + + EXPECT_EQ(lut.getTimeZone(), "Europe/Minsk"); + EXPECT_EQ(lut.getOffsetAtStartOfEpoch(), 3600*3); // UTC-3 + + EXPECT_EQ(lut.toDate(time), 1568581200); + EXPECT_EQ(lut.toMonth(time), 9); + EXPECT_EQ(lut.toQuarter(time), 3); + EXPECT_EQ(lut.toYear(time), 2019); + EXPECT_EQ(lut.toDayOfMonth(time), 16); + + EXPECT_EQ(lut.toFirstDayOfWeek(time), 1568581200 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfWeek(time), DayNum(18155) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfMonth(time), 1567285200 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfMonth(time), DayNum(18140) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayNumOfQuarter(time), DayNum(18078) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfQuarter(time), 1561928400 /*time_t*/); + EXPECT_EQ(lut.toFirstDayOfYear(time), 1546290000 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfYear(time), DayNum(17897) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfNextMonth(time), 1569877200 /*time_t*/); + EXPECT_EQ(lut.toFirstDayOfPrevMonth(time), 1564606800 /*time_t*/); + EXPECT_EQ(lut.daysInMonth(time), 30 /*UInt8*/); + EXPECT_EQ(lut.toDateAndShift(time, 10), 1569445200 /*time_t*/); + EXPECT_EQ(lut.toTime(time), 58811 /*time_t*/); + EXPECT_EQ(lut.toHour(time), 19 /*unsigned*/); + EXPECT_EQ(lut.toSecond(time), 11 /*unsigned*/); + EXPECT_EQ(lut.toMinute(time), 20 /*unsigned*/); + EXPECT_EQ(lut.toStartOfMinute(time), 1568650800 /*time_t*/); + EXPECT_EQ(lut.toStartOfFiveMinute(time), 1568650800 /*time_t*/); + EXPECT_EQ(lut.toStartOfFifteenMinutes(time), 1568650500 /*time_t*/); + EXPECT_EQ(lut.toStartOfTenMinutes(time), 1568650800 /*time_t*/); + EXPECT_EQ(lut.toStartOfHour(time), 1568649600 /*time_t*/); + EXPECT_EQ(lut.toDayNum(time), DayNum(18155) /*DayNum*/); + EXPECT_EQ(lut.toDayOfYear(time), 259 /*unsigned*/); + EXPECT_EQ(lut.toRelativeWeekNum(time), 2594 /*unsigned*/); + EXPECT_EQ(lut.toISOYear(time), 2019 /*unsigned*/); + EXPECT_EQ(lut.toFirstDayNumOfISOYear(time), DayNum(17896) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfISOYear(time), 1546203600 /*time_t*/); + EXPECT_EQ(lut.toISOWeek(time), 38 /*unsigned*/); + EXPECT_EQ(lut.toRelativeMonthNum(time), 24237 /*unsigned*/); + EXPECT_EQ(lut.toRelativeQuarterNum(time), 8078 /*unsigned*/); + EXPECT_EQ(lut.toRelativeHourNum(time), 435736 /*time_t*/); + EXPECT_EQ(lut.toRelativeMinuteNum(time), 26144180 /*time_t*/); + EXPECT_EQ(lut.toStartOfHourInterval(time, 5), 1568646000 /*time_t*/); + EXPECT_EQ(lut.toStartOfMinuteInterval(time, 6), 1568650680 /*time_t*/); + EXPECT_EQ(lut.toStartOfSecondInterval(time, 7), 1568650811 /*time_t*/); + EXPECT_EQ(lut.toNumYYYYMM(time), 201909 /*UInt32*/); + EXPECT_EQ(lut.toNumYYYYMMDD(time), 20190916 /*UInt32*/); + EXPECT_EQ(lut.toNumYYYYMMDDhhmmss(time), 20190916192011 /*UInt64*/); + EXPECT_EQ(lut.addDays(time, 100), 1577290811 /*time_t*/); + EXPECT_EQ(lut.addWeeks(time, 100), 1629130811 /*time_t*/); + EXPECT_EQ(lut.addMonths(time, 100), 1831652411 /*time_t*/); + EXPECT_EQ(lut.addQuarters(time, 100), 2357655611 /*time_t*/); + EXPECT_EQ(lut.addYears(time, 10), 1884270011 /*time_t*/); + EXPECT_EQ(lut.timeToString(time), "2019-09-16 19:20:11" /*std::string*/); + EXPECT_EQ(lut.dateToString(time), "2019-09-16" /*std::string*/); +} + + +TEST(DateLUTTest, TimeValuesAtLeftBoderOfRange) +{ + const DateLUTImpl lut("UTC"); + const time_t time = 0; // 1970-01-01 00:00:00 (Thursday) + + EXPECT_EQ(lut.getTimeZone(), "UTC"); + + EXPECT_EQ(lut.toDate(time), 0); + EXPECT_EQ(lut.toMonth(time), 1); + EXPECT_EQ(lut.toQuarter(time), 1); + EXPECT_EQ(lut.toYear(time), 1970); + EXPECT_EQ(lut.toDayOfMonth(time), 1); + + EXPECT_EQ(lut.toFirstDayOfWeek(time), -259200 /*time_t*/); // 1969-12-29 00:00:00 + EXPECT_EQ(lut.toFirstDayNumOfWeek(time), ExtendedDayNum(-3) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfMonth(time), 0 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfMonth(time), DayNum(0) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayNumOfQuarter(time), DayNum(0) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfQuarter(time), 0 /*time_t*/); + EXPECT_EQ(lut.toFirstDayOfYear(time), 0 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfYear(time), DayNum(0) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfNextMonth(time), 2678400 /*time_t*/); + EXPECT_EQ(lut.toFirstDayOfPrevMonth(time), -2678400 /*time_t*/); // 1969-12-01 00:00:00 + EXPECT_EQ(lut.daysInMonth(time), 31 /*UInt8*/); + EXPECT_EQ(lut.toDateAndShift(time, 10), 864000 /*time_t*/); + EXPECT_EQ(lut.toTime(time), 0 /*time_t*/); + EXPECT_EQ(lut.toHour(time), 0 /*unsigned*/); + EXPECT_EQ(lut.toSecond(time), 0 /*unsigned*/); + EXPECT_EQ(lut.toMinute(time), 0 /*unsigned*/); + EXPECT_EQ(lut.toStartOfMinute(time), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfFiveMinute(time), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfFifteenMinutes(time), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfTenMinutes(time), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfHour(time), 0 /*time_t*/); + EXPECT_EQ(lut.toDayNum(time), DayNum(0) /*DayNum*/); + EXPECT_EQ(lut.toDayOfYear(time), 1 /*unsigned*/); + EXPECT_EQ(lut.toRelativeWeekNum(time), 0 /*unsigned*/); + EXPECT_EQ(lut.toISOYear(time), 1970 /*unsigned*/); + EXPECT_EQ(lut.toFirstDayNumOfISOYear(time), ExtendedDayNum(-3) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfISOYear(time), -259200 /*time_t*/); // 1969-12-29 00:00:00 + EXPECT_EQ(lut.toISOWeek(time), 1 /*unsigned*/); + EXPECT_EQ(lut.toRelativeMonthNum(time), 23641 /*unsigned*/); // ? + EXPECT_EQ(lut.toRelativeQuarterNum(time), 7880 /*unsigned*/); // ? + EXPECT_EQ(lut.toRelativeHourNum(time), 0 /*time_t*/); + EXPECT_EQ(lut.toRelativeMinuteNum(time), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfHourInterval(time, 5), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfMinuteInterval(time, 6), 0 /*time_t*/); + EXPECT_EQ(lut.toStartOfSecondInterval(time, 7), 0 /*time_t*/); + EXPECT_EQ(lut.toNumYYYYMM(time), 197001 /*UInt32*/); + EXPECT_EQ(lut.toNumYYYYMMDD(time), 19700101 /*UInt32*/); + EXPECT_EQ(lut.toNumYYYYMMDDhhmmss(time), 19700101000000 /*UInt64*/); + EXPECT_EQ(lut.addDays(time, 100), 8640000 /*time_t*/); + EXPECT_EQ(lut.addWeeks(time, 100), 60480000 /*time_t*/); + EXPECT_EQ(lut.addMonths(time, 100), 262828800 /*time_t*/); + EXPECT_EQ(lut.addQuarters(time, 100), 788918400 /*time_t*/); + EXPECT_EQ(lut.addYears(time, 10), 315532800 /*time_t*/); + EXPECT_EQ(lut.timeToString(time), "1970-01-01 00:00:00" /*std::string*/); + EXPECT_EQ(lut.dateToString(time), "1970-01-01" /*std::string*/); +} + +TEST(DateLUTTest, TimeValuesAtRightBoderOfRangeOfOLDLut) +{ + // Value is at the right border of the OLD (small) LUT, and provides meaningful values where OLD LUT would provide garbage. + const DateLUTImpl lut("UTC"); + + const time_t time = 4294343873; // 2106-01-31T01:17:53 (Sunday) + + EXPECT_EQ(lut.getTimeZone(), "UTC"); + + EXPECT_EQ(lut.toDate(time), 4294339200); + EXPECT_EQ(lut.toMonth(time), 1); + EXPECT_EQ(lut.toQuarter(time), 1); + EXPECT_EQ(lut.toYear(time), 2106); + EXPECT_EQ(lut.toDayOfMonth(time), 31); + + EXPECT_EQ(lut.toFirstDayOfWeek(time), 4293820800 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfWeek(time), DayNum(49697)); + EXPECT_EQ(lut.toFirstDayOfMonth(time), 4291747200 /*time_t*/); // 2016-01-01 + EXPECT_EQ(lut.toFirstDayNumOfMonth(time), DayNum(49673)); + EXPECT_EQ(lut.toFirstDayNumOfQuarter(time), DayNum(49673) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfQuarter(time), 4291747200 /*time_t*/); + EXPECT_EQ(lut.toFirstDayOfYear(time), 4291747200 /*time_t*/); + EXPECT_EQ(lut.toFirstDayNumOfYear(time), DayNum(49673) /*DayNum*/); + EXPECT_EQ(lut.toFirstDayOfNextMonth(time), 4294425600 /*time_t*/); // 2106-02-01 + EXPECT_EQ(lut.toFirstDayOfPrevMonth(time), 4289068800 /*time_t*/); // 2105-12-01 + EXPECT_EQ(lut.daysInMonth(time), 31 /*UInt8*/); + EXPECT_EQ(lut.toDateAndShift(time, 10), 4295203200 /*time_t*/); // 2106-02-10 + EXPECT_EQ(lut.toTime(time), 4673 /*time_t*/); + EXPECT_EQ(lut.toHour(time), 1 /*unsigned*/); + EXPECT_EQ(lut.toMinute(time), 17 /*unsigned*/); + EXPECT_EQ(lut.toSecond(time), 53 /*unsigned*/); + EXPECT_EQ(lut.toStartOfMinute(time), 4294343820 /*time_t*/); + EXPECT_EQ(lut.toStartOfFiveMinute(time), 4294343700 /*time_t*/); + EXPECT_EQ(lut.toStartOfFifteenMinutes(time), 4294343700 /*time_t*/); + EXPECT_EQ(lut.toStartOfTenMinutes(time), 4294343400 /*time_t*/); + EXPECT_EQ(lut.toStartOfHour(time), 4294342800 /*time_t*/); + EXPECT_EQ(lut.toDayNum(time), DayNum(49703) /*DayNum*/); + EXPECT_EQ(lut.toDayOfYear(time), 31 /*unsigned*/); + EXPECT_EQ(lut.toRelativeWeekNum(time), 7100 /*unsigned*/); + EXPECT_EQ(lut.toISOYear(time), 2106 /*unsigned*/); + EXPECT_EQ(lut.toFirstDayNumOfISOYear(time), DayNum(49676) /*DayNum*/); // 2106-01-04 + EXPECT_EQ(lut.toFirstDayOfISOYear(time), 4292006400 /*time_t*/); + EXPECT_EQ(lut.toISOWeek(time), 4 /*unsigned*/); + EXPECT_EQ(lut.toRelativeMonthNum(time), 25273 /*unsigned*/); + EXPECT_EQ(lut.toRelativeQuarterNum(time), 8424 /*unsigned*/); + EXPECT_EQ(lut.toRelativeHourNum(time), 1192873 /*time_t*/); + EXPECT_EQ(lut.toRelativeMinuteNum(time), 71572397 /*time_t*/); + EXPECT_EQ(lut.toStartOfHourInterval(time, 5), 4294332000 /*time_t*/); + EXPECT_EQ(lut.toStartOfMinuteInterval(time, 6), 4294343520 /*time_t*/); + EXPECT_EQ(lut.toStartOfSecondInterval(time, 7), 4294343872 /*time_t*/); + EXPECT_EQ(lut.toNumYYYYMM(time), 210601 /*UInt32*/); + EXPECT_EQ(lut.toNumYYYYMMDD(time), 21060131 /*UInt32*/); + EXPECT_EQ(lut.toNumYYYYMMDDhhmmss(time), 21060131011753 /*UInt64*/); + EXPECT_EQ(lut.addDays(time, 100), 4302983873 /*time_t*/); + EXPECT_EQ(lut.addWeeks(time, 10), 4300391873 /*time_t*/); + EXPECT_EQ(lut.addMonths(time, 10), 4320523073 /*time_t*/); // 2106-11-30 01:17:53 + EXPECT_EQ(lut.addQuarters(time, 10), 4373140673 /*time_t*/); // 2108-07-31 01:17:53 + EXPECT_EQ(lut.addYears(time, 10), 4609876673 /*time_t*/); // 2116-01-31 01:17:53 + + EXPECT_EQ(lut.timeToString(time), "2106-01-31 01:17:53" /*std::string*/); + EXPECT_EQ(lut.dateToString(time), "2106-01-31" /*std::string*/); +} + + +class DateLUT_TimeZone : public ::testing::TestWithParam +{}; + +TEST_P(DateLUT_TimeZone, DISABLED_LoadAllTimeZones) +{ + // There are some assumptions and assertions about TZ data made in DateLUTImpl which are verified upon loading, + // to make sure that those assertions are true for all timezones we are going to load all of them one by one. + DateLUTImpl{GetParam()}; +} + +// Another long running test, shouldn't be run to often +TEST_P(DateLUT_TimeZone, VaidateTimeComponentsAroundEpoch) +{ + // Converting time around 1970-01-01 to hour-minute-seconds time components + // could be problematic. + const size_t max_failures_per_tz = 3; + const auto timezone_name = GetParam(); + + const auto * test_info = ::testing::UnitTest::GetInstance()->current_test_info(); + const auto lut = DateLUTImpl(timezone_name); + + for (time_t i = -856147870; i < 86400 * 10000; i += 11 * 13 * 17 * 19) + { + SCOPED_TRACE(::testing::Message() + << "\n\tTimezone: " << timezone_name + << "\n\ttimestamp: " << i + << "\n\t offset at start of epoch : " << lut.getOffsetAtStartOfEpoch() + << "\n\t offset_is_whole_number_of_hours_everytime : " << lut.getOffsetIsWholNumberOfHoursEveryWhere() + << "\n\t time_offset_epoch : " << lut.getTimeOffsetEpoch() + << "\n\t offset_at_start_of_lut : " << lut.getTimeOffsetAtStartOfLUT()); + + EXPECT_GE(24, lut.toHour(i)); + EXPECT_GT(60, lut.toMinute(i)); + EXPECT_GT(60, lut.toSecond(i)); + + const auto current_failures = countFailures(*test_info->result()); + if (current_failures.total > 0) + { + if (i < 0) + i = -1; + } + + if (current_failures.total >= max_failures_per_tz) + break; + } +} + +TEST_P(DateLUT_TimeZone, getTimeZone) +{ + const auto & lut = DateLUT::instance(GetParam()); + + EXPECT_EQ(GetParam(), lut.getTimeZone()); +} + +TEST_P(DateLUT_TimeZone, ZeroTime) +{ + const auto & lut = DateLUT::instance(GetParam()); + + EXPECT_EQ(0, lut.toDayNum(time_t{0})); + EXPECT_EQ(0, lut.toDayNum(DayNum{0})); + EXPECT_EQ(0, lut.toDayNum(ExtendedDayNum{0})); +} + +// Group of tests for timezones that have or had some time ago an offset which is not multiple of 15 minutes. +INSTANTIATE_TEST_SUITE_P(ExoticTimezones, + DateLUT_TimeZone, + ::testing::ValuesIn(std::initializer_list{ + "Africa/El_Aaiun", + "Pacific/Apia", + "Pacific/Enderbury", + "Pacific/Fakaofo", + "Pacific/Kiritimati", + }) +); + +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimeZones, + DateLUT_TimeZone, + ::testing::ValuesIn(allTimezones()) +); + +std::ostream & operator<<(std::ostream & ostr, const DateLUTImpl::Values & v) +{ + return ostr << "DateLUTImpl::Values{" + << "\n\t date : " << v.date + << "\n\t year : " << static_cast(v.year) + << "\n\t month : " << static_cast(v.month) + << "\n\t day : " << static_cast(v.day_of_month) + << "\n\t weekday : " << static_cast(v.day_of_week) + << "\n\t days in month : " << static_cast(v.days_in_month) + << "\n\t offset change : " << v.amount_of_offset_change() + << "\n\t offfset change at : " << v.time_at_offset_change() + << "\n}"; +} + +struct TimeRangeParam +{ + const cctz::civil_second begin; + const cctz::civil_second end; + const int step_in_seconds; +}; + +std::ostream & operator<<(std::ostream & ostr, const TimeRangeParam & param) +{ + const auto approximate_step = [](const int step) -> std::string + { + // Convert seconds to a string of seconds or fractional count of minutes/hours/days. + static const size_t multipliers[] = {1 /*seconds to seconds*/, 60 /*seconds to minutes*/, 60 /*minutes to hours*/, 24 /*hours to days*/, 0 /*terminator*/}; + static const char* names[] = {"s", "m", "h", "d", nullptr}; + double result = step; + size_t i = 0; + for (; i < sizeof(multipliers)/sizeof(multipliers[0]) && result > multipliers[i]; ++i) + result /= multipliers[i]; + + char buffer[256] = {'\0'}; + std::snprintf(buffer, sizeof(buffer), "%.1f%s", result, names[i - 1]); + return std::string{buffer}; + }; + + return ostr << param.begin << " : " << param.end << " step: " << param.step_in_seconds << "s (" << approximate_step(param.step_in_seconds) << ")"; +} + +class DateLUT_Timezone_TimeRange : public ::testing::TestWithParam> +{}; + +// refactored test from tests/date_lut3.cpp +TEST_P(DateLUT_Timezone_TimeRange, InRange) +{ + // for a time_t values in range [begin, end) to match with reference obtained from cctz: + // compare date and time components: year, month, day, hours, minutes, seconds, formatted time string. + const auto & [timezone_name, range_data] = GetParam(); + const auto & [begin, end, step] = range_data; + + const auto * test_info = ::testing::UnitTest::GetInstance()->current_test_info(); + static const size_t max_failures_per_case = 3; + cctz::time_zone tz; + ASSERT_TRUE(cctz::load_time_zone(timezone_name, &tz)); + + const auto & lut = DateLUT::instance(timezone_name); + const auto start = cctz::convert(begin, tz).time_since_epoch().count(); + const auto stop = cctz::convert(end, tz).time_since_epoch().count(); + + for (time_t expected_time_t = start; expected_time_t < stop; expected_time_t += step) + { + SCOPED_TRACE(expected_time_t); + + const auto tz_time = cctz::convert(std::chrono::system_clock::from_time_t(expected_time_t), tz); + + EXPECT_EQ(tz_time.year(), lut.toYear(expected_time_t)); + EXPECT_EQ(tz_time.month(), lut.toMonth(expected_time_t)); + EXPECT_EQ(tz_time.day(), lut.toDayOfMonth(expected_time_t)); + EXPECT_EQ(static_cast(cctz::get_weekday(tz_time)) + 1, lut.toDayOfWeek(expected_time_t)); // tm.tm_wday Sunday is 0, while for DateLUTImpl it is 7 + EXPECT_EQ(cctz::get_yearday(tz_time), lut.toDayOfYear(expected_time_t)); + EXPECT_EQ(tz_time.hour(), lut.toHour(expected_time_t)); + EXPECT_EQ(tz_time.minute(), lut.toMinute(expected_time_t)); + EXPECT_EQ(tz_time.second(), lut.toSecond(expected_time_t)); + + const auto time_string = cctz::format("%E4Y-%m-%d %H:%M:%S", std::chrono::system_clock::from_time_t(expected_time_t), tz); + EXPECT_EQ(time_string, lut.timeToString(expected_time_t)); + + // it makes sense to let test execute all checks above to simplify debugging, + // but once we've found a bad apple, no need to dig deeper. + if (countFailures(*test_info->result()).total >= max_failures_per_case) + break; + } +} + +/** Next tests are disabled due to following reasons: + * 1. They are huge and take enormous amount of time to run + * 2. Current implementation of DateLUTImpl is inprecise and some cases fail and it seems impractical to try to fix those. + * 3. Many failures (~300) were fixed while refactoring, about ~40 remain the same and 3 new introduced: + * "Asia/Gaza" + * "Pacific/Enderbury" + * "Pacific/Kiritimati" + * So it would be tricky to skip knonw failures to allow all unit tests to pass. + */ +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010, + DateLUT_Timezone_TimeRange, + ::testing::Combine( + ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(std::initializer_list{ + // Values from tests/date_lut3.cpp + {YYYYMMDDToDay(20101031), YYYYMMDDToDay(20101101), 15 * 60}, + {YYYYMMDDToDay(20100328), YYYYMMDDToDay(20100330), 15 * 60} + })) +); + +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970_WHOLE, + DateLUT_Timezone_TimeRange, + ::testing::Combine( + ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(std::initializer_list{ + // Values from tests/date_lut3.cpp + {YYYYMMDDToDay(19700101), YYYYMMDDToDay(19701231), 3191 /*53m 11s*/}, + })) +); + +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2010_WHOLE, + DateLUT_Timezone_TimeRange, + ::testing::Combine( + ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(std::initializer_list{ + // Values from tests/date_lut3.cpp + {YYYYMMDDToDay(20100101), YYYYMMDDToDay(20101231), 3191 /*53m 11s*/}, + })) +); + +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year2020_WHOLE, + DateLUT_Timezone_TimeRange, + ::testing::Combine( + ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(std::initializer_list{ + // Values from tests/date_lut3.cpp + {YYYYMMDDToDay(20200101), YYYYMMDDToDay(20201231), 3191 /*53m 11s*/}, + })) +); + +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_PreEpoch, + DateLUT_Timezone_TimeRange, + ::testing::Combine( + ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(std::initializer_list{ + {YYYYMMDDToDay(19500101), YYYYMMDDToDay(19600101), 15 * 60}, + {YYYYMMDDToDay(19300101), YYYYMMDDToDay(19350101), 11 * 15 * 60} + })) +); + +INSTANTIATE_TEST_SUITE_P(DISABLED_AllTimezones_Year1970, + DateLUT_Timezone_TimeRange, + ::testing::Combine( + ::testing::ValuesIn(allTimezones()), + ::testing::ValuesIn(std::initializer_list{ + {YYYYMMDDToDay(19700101), YYYYMMDDToDay(19700201), 15 * 60}, + {YYYYMMDDToDay(19700101), YYYYMMDDToDay(19701231), 11 * 13 * 17} +// // 11 was chosen as a number which can't divide product of 2-combinarions of (7, 24, 60), +// // to reduce likelehood of hitting same hour/minute/second values for different days. +// // + 12 is just to make sure that last day is covered fully. +// {0, 0 + 11 * 3600 * 24 + 12, 11}, + })) +); + diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 3c27908741c..63fa5d8a5c9 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -390,7 +390,7 @@ private: for (auto d : chineseNewYearIndicators) { /// Let's celebrate until Lantern Festival - if (d <= days && d + 25u >= days) + if (d <= days && d + 25 >= days) return true; else if (d > days) return false; diff --git a/src/Core/DecimalFunctions.h b/src/Core/DecimalFunctions.h index 355cf1d378a..2cd50ab8d08 100644 --- a/src/Core/DecimalFunctions.h +++ b/src/Core/DecimalFunctions.h @@ -50,9 +50,10 @@ inline auto scaleMultiplier(UInt32 scale) * whole - represents whole part of decimal, can be negative or positive. * fractional - for fractional part of decimal, always positive. */ -template +template struct DecimalComponents { + using T = typename DecimalType::NativeType; T whole; T fractional; }; @@ -106,6 +107,15 @@ inline DecimalType decimalFromComponentsWithMultiplier( return DecimalType(value); } +template +inline DecimalType decimalFromComponentsWithMultiplier( + const DecimalComponents & components, + typename DecimalType::NativeType scale_multiplier) +{ + return decimalFromComponentsWithMultiplier(components.whole, components.fractional, scale_multiplier); +} + + /** Make a decimal value from whole and fractional components with given scale. * * @see `decimalFromComponentsWithMultiplier` for details. @@ -126,7 +136,7 @@ inline DecimalType decimalFromComponents( */ template inline DecimalType decimalFromComponents( - const DecimalComponents & components, + const DecimalComponents & components, UInt32 scale) { return decimalFromComponents(components.whole, components.fractional, scale); @@ -136,7 +146,7 @@ inline DecimalType decimalFromComponents( * This is an optimization to reduce number of calls to scaleMultiplier on known scale. */ template -inline DecimalComponents splitWithScaleMultiplier( +inline DecimalComponents splitWithScaleMultiplier( const DecimalType & decimal, typename DecimalType::NativeType scale_multiplier) { @@ -151,7 +161,7 @@ inline DecimalComponents splitWithScaleMultipl /// Split decimal into components: whole and fractional part, @see `DecimalComponents` for details. template -inline DecimalComponents split(const DecimalType & decimal, UInt32 scale) +inline DecimalComponents split(const DecimalType & decimal, UInt32 scale) { if (scale == 0) { diff --git a/src/Core/MySQL/MySQLReplication.cpp b/src/Core/MySQL/MySQLReplication.cpp index 1b202c4edb4..3e9c5230955 100644 --- a/src/Core/MySQL/MySQLReplication.cpp +++ b/src/Core/MySQL/MySQLReplication.cpp @@ -420,8 +420,8 @@ namespace MySQLReplication UInt32 i24 = 0; payload.readStrict(reinterpret_cast(&i24), 3); - DayNum date_day_number = DateLUT::instance().makeDayNum( - static_cast((i24 >> 9) & 0x7fff), static_cast((i24 >> 5) & 0xf), static_cast(i24 & 0x1f)); + const DayNum date_day_number{DateLUT::instance().makeDayNum( + static_cast((i24 >> 9) & 0x7fff), static_cast((i24 >> 5) & 0xf), static_cast(i24 & 0x1f)).toUnderType()}; row.push_back(Field(date_day_number.toUnderType())); break; @@ -443,7 +443,7 @@ namespace MySQLReplication row.push_back(Field{UInt32(date_time)}); else { - DB::DecimalUtils::DecimalComponents components{ + DB::DecimalUtils::DecimalComponents components{ static_cast(date_time), 0}; components.fractional = fsp; @@ -462,7 +462,7 @@ namespace MySQLReplication row.push_back(Field{sec}); else { - DB::DecimalUtils::DecimalComponents components{ + DB::DecimalUtils::DecimalComponents components{ static_cast(sec), 0}; components.fractional = fsp; diff --git a/src/Core/tests/gtest_DecimalFunctions.cpp b/src/Core/tests/gtest_DecimalFunctions.cpp index be64661176b..1069a810d64 100644 --- a/src/Core/tests/gtest_DecimalFunctions.cpp +++ b/src/Core/tests/gtest_DecimalFunctions.cpp @@ -14,7 +14,7 @@ struct DecimalUtilsSplitAndCombineTestParam Decimal64 decimal_value; uint8_t scale; - DecimalUtils::DecimalComponents components; + DecimalUtils::DecimalComponents components; }; std::ostream & operator << (std::ostream & ostr, const DecimalUtilsSplitAndCombineTestParam & param) diff --git a/src/DataStreams/MongoDBBlockInputStream.cpp b/src/DataStreams/MongoDBBlockInputStream.cpp index 5463d95151b..e4ddcd09ede 100644 --- a/src/DataStreams/MongoDBBlockInputStream.cpp +++ b/src/DataStreams/MongoDBBlockInputStream.cpp @@ -270,8 +270,8 @@ namespace throw Exception{"Type mismatch, expected Timestamp, got type id = " + toString(value.type()) + " for column " + name, ErrorCodes::TYPE_MISMATCH}; - assert_cast(column).getData().push_back(UInt16{DateLUT::instance().toDayNum( - static_cast &>(value).value().epochTime())}); + assert_cast(column).getData().push_back(static_cast(DateLUT::instance().toDayNum( + static_cast &>(value).value().epochTime()))); break; } diff --git a/src/DataTypes/DataTypeDateTime64.h b/src/DataTypes/DataTypeDateTime64.h index 198c3739f58..ec3f2fde889 100644 --- a/src/DataTypes/DataTypeDateTime64.h +++ b/src/DataTypes/DataTypeDateTime64.h @@ -48,66 +48,5 @@ public: bool canBePromoted() const override { return false; } }; -/** Tansform-type wrapper for DateTime64, applies given Transform to DateTime64 value or only to a whole part of it. - * - * Depending on what overloads of Transform::execute() are available, when called with DateTime64 value, - * invokes Transform::execute() with: - * * whole part of DateTime64 value, discarding fractional part. - * * DateTime64 value and scale factor. - * - * Suitable Transfotm-types are commonly used in Date/DateTime manipulation functions, - * and should implement static (or const) function with following signatures: - * R execute(UInt32 whole_value, ... , const TimeZoneImpl &) - * OR - * R execute(DateTime64 value, Int64 scale_factor, ... , const TimeZoneImpl &) - * - * Where R and T could be arbitrary types. -*/ -template -class TransformDateTime64 : public Transform -{ -private: - // Detect if Transform::execute is const or static method - // with signature defined by template args (ignoring result type). - template - struct TransformHasExecuteOverload : std::false_type {}; - - template - struct TransformHasExecuteOverload().execute(std::declval()...))>, Args...> - : std::true_type {}; - - template - static constexpr bool TransformHasExecuteOverload_v = TransformHasExecuteOverload::value; - -public: - static constexpr auto name = Transform::name; - - using Transform::execute; - - // non-explicit constructor to allow creating from scale value (or with no scale at all), indispensable in some contexts. - TransformDateTime64(UInt32 scale_ = 0) - : scale_multiplier(DecimalUtils::scaleMultiplier(scale_)) - {} - - template - inline auto execute(const DateTime64 & t, Args && ... args) const - { - const auto transform = static_cast(this); - - if constexpr (TransformHasExecuteOverload_v) - { - return transform->execute(t, scale_multiplier, std::forward(args)...); - } - else - { - const auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); - return transform->execute(static_cast(components.whole), std::forward(args)...); - } - } - -private: - DateTime64::NativeType scale_multiplier = 1; -}; - } diff --git a/src/Functions/CustomWeekTransforms.h b/src/Functions/CustomWeekTransforms.h index afcbadc835c..28da546eb93 100644 --- a/src/Functions/CustomWeekTransforms.h +++ b/src/Functions/CustomWeekTransforms.h @@ -33,14 +33,21 @@ static inline UInt32 dateIsNotSupported(const char * name) /// This factor transformation will say that the function is monotone everywhere. struct ZeroTransform { - static inline UInt16 execute(UInt32, UInt8, const DateLUTImpl &) { return 0; } static inline UInt16 execute(UInt16, UInt8, const DateLUTImpl &) { return 0; } + static inline UInt16 execute(UInt32, UInt8, const DateLUTImpl &) { return 0; } + static inline UInt16 execute(Int64, UInt8, const DateLUTImpl &) { return 0; } }; struct ToWeekImpl { static constexpr auto name = "toWeek"; + static inline UInt8 execute(Int64 t, UInt8 week_mode, const DateLUTImpl & time_zone) + { + // TODO: ditch conversion to DayNum, since it doesn't support extended range. + YearWeek yw = time_zone.toYearWeek(time_zone.toDayNum(t), week_mode); + return yw.second; + } static inline UInt8 execute(UInt32 t, UInt8 week_mode, const DateLUTImpl & time_zone) { YearWeek yw = time_zone.toYearWeek(time_zone.toDayNum(t), week_mode); @@ -59,6 +66,13 @@ struct ToYearWeekImpl { static constexpr auto name = "toYearWeek"; + static inline UInt32 execute(Int64 t, UInt8 week_mode, const DateLUTImpl & time_zone) + { + // TODO: ditch toDayNum() + YearWeek yw = time_zone.toYearWeek(time_zone.toDayNum(t), week_mode | static_cast(WeekModeFlag::YEAR)); + return yw.first * 100 + yw.second; + } + static inline UInt32 execute(UInt32 t, UInt8 week_mode, const DateLUTImpl & time_zone) { YearWeek yw = time_zone.toYearWeek(time_zone.toDayNum(t), week_mode | static_cast(WeekModeFlag::YEAR)); @@ -77,13 +91,19 @@ struct ToStartOfWeekImpl { static constexpr auto name = "toStartOfWeek"; + static inline UInt16 execute(Int64 t, UInt8 week_mode, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode); +// return time_zone.toFirstDayNumOfWeek(t, week_mode); + } static inline UInt16 execute(UInt32 t, UInt8 week_mode, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode); +// return time_zone.toFirstDayNumOfWeek(t, week_mode); } static inline UInt16 execute(UInt16 d, UInt8 week_mode, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfWeek(DayNum(d), week_mode); + return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d), week_mode); } using FactorTransform = ZeroTransform; diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index 333b397312d..c299b9c4169 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -3,6 +3,7 @@ #include #include #include +//#include #include #include #include @@ -33,14 +34,15 @@ namespace ErrorCodes * factor-transformation F is "round to the nearest month" (2015-02-03 -> 2015-02-01). */ -static inline UInt32 dateIsNotSupported(const char * name) -{ - throw Exception("Illegal type Date of argument for function " + std::string(name), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); -} + static inline UInt32 dateIsNotSupported(const char * name) + { + throw Exception("Illegal type Date of argument for function " + std::string(name), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } /// This factor transformation will say that the function is monotone everywhere. struct ZeroTransform { + static inline UInt16 execute(Int64, const DateLUTImpl &) { return 0; } static inline UInt16 execute(UInt32, const DateLUTImpl &) { return 0; } static inline UInt16 execute(UInt16, const DateLUTImpl &) { return 0; } }; @@ -49,6 +51,10 @@ struct ToDateImpl { static constexpr auto name = "toDate"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return UInt16(time_zone.toDayNum(t)); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return UInt16(time_zone.toDayNum(t)); @@ -65,13 +71,18 @@ struct ToStartOfDayImpl { static constexpr auto name = "toStartOfDay"; + //TODO: right now it is hardcoded to produce DateTime only, needs fixing later. See date_and_time_type_details::ResultDataTypeMap for deduction of result type example. + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toDate(static_cast(t.whole)); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toDate(t); } static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toDate(DayNum(d)); + return time_zone.toDate(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -81,13 +92,19 @@ struct ToMondayImpl { static constexpr auto name = "toMonday"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + //return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t)); + return time_zone.toFirstDayNumOfWeek(t); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t)); + //return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t)); + return time_zone.toFirstDayNumOfWeek(t); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfWeek(DayNum(d)); + return time_zone.toFirstDayNumOfWeek(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -97,13 +114,17 @@ struct ToStartOfMonthImpl { static constexpr auto name = "toStartOfMonth"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfMonth(time_zone.toDayNum(t)); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfMonth(time_zone.toDayNum(t)); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfMonth(DayNum(d)); + return time_zone.toFirstDayNumOfMonth(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -113,13 +134,17 @@ struct ToStartOfQuarterImpl { static constexpr auto name = "toStartOfQuarter"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfQuarter(time_zone.toDayNum(t)); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfQuarter(time_zone.toDayNum(t)); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfQuarter(DayNum(d)); + return time_zone.toFirstDayNumOfQuarter(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -129,13 +154,17 @@ struct ToStartOfYearImpl { static constexpr auto name = "toStartOfYear"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfYear(time_zone.toDayNum(t)); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfYear(time_zone.toDayNum(t)); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfYear(DayNum(d)); + return time_zone.toFirstDayNumOfYear(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -144,9 +173,13 @@ struct ToStartOfYearImpl struct ToTimeImpl { + /// When transforming to time, the date will be equated to 1970-01-01. static constexpr auto name = "toTime"; - /// When transforming to time, the date will be equated to 1970-01-02. + static UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toTime(t.whole) + 86400; + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toTime(t) + 86400; @@ -164,6 +197,10 @@ struct ToStartOfMinuteImpl { static constexpr auto name = "toStartOfMinute"; + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfMinute(t.whole); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfMinute(t); @@ -215,6 +252,10 @@ struct ToStartOfFiveMinuteImpl { static constexpr auto name = "toStartOfFiveMinute"; + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfFiveMinute(t.whole); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfFiveMinute(t); @@ -231,6 +272,10 @@ struct ToStartOfTenMinutesImpl { static constexpr auto name = "toStartOfTenMinutes"; + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfTenMinutes(t.whole); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfTenMinutes(t); @@ -247,6 +292,10 @@ struct ToStartOfFifteenMinutesImpl { static constexpr auto name = "toStartOfFifteenMinutes"; + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfFifteenMinutes(t.whole); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfFifteenMinutes(t); @@ -264,6 +313,12 @@ struct TimeSlotImpl { static constexpr auto name = "timeSlot"; + //static inline DecimalUtils::DecimalComponents execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl &) + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl &) + { + return t.whole / 1800 * 1800; + } + static inline UInt32 execute(UInt32 t, const DateLUTImpl &) { return t / 1800 * 1800; @@ -281,6 +336,11 @@ struct ToStartOfHourImpl { static constexpr auto name = "toStartOfHour"; + static inline UInt32 execute(const DecimalUtils::DecimalComponents & t, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfHour(t.whole); + } + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toStartOfHour(t); @@ -298,13 +358,17 @@ struct ToYearImpl { static constexpr auto name = "toYear"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toYear(t); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toYear(t); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toYear(DayNum(d)); + return time_zone.toYear(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -314,13 +378,17 @@ struct ToQuarterImpl { static constexpr auto name = "toQuarter"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toQuarter(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toQuarter(t); } static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toQuarter(DayNum(d)); + return time_zone.toQuarter(ExtendedDayNum(d)); } using FactorTransform = ToStartOfYearImpl; @@ -330,13 +398,17 @@ struct ToMonthImpl { static constexpr auto name = "toMonth"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toMonth(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toMonth(t); } static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toMonth(DayNum(d)); + return time_zone.toMonth(ExtendedDayNum(d)); } using FactorTransform = ToStartOfYearImpl; @@ -346,13 +418,17 @@ struct ToDayOfMonthImpl { static constexpr auto name = "toDayOfMonth"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toDayOfMonth(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toDayOfMonth(t); } static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toDayOfMonth(DayNum(d)); + return time_zone.toDayOfMonth(ExtendedDayNum(d)); } using FactorTransform = ToStartOfMonthImpl; @@ -362,13 +438,17 @@ struct ToDayOfWeekImpl { static constexpr auto name = "toDayOfWeek"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toDayOfWeek(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toDayOfWeek(t); } static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toDayOfWeek(DayNum(d)); + return time_zone.toDayOfWeek(ExtendedDayNum(d)); } using FactorTransform = ToMondayImpl; @@ -378,13 +458,17 @@ struct ToDayOfYearImpl { static constexpr auto name = "toDayOfYear"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toDayOfYear(t); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toDayOfYear(t); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toDayOfYear(DayNum(d)); + return time_zone.toDayOfYear(ExtendedDayNum(d)); } using FactorTransform = ToStartOfYearImpl; @@ -394,6 +478,10 @@ struct ToHourImpl { static constexpr auto name = "toHour"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toHour(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toHour(t); @@ -411,6 +499,11 @@ struct TimezoneOffsetImpl { static constexpr auto name = "timezoneOffset"; + static inline time_t execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.timezoneOffset(t); + } + static inline time_t execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.timezoneOffset(t); @@ -428,6 +521,10 @@ struct ToMinuteImpl { static constexpr auto name = "toMinute"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toMinute(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toMinute(t); @@ -444,6 +541,10 @@ struct ToSecondImpl { static constexpr auto name = "toSecond"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toSecond(t); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toSecond(t); @@ -460,13 +561,17 @@ struct ToISOYearImpl { static constexpr auto name = "toISOYear"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toISOYear(time_zone.toDayNum(t)); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toISOYear(time_zone.toDayNum(t)); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toISOYear(DayNum(d)); + return time_zone.toISOYear(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -476,13 +581,17 @@ struct ToStartOfISOYearImpl { static constexpr auto name = "toStartOfISOYear"; + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toFirstDayNumOfISOYear(time_zone.toDayNum(t)); + } static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toFirstDayNumOfISOYear(time_zone.toDayNum(t)); } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toFirstDayNumOfISOYear(DayNum(d)); + return time_zone.toFirstDayNumOfISOYear(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -492,13 +601,17 @@ struct ToISOWeekImpl { static constexpr auto name = "toISOWeek"; + static inline UInt8 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toISOWeek(time_zone.toDayNum(t)); + } static inline UInt8 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toISOWeek(time_zone.toDayNum(t)); } static inline UInt8 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toISOWeek(DayNum(d)); + return time_zone.toISOWeek(ExtendedDayNum(d)); } using FactorTransform = ToISOYearImpl; @@ -508,13 +621,17 @@ struct ToRelativeYearNumImpl { static constexpr auto name = "toRelativeYearNum"; - static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toYear(t); } + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toYear(static_cast(t)); + } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toYear(DayNum(d)); + return time_zone.toYear(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -524,13 +641,17 @@ struct ToRelativeQuarterNumImpl { static constexpr auto name = "toRelativeQuarterNum"; - static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeQuarterNum(t); } + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toRelativeQuarterNum(static_cast(t)); + } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toRelativeQuarterNum(DayNum(d)); + return time_zone.toRelativeQuarterNum(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -540,13 +661,17 @@ struct ToRelativeMonthNumImpl { static constexpr auto name = "toRelativeMonthNum"; - static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeMonthNum(t); } + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toRelativeMonthNum(static_cast(t)); + } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toRelativeMonthNum(DayNum(d)); + return time_zone.toRelativeMonthNum(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -556,13 +681,17 @@ struct ToRelativeWeekNumImpl { static constexpr auto name = "toRelativeWeekNum"; - static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeWeekNum(t); } + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toRelativeWeekNum(static_cast(t)); + } static inline UInt16 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toRelativeWeekNum(DayNum(d)); + return time_zone.toRelativeWeekNum(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -572,10 +701,14 @@ struct ToRelativeDayNumImpl { static constexpr auto name = "toRelativeDayNum"; - static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt16 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toDayNum(t); } + static inline UInt16 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toDayNum(static_cast(t)); + } static inline UInt16 execute(UInt16 d, const DateLUTImpl &) { return static_cast(d); @@ -589,13 +722,17 @@ struct ToRelativeHourNumImpl { static constexpr auto name = "toRelativeHourNum"; - static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeHourNum(t); } + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toRelativeHourNum(static_cast(t)); + } static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toRelativeHourNum(DayNum(d)); + return time_zone.toRelativeHourNum(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -605,13 +742,17 @@ struct ToRelativeMinuteNumImpl { static constexpr auto name = "toRelativeMinuteNum"; - static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) + static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) { return time_zone.toRelativeMinuteNum(t); } + static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) + { + return time_zone.toRelativeMinuteNum(static_cast(t)); + } static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.toRelativeMinuteNum(DayNum(d)); + return time_zone.toRelativeMinuteNum(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -621,13 +762,17 @@ struct ToRelativeSecondNumImpl { static constexpr auto name = "toRelativeSecondNum"; + static inline Int64 execute(Int64 t, const DateLUTImpl &) + { + return t; + } static inline UInt32 execute(UInt32 t, const DateLUTImpl &) { return t; } static inline UInt32 execute(UInt16 d, const DateLUTImpl & time_zone) { - return time_zone.fromDayNum(DayNum(d)); + return time_zone.fromDayNum(ExtendedDayNum(d)); } using FactorTransform = ZeroTransform; @@ -637,6 +782,10 @@ struct ToYYYYMMImpl { static constexpr auto name = "toYYYYMM"; + static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toNumYYYYMM(t); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toNumYYYYMM(t); @@ -653,6 +802,10 @@ struct ToYYYYMMDDImpl { static constexpr auto name = "toYYYYMMDD"; + static inline UInt32 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toNumYYYYMMDD(t); + } static inline UInt32 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toNumYYYYMMDD(t); @@ -669,6 +822,10 @@ struct ToYYYYMMDDhhmmssImpl { static constexpr auto name = "toYYYYMMDDhhmmss"; + static inline UInt64 execute(Int64 t, const DateLUTImpl & time_zone) + { + return time_zone.toNumYYYYMMDDhhmmss(t); + } static inline UInt64 execute(UInt32 t, const DateLUTImpl & time_zone) { return time_zone.toNumYYYYMMDDhhmmss(t); diff --git a/src/Functions/FunctionCustomWeekToSomething.h b/src/Functions/FunctionCustomWeekToSomething.h index 8a343cffb95..5634ea11584 100644 --- a/src/Functions/FunctionCustomWeekToSomething.h +++ b/src/Functions/FunctionCustomWeekToSomething.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include diff --git a/src/Functions/FunctionDateOrDateTimeAddInterval.h b/src/Functions/FunctionDateOrDateTimeAddInterval.h index 5f964b899b4..2b0082f4334 100644 --- a/src/Functions/FunctionDateOrDateTimeAddInterval.h +++ b/src/Functions/FunctionDateOrDateTimeAddInterval.h @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -25,31 +26,6 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } -/// AddOnDateTime64DefaultImpl provides default implementation of add-X functionality for DateTime64. -/// -/// Default implementation is not to change fractional part, but only modify whole part as if it was DateTime. -/// That means large whole values (for scale less than 9) might not fit into UInt32-range, -/// and hence default implementation will produce incorrect results. -template -struct AddOnDateTime64DefaultImpl -{ - AddOnDateTime64DefaultImpl(UInt32 scale_ = 0) - : scale_multiplier(DecimalUtils::scaleMultiplier(scale_)) - {} - - // Default implementation for add/sub on DateTime64: do math on whole part (the same way as for DateTime), leave fractional as it is. - inline DateTime64 execute(const DateTime64 & t, Int64 delta, const DateLUTImpl & time_zone) const - { - const auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); - - const auto whole = static_cast(this)->execute(static_cast(components.whole), delta, time_zone); - return DecimalUtils::decimalFromComponentsWithMultiplier(static_cast(whole), components.fractional, scale_multiplier); - } - - UInt32 scale_multiplier = 1; -}; - - /// Type of first argument of 'execute' function overload defines what INPUT DataType it is used for. /// Return type defines what is the OUTPUT (return) type of the CH function. /// Corresponding types: @@ -60,14 +36,15 @@ struct AddOnDateTime64DefaultImpl /// - 'AddSecondsImpl::execute(UInt32, ...) -> UInt32' is available to the ClickHouse users as 'addSeconds(DateTime, ...) -> DateTime' /// - 'AddSecondsImpl::execute(UInt16, ...) -> UInt32' is available to the ClickHouse users as 'addSeconds(Date, ...) -> DateTime' -struct AddSecondsImpl : public AddOnDateTime64DefaultImpl +struct AddSecondsImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addSeconds"; + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl &) + { + return {t.whole + delta, t.fractional}; + } + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &) { return t + delta; @@ -75,18 +52,19 @@ struct AddSecondsImpl : public AddOnDateTime64DefaultImpl static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { - return time_zone.fromDayNum(DayNum(d)) + delta; + return time_zone.fromDayNum(ExtendedDayNum(d)) + delta; } }; -struct AddMinutesImpl : public AddOnDateTime64DefaultImpl +struct AddMinutesImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addMinutes"; + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl &) + { + return {t.whole + delta * 60, t.fractional}; + } + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &) { return t + delta * 60; @@ -94,18 +72,18 @@ struct AddMinutesImpl : public AddOnDateTime64DefaultImpl static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { - return time_zone.fromDayNum(DayNum(d)) + delta * 60; + return time_zone.fromDayNum(ExtendedDayNum(d)) + delta * 60; } }; -struct AddHoursImpl : public AddOnDateTime64DefaultImpl +struct AddHoursImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addHours"; + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl &) + { + return {t.whole + delta * 3600, t.fractional}; + } static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl &) { return t + delta * 3600; @@ -113,19 +91,20 @@ struct AddHoursImpl : public AddOnDateTime64DefaultImpl static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { - return time_zone.fromDayNum(DayNum(d)) + delta * 3600; + return time_zone.fromDayNum(ExtendedDayNum(d)) + delta * 3600; } }; -struct AddDaysImpl : public AddOnDateTime64DefaultImpl +struct AddDaysImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addDays"; - static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone) + { + return {time_zone.addDays(t.whole, delta), t.fractional}; + } + + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addDays(t, delta); } @@ -136,14 +115,15 @@ struct AddDaysImpl : public AddOnDateTime64DefaultImpl } }; -struct AddWeeksImpl : public AddOnDateTime64DefaultImpl +struct AddWeeksImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addWeeks"; + static inline NO_SANITIZE_UNDEFINED DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone) + { + return {time_zone.addWeeks(t.whole, delta), t.fractional}; + } + static inline NO_SANITIZE_UNDEFINED UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addWeeks(t, delta); @@ -155,14 +135,15 @@ struct AddWeeksImpl : public AddOnDateTime64DefaultImpl } }; -struct AddMonthsImpl : public AddOnDateTime64DefaultImpl +struct AddMonthsImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addMonths"; + static inline DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone) + { + return {time_zone.addMonths(t.whole, delta), t.fractional}; + } + static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addMonths(t, delta); @@ -170,18 +151,19 @@ struct AddMonthsImpl : public AddOnDateTime64DefaultImpl static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { - return time_zone.addMonths(DayNum(d), delta); + return time_zone.addMonths(ExtendedDayNum(d), delta); } }; -struct AddQuartersImpl : public AddOnDateTime64DefaultImpl +struct AddQuartersImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addQuarters"; + static inline DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone) + { + return {time_zone.addQuarters(t.whole, delta), t.fractional}; + } + static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addQuarters(t, delta); @@ -189,18 +171,19 @@ struct AddQuartersImpl : public AddOnDateTime64DefaultImpl static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { - return time_zone.addQuarters(DayNum(d), delta); + return time_zone.addQuarters(ExtendedDayNum(d), delta); } }; -struct AddYearsImpl : public AddOnDateTime64DefaultImpl +struct AddYearsImpl { - using Base = AddOnDateTime64DefaultImpl; - using Base::Base; - using Base::execute; - static constexpr auto name = "addYears"; + static inline DecimalUtils::DecimalComponents execute(DecimalUtils::DecimalComponents t, Int64 delta, const DateLUTImpl & time_zone) + { + return {time_zone.addYears(t.whole, delta), t.fractional}; + } + static inline UInt32 execute(UInt32 t, Int64 delta, const DateLUTImpl & time_zone) { return time_zone.addYears(t, delta); @@ -208,7 +191,7 @@ struct AddYearsImpl : public AddOnDateTime64DefaultImpl static inline UInt16 execute(UInt16 d, Int64 delta, const DateLUTImpl & time_zone) { - return time_zone.addYears(DayNum(d), delta); + return time_zone.addYears(ExtendedDayNum(d), delta); } }; @@ -351,6 +334,7 @@ template <> struct ResultDataTypeMap { using ResultDataType = DataTy template <> struct ResultDataTypeMap { using ResultDataType = DataTypeDateTime; }; template <> struct ResultDataTypeMap { using ResultDataType = DataTypeDateTime; }; template <> struct ResultDataTypeMap { using ResultDataType = DataTypeDateTime64; }; +template <> struct ResultDataTypeMap { using ResultDataType = DataTypeDateTime64; }; } template @@ -417,10 +401,18 @@ public: } } + // TransformDateTime64 helps choosing correct overload of exec and does some transformations + // on input and output parameters to simplify support of DateTime64 in concrete Transform. + template + using TransformType = std::conditional_t< + std::is_same_v, + TransformDateTime64, + Transform>; + /// Helper templates to deduce return type based on argument type, since some overloads may promote or denote types, /// e.g. addSeconds(Date, 1) => DateTime template - using TransformExecuteReturnType = decltype(std::declval().execute(FieldType(), 0, std::declval())); + using TransformExecuteReturnType = decltype(std::declval>().execute(FieldType(), 0, std::declval())); // Deduces RETURN DataType from INPUT DataType, based on return type of Transform{}.execute(INPUT_TYPE, UInt64, DateLUTImpl). // e.g. for Transform-type that has execute()-overload with 'UInt16' input and 'UInt32' return, @@ -475,8 +467,9 @@ public: } else if (const auto * datetime64_type = assert_cast(from_type)) { - return DateTimeAddIntervalImpl, Transform>::execute( - Transform{datetime64_type->getScale()}, arguments, result_type); + using WrappedTransformType = TransformType; + return DateTimeAddIntervalImpl, WrappedTransformType>::execute( + WrappedTransformType{datetime64_type->getScale()}, arguments, result_type); } else throw Exception("Illegal type " + arguments[0].type->getName() + " of first argument of function " + getName(), diff --git a/src/Functions/FunctionDateOrDateTimeToSomething.h b/src/Functions/FunctionDateOrDateTimeToSomething.h index e0676f3dc0f..abe859e2f29 100644 --- a/src/Functions/FunctionDateOrDateTimeToSomething.h +++ b/src/Functions/FunctionDateOrDateTimeToSomething.h @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -107,6 +108,7 @@ public: else if (which.isDateTime64()) { const auto scale = static_cast(from_type)->getScale(); + const TransformDateTime64 transformer(scale); return DateTimeTransformImpl::execute(arguments, result_type, input_rows_count, transformer); } @@ -133,7 +135,6 @@ public: /// This method is called only if the function has one argument. Therefore, we do not care about the non-local time zone. const DateLUTImpl & date_lut = DateLUT::instance(); - if (left.isNull() || right.isNull()) return is_not_monotonic; diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 2e2a4ce9cfa..62577b8b402 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -310,10 +311,15 @@ struct ToDateTimeImpl return time_zone.fromDayNum(DayNum(d)); } - // no-op conversion from DateTime to DateTime, used in DateTime64 to DateTime conversion. - static inline UInt32 execute(UInt32 d, const DateLUTImpl & /*time_zone*/) + static inline UInt32 execute(UInt32 dt, const DateLUTImpl & /*time_zone*/) { - return d; + return dt; + } + + // TODO: return UInt32 ??? + static inline Int64 execute(Int64 dt64, const DateLUTImpl & /*time_zone*/) + { + return dt64; } }; @@ -329,6 +335,7 @@ struct ToDateTransform32Or64 static inline NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl & time_zone) { + // since converting to Date, no need in values outside of default LUT range. return (from < 0xFFFF) ? from : time_zone.toDayNum(std::min(time_t(from), time_t(0xFFFFFFFF))); @@ -342,6 +349,7 @@ struct ToDateTransform32Or64Signed static inline NO_SANITIZE_UNDEFINED ToType execute(const FromType & from, const DateLUTImpl & time_zone) { + // TODO: decide narrow or extended range based on FromType /// The function should be monotonic (better for query optimizations), so we saturate instead of overflow. if (from < 0) return 0; @@ -447,35 +455,8 @@ template struct ConvertImpl struct ConvertImpl : DateTimeTransformImpl> {}; - -/** Conversion of Date or DateTime to DateTime64: add zero sub-second part. - */ -struct ToDateTime64Transform -{ - static constexpr auto name = "toDateTime64"; - - const DateTime64::NativeType scale_multiplier = 1; - - ToDateTime64Transform(UInt32 scale = 0) - : scale_multiplier(DecimalUtils::scaleMultiplier(scale)) - {} - - inline DateTime64::NativeType execute(UInt16 d, const DateLUTImpl & time_zone) const - { - const auto dt = ToDateTimeImpl::execute(d, time_zone); - return execute(dt, time_zone); - } - - inline DateTime64::NativeType execute(UInt32 dt, const DateLUTImpl & /*time_zone*/) const - { - return DecimalUtils::decimalFromComponentsWithMultiplier(dt, 0, scale_multiplier); - } -}; - -template struct ConvertImpl - : DateTimeTransformImpl {}; -template struct ConvertImpl - : DateTimeTransformImpl {}; +const time_t LUT_MIN_TIME = -1420070400l; // 1925-01-01 UTC +const time_t LUT_MAX_TIME = 9877248000l; // 2282-12-31 UTC /** Conversion of numeric to DateTime64 */ @@ -493,7 +474,7 @@ struct ToDateTime64TransformUnsigned inline NO_SANITIZE_UNDEFINED DateTime64::NativeType execute(FromType from, const DateLUTImpl &) const { - from = std::min(time_t(from), time_t(0xFFFFFFFF)); + from = std::min(from, LUT_MAX_TIME); return DecimalUtils::decimalFromComponentsWithMultiplier(from, 0, scale_multiplier); } }; @@ -510,9 +491,8 @@ struct ToDateTime64TransformSigned inline NO_SANITIZE_UNDEFINED DateTime64::NativeType execute(FromType from, const DateLUTImpl &) const { - if (from < 0) - return 0; - from = std::min(time_t(from), time_t(0xFFFFFFFF)); + from = std::max(from, LUT_MIN_TIME); + from = std::min(from, LUT_MAX_TIME); return DecimalUtils::decimalFromComponentsWithMultiplier(from, 0, scale_multiplier); } }; @@ -551,6 +531,7 @@ template struct ConvertImpl struct ConvertImpl : DateTimeTransformImpl> {}; + /** Conversion of DateTime64 to Date or DateTime: discards fractional part. */ template @@ -571,10 +552,41 @@ struct FromDateTime64Transform } }; +/** Conversion of DateTime64 to Date or DateTime: discards fractional part. + */ template struct ConvertImpl - : DateTimeTransformImpl> {}; + : DateTimeTransformImpl> {}; template struct ConvertImpl - : DateTimeTransformImpl> {}; + : DateTimeTransformImpl> {}; + +struct ToDateTime64Transform +{ + static constexpr auto name = "toDateTime64"; + + const DateTime64::NativeType scale_multiplier = 1; + + ToDateTime64Transform(UInt32 scale = 0) + : scale_multiplier(DecimalUtils::scaleMultiplier(scale)) + {} + + inline DateTime64::NativeType execute(UInt16 d, const DateLUTImpl & time_zone) const + { + const auto dt = ToDateTimeImpl::execute(d, time_zone); + return execute(dt, time_zone); + } + + inline DateTime64::NativeType execute(UInt32 dt, const DateLUTImpl & /*time_zone*/) const + { + return DecimalUtils::decimalFromComponentsWithMultiplier(dt, 0, scale_multiplier); + } +}; + +/** Conversion of Date or DateTime to DateTime64: add zero sub-second part. + */ +template struct ConvertImpl + : DateTimeTransformImpl {}; +template struct ConvertImpl + : DateTimeTransformImpl {}; /** Transformation of numbers, dates, datetimes to strings: through formatting. @@ -658,7 +670,6 @@ struct ConvertImpl(*col_with_type_and_name.type); const DateLUTImpl * time_zone = nullptr; - /// For argument of DateTime type, second argument with time zone could be specified. if constexpr (std::is_same_v || std::is_same_v) time_zone = &extractTimeZoneFromFunctionArguments(arguments, 1, 0); @@ -754,6 +765,7 @@ inline void parseImpl(DataTypeDate::FieldType & x, ReadBuffer & rb x = tmp; } +// NOTE: no need of extra overload of DateTime64, since readDateTimeText64 has different signature and that case is explicitly handled in the calling code. template <> inline void parseImpl(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone) { @@ -762,6 +774,7 @@ inline void parseImpl(DataTypeDateTime::FieldType & x, ReadBuf x = tmp; } + template <> inline void parseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb, const DateLUTImpl *) { @@ -989,9 +1002,18 @@ struct ConvertThroughParsing } else if constexpr (parsing_mode == ConvertFromStringParsingMode::BestEffortUS) { - time_t res; - parseDateTimeBestEffortUS(res, read_buffer, *local_time_zone, *utc_time_zone); - vec_to[i] = res; + if constexpr (to_datetime64) + { + DateTime64 res = 0; + parseDateTime64BestEffortUS(res, vec_to.getScale(), read_buffer, *local_time_zone, *utc_time_zone); + vec_to[i] = res; + } + else + { + time_t res; + parseDateTimeBestEffortUS(res, read_buffer, *local_time_zone, *utc_time_zone); + vec_to[i] = res; + } } else { diff --git a/src/Functions/TransformDateTime64.h b/src/Functions/TransformDateTime64.h new file mode 100644 index 00000000000..0a5e36cd2bd --- /dev/null +++ b/src/Functions/TransformDateTime64.h @@ -0,0 +1,92 @@ +#pragma once + +#include +#include + +namespace DB +{ +/** Tansform-type wrapper for DateTime64, simplifies DateTime64 support for given Transform. + * + * Depending on what overloads of Transform::execute() are available, when called with DateTime64 value, + * invokes Transform::execute() with either: + * * whole part of DateTime64 value, discarding fractional part (1) + * * DateTime64 value and scale factor (2) + * * DateTime64 broken down to components, result of execute is then re-assembled back into DateTime64 value (3) + * + * Suitable Transfotm-types are commonly used in Date/DateTime manipulation functions, + * and should implement static (or const) function with following signatures: + * 1: + * R execute(Int64 whole_value, ... ) + * 2: + * R execute(DateTime64 value, Int64 scale_multiplier, ... ) + * 3: + * R execute(DecimalUtils::DecimalComponents components, ... ) + * + * Where R could be of arbitrary type, in case of (3) if R is DecimalUtils::DecimalComponents, result is re-assembed back into DateTime64. +*/ +template +class TransformDateTime64 +{ +private: + // Detect if Transform::execute is const or static method + // with signature defined by template args (ignoring result type). + template + struct TransformHasExecuteOverload : std::false_type {}; + + template + struct TransformHasExecuteOverload().execute(std::declval()...))>, Args...> + : std::true_type {}; + + template + static constexpr bool TransformHasExecuteOverload_v = TransformHasExecuteOverload::value; + +public: + static constexpr auto name = Transform::name; + + // non-explicit constructor to allow creating from scale value (or with no scale at all), indispensable in some contexts. + TransformDateTime64(UInt32 scale_ = 0) + : scale_multiplier(DecimalUtils::scaleMultiplier(scale_)) + {} + + template + inline auto execute(const DateTime64 & t, Args && ... args) const + { + if constexpr (TransformHasExecuteOverload_v) + { + return wrapped_transform.execute(t, scale_multiplier, std::forward(args)...); + } + else if constexpr (TransformHasExecuteOverload_v, Args...>) + { + auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); + + const auto result = wrapped_transform.execute(components, std::forward(args)...); + using ResultType = std::decay_t; + + if constexpr (std::is_same_v, ResultType>) + { + return DecimalUtils::decimalFromComponentsWithMultiplier(result, scale_multiplier); + } + else + { + return result; + } + } + else + { + const auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); + return wrapped_transform.execute(static_cast(components.whole), std::forward(args)...); + } + } + + template >>> + inline auto execute(const T & t, Args && ... args) const + { + return wrapped_transform.execute(t, std::forward(args)...); + } + +private: + DateTime64::NativeType scale_multiplier = 1; + Transform wrapped_transform = {}; +}; + +} diff --git a/src/Functions/dateDiff.cpp b/src/Functions/dateDiff.cpp index 54833eb359f..f660b92efc5 100644 --- a/src/Functions/dateDiff.cpp +++ b/src/Functions/dateDiff.cpp @@ -97,8 +97,8 @@ public: size_t rows = input_rows_count; auto res = ColumnInt64::create(rows); - const DateLUTImpl & timezone_x = extractTimeZoneFromFunctionArguments(arguments, 3, 1); - const DateLUTImpl & timezone_y = extractTimeZoneFromFunctionArguments(arguments, 3, 2); + const auto & timezone_x = extractTimeZoneFromFunctionArguments(arguments, 3, 1); + const auto & timezone_y = extractTimeZoneFromFunctionArguments(arguments, 3, 2); if (unit == "year" || unit == "yy" || unit == "yyyy") dispatchForColumns(x, y, timezone_x, timezone_y, res->getData()); diff --git a/src/Functions/extractTimeZoneFromFunctionArguments.cpp b/src/Functions/extractTimeZoneFromFunctionArguments.cpp index 9d6e54a599e..0ba08b3c612 100644 --- a/src/Functions/extractTimeZoneFromFunctionArguments.cpp +++ b/src/Functions/extractTimeZoneFromFunctionArguments.cpp @@ -66,10 +66,11 @@ const DateLUTImpl & extractTimeZoneFromFunctionArguments(const ColumnsWithTypeAn if (arguments.empty()) return DateLUT::instance(); + const auto & dt_arg = arguments[datetime_arg_num].type.get(); /// If time zone is attached to an argument of type DateTime. - if (const auto * type = checkAndGetDataType(arguments[datetime_arg_num].type.get())) + if (const auto * type = checkAndGetDataType(dt_arg)) return type->getTimeZone(); - if (const auto * type = checkAndGetDataType(arguments[datetime_arg_num].type.get())) + if (const auto * type = checkAndGetDataType(dt_arg)) return type->getTimeZone(); return DateLUT::instance(); diff --git a/src/Functions/formatDateTime.cpp b/src/Functions/formatDateTime.cpp index fd909ed6fce..5128f077c5a 100644 --- a/src/Functions/formatDateTime.cpp +++ b/src/Functions/formatDateTime.cpp @@ -46,9 +46,8 @@ template <> struct ActionValueTypeMap { using ActionValueTyp template <> struct ActionValueTypeMap { using ActionValueType = UInt32; }; template <> struct ActionValueTypeMap { using ActionValueType = UInt16; }; template <> struct ActionValueTypeMap { using ActionValueType = UInt32; }; -// TODO(vnemkov): once there is support for Int64 in LUT, make that Int64. // TODO(vnemkov): to add sub-second format instruction, make that DateTime64 and do some math in Action. -template <> struct ActionValueTypeMap { using ActionValueType = UInt32; }; +template <> struct ActionValueTypeMap { using ActionValueType = Int64; }; /** formatDateTime(time, 'pattern') @@ -434,7 +433,6 @@ public: time_zone_tmp = &DateLUT::instance(); const DateLUTImpl & time_zone = *time_zone_tmp; - const auto & vec = times->getData(); UInt32 scale [[maybe_unused]] = 0; @@ -519,6 +517,8 @@ public: { if constexpr (std::is_same_v) instructions.emplace_back(func, shift); + else if constexpr (std::is_same_v) + instructions.emplace_back(func, shift); else add_shift(shift); }; diff --git a/src/Functions/now64.cpp b/src/Functions/now64.cpp index feb821fde82..32c7a95de17 100644 --- a/src/Functions/now64.cpp +++ b/src/Functions/now64.cpp @@ -30,7 +30,7 @@ Field nowSubsecond(UInt32 scale) if (clock_gettime(CLOCK_REALTIME, &spec)) throwFromErrno("Cannot clock_gettime.", ErrorCodes::CANNOT_CLOCK_GETTIME); - DecimalUtils::DecimalComponents components{spec.tv_sec, spec.tv_nsec}; + DecimalUtils::DecimalComponents components{spec.tv_sec, spec.tv_nsec}; // clock_gettime produces subsecond part in nanoseconds, but decimalFromComponents fractional is scale-dependent. // Andjust fractional to scale, e.g. for 123456789 nanoseconds: diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp index 6f5a52ca182..f194da166aa 100644 --- a/src/Functions/toStartOfInterval.cpp +++ b/src/Functions/toStartOfInterval.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -35,13 +36,18 @@ namespace static UInt16 execute(UInt16 d, UInt64 years, const DateLUTImpl & time_zone) { - return time_zone.toStartOfYearInterval(DayNum(d), years); + return time_zone.toStartOfYearInterval(ExtendedDayNum(d), years); } static UInt16 execute(UInt32 t, UInt64 years, const DateLUTImpl & time_zone) { return time_zone.toStartOfYearInterval(time_zone.toDayNum(t), years); } + + static UInt16 execute(Int64 t, UInt64 years, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfYearInterval(time_zone.toDayNum(t), years); + } }; template <> @@ -51,13 +57,18 @@ namespace static UInt16 execute(UInt16 d, UInt64 quarters, const DateLUTImpl & time_zone) { - return time_zone.toStartOfQuarterInterval(DayNum(d), quarters); + return time_zone.toStartOfQuarterInterval(ExtendedDayNum(d), quarters); } static UInt16 execute(UInt32 t, UInt64 quarters, const DateLUTImpl & time_zone) { return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t), quarters); } + + static UInt16 execute(Int64 t, UInt64 quarters, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfQuarterInterval(time_zone.toDayNum(t), quarters); + } }; template <> @@ -67,13 +78,18 @@ namespace static UInt16 execute(UInt16 d, UInt64 months, const DateLUTImpl & time_zone) { - return time_zone.toStartOfMonthInterval(DayNum(d), months); + return time_zone.toStartOfMonthInterval(ExtendedDayNum(d), months); } static UInt16 execute(UInt32 t, UInt64 months, const DateLUTImpl & time_zone) { return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t), months); } + + static UInt16 execute(Int64 t, UInt64 months, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfMonthInterval(time_zone.toDayNum(t), months); + } }; template <> @@ -83,13 +99,18 @@ namespace static UInt16 execute(UInt16 d, UInt64 weeks, const DateLUTImpl & time_zone) { - return time_zone.toStartOfWeekInterval(DayNum(d), weeks); + return time_zone.toStartOfWeekInterval(ExtendedDayNum(d), weeks); } static UInt16 execute(UInt32 t, UInt64 weeks, const DateLUTImpl & time_zone) { return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t), weeks); } + + static UInt16 execute(Int64 t, UInt64 weeks, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfWeekInterval(time_zone.toDayNum(t), weeks); + } }; template <> @@ -99,13 +120,18 @@ namespace static UInt32 execute(UInt16 d, UInt64 days, const DateLUTImpl & time_zone) { - return time_zone.toStartOfDayInterval(DayNum(d), days); + return time_zone.toStartOfDayInterval(ExtendedDayNum(d), days); } static UInt32 execute(UInt32 t, UInt64 days, const DateLUTImpl & time_zone) { return time_zone.toStartOfDayInterval(time_zone.toDayNum(t), days); } + + static UInt32 execute(Int64 t, UInt64 days, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfDayInterval(time_zone.toDayNum(t), days); + } }; template <> @@ -114,8 +140,8 @@ namespace static constexpr auto name = function_name; static UInt32 execute(UInt16, UInt64, const DateLUTImpl &) { return dateIsNotSupported(function_name); } - static UInt32 execute(UInt32 t, UInt64 hours, const DateLUTImpl & time_zone) { return time_zone.toStartOfHourInterval(t, hours); } + static UInt32 execute(Int64 t, UInt64 hours, const DateLUTImpl & time_zone) { return time_zone.toStartOfHourInterval(t, hours); } }; template <> @@ -129,6 +155,11 @@ namespace { return time_zone.toStartOfMinuteInterval(t, minutes); } + + static UInt32 execute(Int64 t, UInt64 minutes, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfMinuteInterval(t, minutes); + } }; template <> @@ -142,6 +173,11 @@ namespace { return time_zone.toStartOfSecondInterval(t, seconds); } + + static Int64 execute(Int64 t, UInt64 seconds, const DateLUTImpl & time_zone) + { + return time_zone.toStartOfSecondInterval(t, seconds); + } }; @@ -230,7 +266,7 @@ public: { const auto & time_column = arguments[0]; const auto & interval_column = arguments[1]; - const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(arguments, 2, 0); + const auto & time_zone = extractTimeZoneFromFunctionArguments(arguments, 2, 0); auto result_column = dispatchForColumns(time_column, interval_column, time_zone); return result_column; } diff --git a/src/Functions/today.cpp b/src/Functions/today.cpp index 65373058540..511af881d73 100644 --- a/src/Functions/today.cpp +++ b/src/Functions/today.cpp @@ -77,7 +77,7 @@ public: FunctionBaseImplPtr build(const ColumnsWithTypeAndName &, const DataTypePtr &) const override { - return std::make_unique(DateLUT::instance().toDayNum(time(nullptr))); + return std::make_unique(DayNum(DateLUT::instance().toDayNum(time(nullptr)).toUnderType())); } }; diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index e33de04f322..369237f329d 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -747,7 +747,7 @@ inline ReturnType readDateTimeTextImpl(DateTime64 & datetime64, UInt32 scale, Re return ReturnType(false); } - DB::DecimalUtils::DecimalComponents components{static_cast(whole), 0}; + DB::DecimalUtils::DecimalComponents components{static_cast(whole), 0}; if (!buf.eof() && *buf.position() == '.') { @@ -791,9 +791,9 @@ inline ReturnType readDateTimeTextImpl(DateTime64 & datetime64, UInt32 scale, Re return ReturnType(true); } -inline void readDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) +inline void readDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { - readDateTimeTextImpl(datetime, buf, date_lut); + readDateTimeTextImpl(datetime, buf, time_zone); } inline void readDateTime64Text(DateTime64 & datetime64, UInt32 scale, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) @@ -801,9 +801,9 @@ inline void readDateTime64Text(DateTime64 & datetime64, UInt32 scale, ReadBuffer readDateTimeTextImpl(datetime64, scale, buf, date_lut); } -inline bool tryReadDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) +inline bool tryReadDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { - return readDateTimeTextImpl(datetime, buf, date_lut); + return readDateTimeTextImpl(datetime, buf, time_zone); } inline bool tryReadDateTime64Text(DateTime64 & datetime64, UInt32 scale, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index a382ae13cdd..72c6b69114f 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -819,12 +819,12 @@ inline void writeDateTimeText(const LocalDateTime & datetime, WriteBuffer & buf) /// In the format YYYY-MM-DD HH:MM:SS, according to the specified time zone. template -inline void writeDateTimeText(time_t datetime, WriteBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) +inline void writeDateTimeText(time_t datetime, WriteBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { - const auto & values = date_lut.getValues(datetime); + const auto & values = time_zone.getValues(datetime); writeDateTimeText( LocalDateTime(values.year, values.month, values.day_of_month, - date_lut.toHour(datetime), date_lut.toMinute(datetime), date_lut.toSecond(datetime)), buf); + time_zone.toHour(datetime), time_zone.toMinute(datetime), time_zone.toSecond(datetime)), buf); } /// In the format YYYY-MM-DD HH:MM:SS.NNNNNNNNN, according to the specified time zone. @@ -849,9 +849,9 @@ inline void writeDateTimeText(DateTime64 datetime64, UInt32 scale, WriteBuffer & /// In the RFC 1123 format: "Tue, 03 Dec 2019 00:11:50 GMT". You must provide GMT DateLUT. /// This is needed for HTTP requests. -inline void writeDateTimeTextRFC1123(time_t datetime, WriteBuffer & buf, const DateLUTImpl & date_lut) +inline void writeDateTimeTextRFC1123(time_t datetime, WriteBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { - const auto & values = date_lut.getValues(datetime); + const auto & values = time_zone.getValues(datetime); static const char week_days[3 * 8 + 1] = "XXX" "Mon" "Tue" "Wed" "Thu" "Fri" "Sat" "Sun"; static const char months[3 * 13 + 1] = "XXX" "Jan" "Feb" "Mar" "Apr" "May" "Jun" "Jul" "Aug" "Sep" "Oct" "Nov" "Dec"; @@ -865,11 +865,11 @@ inline void writeDateTimeTextRFC1123(time_t datetime, WriteBuffer & buf, const D buf.write(&digits100[values.year / 100 * 2], 2); buf.write(&digits100[values.year % 100 * 2], 2); buf.write(' '); - buf.write(&digits100[date_lut.toHour(datetime) * 2], 2); + buf.write(&digits100[time_zone.toHour(datetime) * 2], 2); buf.write(':'); - buf.write(&digits100[date_lut.toMinute(datetime) * 2], 2); + buf.write(&digits100[time_zone.toMinute(datetime) * 2], 2); buf.write(':'); - buf.write(&digits100[date_lut.toSecond(datetime) * 2], 2); + buf.write(&digits100[time_zone.toSecond(datetime) * 2], 2); buf.write(" GMT", 4); } diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index 47a298ede29..26745a8f138 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -600,7 +600,7 @@ ReturnType parseDateTimeBestEffortImpl( return ReturnType(true); } -template +template ReturnType parseDateTime64BestEffortImpl(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) { time_t whole; @@ -608,12 +608,12 @@ ReturnType parseDateTime64BestEffortImpl(DateTime64 & res, UInt32 scale, ReadBuf if constexpr (std::is_same_v) { - if (!parseDateTimeBestEffortImpl(whole, in, local_time_zone, utc_time_zone, &subsecond)) + if (!parseDateTimeBestEffortImpl(whole, in, local_time_zone, utc_time_zone, &subsecond)) return false; } else { - parseDateTimeBestEffortImpl(whole, in, local_time_zone, utc_time_zone, &subsecond); + parseDateTimeBestEffortImpl(whole, in, local_time_zone, utc_time_zone, &subsecond); } @@ -661,12 +661,17 @@ bool tryParseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTIm void parseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) { - return parseDateTime64BestEffortImpl(res, scale, in, local_time_zone, utc_time_zone); + return parseDateTime64BestEffortImpl(res, scale, in, local_time_zone, utc_time_zone); +} + +void parseDateTime64BestEffortUS(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) +{ + return parseDateTime64BestEffortImpl(res, scale, in, local_time_zone, utc_time_zone); } bool tryParseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) { - return parseDateTime64BestEffortImpl(res, scale, in, local_time_zone, utc_time_zone); + return parseDateTime64BestEffortImpl(res, scale, in, local_time_zone, utc_time_zone); } } diff --git a/src/IO/parseDateTimeBestEffort.h b/src/IO/parseDateTimeBestEffort.h index 65e92cbee42..fe3da24a797 100644 --- a/src/IO/parseDateTimeBestEffort.h +++ b/src/IO/parseDateTimeBestEffort.h @@ -61,6 +61,7 @@ bool tryParseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl void parseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); bool tryParseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); void parseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); +void parseDateTime64BestEffortUS(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); bool tryParseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); } diff --git a/src/Interpreters/CrashLog.cpp b/src/Interpreters/CrashLog.cpp index bf81a2e8aba..5067acd4a5c 100644 --- a/src/Interpreters/CrashLog.cpp +++ b/src/Interpreters/CrashLog.cpp @@ -40,7 +40,7 @@ void CrashLogElement::appendToBlock(MutableColumns & columns) const { size_t i = 0; - columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(timestamp_ns); columns[i++]->insert(signal); diff --git a/src/Interpreters/MetricLog.cpp b/src/Interpreters/MetricLog.cpp index ce5d5793b87..fd1c120f18c 100644 --- a/src/Interpreters/MetricLog.cpp +++ b/src/Interpreters/MetricLog.cpp @@ -41,7 +41,7 @@ void MetricLogElement::appendToBlock(MutableColumns & columns) const { size_t column_idx = 0; - columns[column_idx++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[column_idx++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[column_idx++]->insert(event_time); columns[column_idx++]->insert(event_time_microseconds); columns[column_idx++]->insert(milliseconds); diff --git a/src/Interpreters/OpenTelemetrySpanLog.cpp b/src/Interpreters/OpenTelemetrySpanLog.cpp index e1df145cf51..f9ae6518af0 100644 --- a/src/Interpreters/OpenTelemetrySpanLog.cpp +++ b/src/Interpreters/OpenTelemetrySpanLog.cpp @@ -49,7 +49,7 @@ void OpenTelemetrySpanLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(operation_name); columns[i++]->insert(start_time_us); columns[i++]->insert(finish_time_us); - columns[i++]->insert(DateLUT::instance().toDayNum(finish_time_us / 1000000)); + columns[i++]->insert(DateLUT::instance().toDayNum(finish_time_us / 1000000).toUnderType()); columns[i++]->insert(attribute_names); // The user might add some ints values, and we will have Int Field, and the // insert will fail because the column requires Strings. Convert the fields diff --git a/src/Interpreters/PartLog.cpp b/src/Interpreters/PartLog.cpp index 860666a0035..c180a4dd254 100644 --- a/src/Interpreters/PartLog.cpp +++ b/src/Interpreters/PartLog.cpp @@ -71,7 +71,7 @@ void PartLogElement::appendToBlock(MutableColumns & columns) const columns[i++]->insert(query_id); columns[i++]->insert(event_type); - columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(event_time_microseconds); columns[i++]->insert(duration_ms); diff --git a/src/Interpreters/QueryLog.cpp b/src/Interpreters/QueryLog.cpp index 82b957f895b..b6902468242 100644 --- a/src/Interpreters/QueryLog.cpp +++ b/src/Interpreters/QueryLog.cpp @@ -119,7 +119,7 @@ void QueryLogElement::appendToBlock(MutableColumns & columns) const size_t i = 0; columns[i++]->insert(type); - columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(event_time_microseconds); columns[i++]->insert(query_start_time); diff --git a/src/Interpreters/QueryThreadLog.cpp b/src/Interpreters/QueryThreadLog.cpp index f1cce1a3da9..31f1fddc87f 100644 --- a/src/Interpreters/QueryThreadLog.cpp +++ b/src/Interpreters/QueryThreadLog.cpp @@ -76,7 +76,7 @@ void QueryThreadLogElement::appendToBlock(MutableColumns & columns) const { size_t i = 0; - columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(event_time_microseconds); columns[i++]->insert(query_start_time); diff --git a/src/Interpreters/TextLog.cpp b/src/Interpreters/TextLog.cpp index f60b6acae6f..489bb302ad0 100644 --- a/src/Interpreters/TextLog.cpp +++ b/src/Interpreters/TextLog.cpp @@ -55,7 +55,7 @@ void TextLogElement::appendToBlock(MutableColumns & columns) const { size_t i = 0; - columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(event_time_microseconds); columns[i++]->insert(microseconds); diff --git a/src/Interpreters/TraceLog.cpp b/src/Interpreters/TraceLog.cpp index 40bcc0db445..fe7512f2f00 100644 --- a/src/Interpreters/TraceLog.cpp +++ b/src/Interpreters/TraceLog.cpp @@ -42,7 +42,7 @@ void TraceLogElement::appendToBlock(MutableColumns & columns) const { size_t i = 0; - columns[i++]->insert(DateLUT::instance().toDayNum(event_time)); + columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType()); columns[i++]->insert(event_time); columns[i++]->insert(event_time_microseconds); columns[i++]->insert(timestamp_ns); diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp index 1d93ef56dea..d47f64cb1dc 100644 --- a/src/Interpreters/convertFieldToType.cpp +++ b/src/Interpreters/convertFieldToType.cpp @@ -141,7 +141,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID /// Conversion between Date and DateTime and vice versa. if (which_type.isDate() && which_from_type.isDateTime()) { - return static_cast(*from_type_hint).getTimeZone().toDayNum(src.get()); + return static_cast(static_cast(*from_type_hint).getTimeZone().toDayNum(src.get()).toUnderType()); } else if (which_type.isDateTime() && which_from_type.isDate()) { diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index f2c88cdedd9..8bf785afa0f 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4369,7 +4369,7 @@ static String getPartNamePossiblyFake(MergeTreeDataFormatVersion format_version, /// The date range is all month long. const auto & lut = DateLUT::instance(); time_t start_time = lut.YYYYMMDDToDate(parse(part_info.partition_id + "01")); - DayNum left_date = lut.toDayNum(start_time); + DayNum left_date = DayNum{lut.toDayNum(start_time).toUnderType()}; DayNum right_date = DayNum(static_cast(left_date) + lut.daysInMonth(start_time) - 1); return part_info.getPartNameV0(left_date, right_date); } diff --git a/src/Storages/tests/part_name.cpp b/src/Storages/tests/part_name.cpp index 79c5578a8ca..aeadfd208cc 100644 --- a/src/Storages/tests/part_name.cpp +++ b/src/Storages/tests/part_name.cpp @@ -5,7 +5,7 @@ int main(int, char **) { - DayNum today = DateLUT::instance().toDayNum(time(nullptr)); + const DayNum today{DateLUT::instance().toDayNum(time(nullptr)).toUnderType()}; for (DayNum date = today; DayNum(date + 10) > today; --date) { diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility.python b/tests/queries/0_stateless/00921_datetime64_compatibility.python index bf0ae8a72ac..c8b9620629d 100644 --- a/tests/queries/0_stateless/00921_datetime64_compatibility.python +++ b/tests/queries/0_stateless/00921_datetime64_compatibility.python @@ -86,8 +86,7 @@ CAST(N as DateTime64(9, 'Europe/Minsk')) formatDateTime(N, '%C %d %D %e %F %H %I %j %m %M %p %R %S %T %u %V %w %y %Y %%') """.splitlines() -# Expanded later to cartesian product of all arguments. -# NOTE: {N} to be turned into N after str.format() for keys (format string), but not for list of values! +# Expanded later to cartesian product of all arguments, using format string. extra_ops = [ # With same type: ( @@ -179,7 +178,7 @@ def escape_string(s): def execute_functions_for_types(functions, types): - # TODO: use string.Template here to allow lines that do not contain type, like: SELECT CAST(toDateTime64(1234567890), 'DateTime64') + # NOTE: use string.Template here to allow lines with missing keys, like type, e.g. SELECT CAST(toDateTime64(1234567890), 'DateTime64') for func in functions: print(("""SELECT 'SELECT {func}';""".format(func=escape_string(func)))) for dt in types: diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility.reference b/tests/queries/0_stateless/00921_datetime64_compatibility.reference index 004f4f5e824..67413512e06 100644 --- a/tests/queries/0_stateless/00921_datetime64_compatibility.reference +++ b/tests/queries/0_stateless/00921_datetime64_compatibility.reference @@ -1,5 +1,4 @@ SELECT toTimeZone(N, \'UTC\') - Code: 43 "DateTime('UTC')","2019-09-16 16:20:11" "DateTime64(3, 'UTC')","2019-09-16 16:20:11.234" @@ -35,25 +34,21 @@ SELECT toDayOfWeek(N) "UInt8",1 ------------------------------------------ SELECT toHour(N) - Code: 43 "UInt8",19 "UInt8",19 ------------------------------------------ SELECT toMinute(N) - Code: 43 "UInt8",20 "UInt8",20 ------------------------------------------ SELECT toSecond(N) - Code: 43 "UInt8",11 "UInt8",11 ------------------------------------------ SELECT toUnixTimestamp(N) - Code: 44 "UInt32",1568650811 "UInt32",1568650811 @@ -94,31 +89,26 @@ SELECT toStartOfDay(N) "DateTime('Europe/Minsk')","2019-09-16 00:00:00" ------------------------------------------ SELECT toStartOfHour(N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:00:00" "DateTime('Europe/Minsk')","2019-09-16 19:00:00" ------------------------------------------ SELECT toStartOfMinute(N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:20:00" "DateTime('Europe/Minsk')","2019-09-16 19:20:00" ------------------------------------------ SELECT toStartOfFiveMinute(N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:20:00" "DateTime('Europe/Minsk')","2019-09-16 19:20:00" ------------------------------------------ SELECT toStartOfTenMinutes(N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:20:00" "DateTime('Europe/Minsk')","2019-09-16 19:20:00" ------------------------------------------ SELECT toStartOfFifteenMinutes(N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:15:00" "DateTime('Europe/Minsk')","2019-09-16 19:15:00" @@ -139,7 +129,6 @@ SELECT toStartOfInterval(N, INTERVAL 1 day) "DateTime('Europe/Minsk')","2019-09-16 00:00:00" ------------------------------------------ SELECT toStartOfInterval(N, INTERVAL 15 minute) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:15:00" "DateTime('Europe/Minsk')","2019-09-16 19:15:00" @@ -160,13 +149,11 @@ SELECT date_trunc(\'day\', N) "DateTime('Europe/Minsk')","2019-09-16 00:00:00" ------------------------------------------ SELECT date_trunc(\'minute\', N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:20:00" "DateTime('Europe/Minsk')","2019-09-16 19:20:00" ------------------------------------------ SELECT toTime(N) - Code: 43 "DateTime('Europe/Minsk')","1970-01-02 19:20:11" "DateTime('Europe/Minsk')","1970-01-02 19:20:11" @@ -232,7 +219,6 @@ SELECT toYearWeek(N) "UInt32",201937 ------------------------------------------ SELECT timeSlot(N) - Code: 43 "DateTime('Europe/Minsk')","2019-09-16 19:00:00" "DateTime('Europe/Minsk')","2019-09-16 19:00:00" @@ -375,15 +361,11 @@ SELECT formatDateTime(N, \'%C %d %D %e %F %H %I %j %m %M %p %R %S %T %u %V %w %y SELECT N - N "Int32",0 "Int32",0 - Code: 43 ------------------------------------------ SELECT N + N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N != N @@ -417,47 +399,33 @@ SELECT N >= N "UInt8",1 ------------------------------------------ SELECT N - DT - Code: 43 "Int32",0 - Code: 43 ------------------------------------------ SELECT DT - N - Code: 43 "Int32",0 - Code: 43 ------------------------------------------ SELECT N - D "Int32",0 - Code: 43 - Code: 43 ------------------------------------------ SELECT D - N "Int32",0 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - DT64 - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT DT64 - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N != DT @@ -726,11 +694,8 @@ SELECT N - toUInt8(1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:10.234" ------------------------------------------ SELECT toUInt8(1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toInt8(-1) @@ -739,11 +704,8 @@ SELECT N - toInt8(-1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:12.234" ------------------------------------------ SELECT toInt8(-1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toUInt16(1) @@ -752,11 +714,8 @@ SELECT N - toUInt16(1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:10.234" ------------------------------------------ SELECT toUInt16(1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toInt16(-1) @@ -765,11 +724,8 @@ SELECT N - toInt16(-1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:12.234" ------------------------------------------ SELECT toInt16(-1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toUInt32(1) @@ -778,11 +734,8 @@ SELECT N - toUInt32(1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:10.234" ------------------------------------------ SELECT toUInt32(1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toInt32(-1) @@ -791,11 +744,8 @@ SELECT N - toInt32(-1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:12.234" ------------------------------------------ SELECT toInt32(-1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toUInt64(1) @@ -804,11 +754,8 @@ SELECT N - toUInt64(1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:10.234" ------------------------------------------ SELECT toUInt64(1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N - toInt64(-1) @@ -817,585 +764,486 @@ SELECT N - toInt64(-1) "DateTime64(3, 'Europe/Minsk')","2019-09-16 19:20:12.234" ------------------------------------------ SELECT toInt64(-1) - N - Code: 43 - Code: 43 - Code: 43 ------------------------------------------ SELECT N == toUInt8(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt8(1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toInt8(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt8(-1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toUInt16(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt16(1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toInt16(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt16(-1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toUInt32(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt32(1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toInt32(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt32(-1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toUInt64(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt64(1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N == toInt64(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt64(-1) == N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N != toUInt8(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt8(1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toInt8(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt8(-1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toUInt16(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt16(1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toInt16(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt16(-1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toUInt32(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt32(1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toInt32(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt32(-1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toUInt64(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt64(1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N != toInt64(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt64(-1) != N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toUInt8(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt8(1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toInt8(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt8(-1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toUInt16(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt16(1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toInt16(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt16(-1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toUInt32(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt32(1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toInt32(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt32(-1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toUInt64(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt64(1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N < toInt64(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt64(-1) < N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toUInt8(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt8(1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toInt8(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt8(-1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toUInt16(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt16(1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toInt16(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt16(-1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toUInt32(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt32(1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toInt32(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt32(-1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toUInt64(1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toUInt64(1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N <= toInt64(-1) - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT toInt64(-1) <= N - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT N > toUInt8(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt8(1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toInt8(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt8(-1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toUInt16(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt16(1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toInt16(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt16(-1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toUInt32(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt32(1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toInt32(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt32(-1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toUInt64(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt64(1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N > toInt64(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt64(-1) > N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toUInt8(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt8(1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toInt8(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt8(-1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toUInt16(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt16(1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toInt16(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt16(-1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toUInt32(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt32(1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toInt32(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt32(-1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toUInt64(1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toUInt64(1) >= N - Code: 43 "UInt8",0 "UInt8",0 ------------------------------------------ SELECT N >= toInt64(-1) - Code: 43 "UInt8",1 "UInt8",1 ------------------------------------------ SELECT toInt64(-1) >= N - Code: 43 "UInt8",0 "UInt8",0 diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility.sh b/tests/queries/0_stateless/00921_datetime64_compatibility.sh index 1617e5b1f77..5f5034819e4 100755 --- a/tests/queries/0_stateless/00921_datetime64_compatibility.sh +++ b/tests/queries/0_stateless/00921_datetime64_compatibility.sh @@ -13,4 +13,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) python3 "${CURDIR}"/00921_datetime64_compatibility.python \ | ${CLICKHOUSE_CLIENT} --ignore-error -T -nm --calculate_text_stack_trace 0 --log-level 'error' 2>&1 \ - | sed 's/Received exception .*//g; s/^\(Code: [0-9]\+\).*$/\1/g' + | grep -v 'Received exception .*$' | sed 's/^\(Code: [0-9]\+\).*$/\1/g' diff --git a/tests/queries/0_stateless/01252_weird_time_zone.reference b/tests/queries/0_stateless/01252_weird_time_zone.reference index f2968d4efa6..90f5bf0e30d 100644 --- a/tests/queries/0_stateless/01252_weird_time_zone.reference +++ b/tests/queries/0_stateless/01252_weird_time_zone.reference @@ -1,7 +1,7 @@ -2020-01-02 03:04:05 2020-01-02 00:00:00 3 -2020-01-02 03:04:05 2020-01-02 00:00:00 3 -2020-01-02 03:04:05 2020-01-02 00:00:00 3 -2020-01-02 03:04:05 2020-01-02 00:00:00 3 -2020-01-02 03:04:05 2020-01-02 00:00:00 3 -2020-01-02 03:04:05 2020-01-02 00:00:00 3 -2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Pacific/Kiritimati 2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Africa/El_Aaiun 2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Asia/Pyongyang 2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Pacific/Kwajalein 2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Pacific/Apia 2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Pacific/Enderbury 2020-01-02 03:04:05 2020-01-02 00:00:00 3 +Pacific/Fakaofo 2020-01-02 03:04:05 2020-01-02 00:00:00 3 diff --git a/tests/queries/0_stateless/01252_weird_time_zone.sql b/tests/queries/0_stateless/01252_weird_time_zone.sql index 68ea903a797..c4919ca4fe0 100644 --- a/tests/queries/0_stateless/01252_weird_time_zone.sql +++ b/tests/queries/0_stateless/01252_weird_time_zone.sql @@ -1,15 +1,15 @@ -SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Kiritimati') AS x, toStartOfDay(x), toHour(x); -SELECT toDateTime('2020-01-02 03:04:05', 'Africa/El_Aaiun') AS x, toStartOfDay(x), toHour(x); -SELECT toDateTime('2020-01-02 03:04:05', 'Asia/Pyongyang') AS x, toStartOfDay(x), toHour(x); -SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Kwajalein') AS x, toStartOfDay(x), toHour(x); -SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Apia') AS x, toStartOfDay(x), toHour(x); -SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Enderbury') AS x, toStartOfDay(x), toHour(x); -SELECT toDateTime('2020-01-02 03:04:05', 'Pacific/Fakaofo') AS x, toStartOfDay(x), toHour(x); +SELECT 'Pacific/Kiritimati', toDateTime('2020-01-02 03:04:05', 'Pacific/Kiritimati') AS x, toStartOfDay(x), toHour(x); +SELECT 'Africa/El_Aaiun', toDateTime('2020-01-02 03:04:05', 'Africa/El_Aaiun') AS x, toStartOfDay(x), toHour(x); +SELECT 'Asia/Pyongyang', toDateTime('2020-01-02 03:04:05', 'Asia/Pyongyang') AS x, toStartOfDay(x), toHour(x); +SELECT 'Pacific/Kwajalein', toDateTime('2020-01-02 03:04:05', 'Pacific/Kwajalein') AS x, toStartOfDay(x), toHour(x); +SELECT 'Pacific/Apia', toDateTime('2020-01-02 03:04:05', 'Pacific/Apia') AS x, toStartOfDay(x), toHour(x); +SELECT 'Pacific/Enderbury', toDateTime('2020-01-02 03:04:05', 'Pacific/Enderbury') AS x, toStartOfDay(x), toHour(x); +SELECT 'Pacific/Fakaofo', toDateTime('2020-01-02 03:04:05', 'Pacific/Fakaofo') AS x, toStartOfDay(x), toHour(x); -SELECT toHour(toDateTime(rand(), 'Pacific/Kiritimati') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; -SELECT toHour(toDateTime(rand(), 'Africa/El_Aaiun') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; -SELECT toHour(toDateTime(rand(), 'Asia/Pyongyang') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; -SELECT toHour(toDateTime(rand(), 'Pacific/Kwajalein') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; -SELECT toHour(toDateTime(rand(), 'Pacific/Apia') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; -SELECT toHour(toDateTime(rand(), 'Pacific/Enderbury') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; -SELECT toHour(toDateTime(rand(), 'Pacific/Fakaofo') AS t) AS h, t FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Pacific/Kiritimati', rand() as r, toHour(toDateTime(r, 'Pacific/Kiritimati') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Africa/El_Aaiun', rand() as r, toHour(toDateTime(r, 'Africa/El_Aaiun') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Asia/Pyongyang', rand() as r, toHour(toDateTime(r, 'Asia/Pyongyang') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Pacific/Kwajalein', rand() as r, toHour(toDateTime(r, 'Pacific/Kwajalein') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Pacific/Apia', rand() as r, toHour(toDateTime(r, 'Pacific/Apia') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Pacific/Enderbury', rand() as r, toHour(toDateTime(r, 'Pacific/Enderbury') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; +SELECT 'Pacific/Fakaofo', rand() as r, toHour(toDateTime(r, 'Pacific/Fakaofo') AS t) AS h, t, toTypeName(t) FROM numbers(1000000) WHERE h < 0 OR h > 23 ORDER BY h LIMIT 1 BY h; diff --git a/tests/queries/0_stateless/01440_to_date_monotonicity.reference b/tests/queries/0_stateless/01440_to_date_monotonicity.reference index 96732e5996c..74716fe6223 100644 --- a/tests/queries/0_stateless/01440_to_date_monotonicity.reference +++ b/tests/queries/0_stateless/01440_to_date_monotonicity.reference @@ -1,4 +1,4 @@ 0 -1970-01-01 2106-02-07 1970-04-11 1970-01-01 2106-02-07 +1970-01-01 2106-02-07 1970-04-11 1970-01-01 2149-06-06 1970-01-01 03:00:00 2106-02-07 09:28:15 1970-01-01 03:16:40 2000-01-01 13:12:12 diff --git a/tests/queries/0_stateless/01561_Date_and_DateTime64_comparision.sql b/tests/queries/0_stateless/01561_Date_and_DateTime64_comparision.sql index 7e75d871e07..a61bcff4db7 100644 --- a/tests/queries/0_stateless/01561_Date_and_DateTime64_comparision.sql +++ b/tests/queries/0_stateless/01561_Date_and_DateTime64_comparision.sql @@ -6,7 +6,7 @@ SELECT dt64 < d, toDate(dt64) < d, dt64 < toDateTime64(d, 1, 'UTC'), - + '<=', dt64 <= d, toDate(dt64) <= d, @@ -16,7 +16,7 @@ SELECT dt64 = d, toDate(dt64) = d, dt64 = toDateTime64(d, 1, 'UTC'), - + '>=', dt64 >= d, toDate(dt64) >= d, @@ -31,7 +31,7 @@ SELECT dt64 != d, toDate(dt64) != d, dt64 != toDateTime64(d, 1, 'UTC') -FROM +FROM ( WITH toDateTime('2019-09-16 19:20:11') as val SELECT diff --git a/tests/queries/0_stateless/01631_date_overflow_as_partition_key.reference b/tests/queries/0_stateless/01631_date_overflow_as_partition_key.reference index dbcd92da11c..62f620f3ba9 100644 --- a/tests/queries/0_stateless/01631_date_overflow_as_partition_key.reference +++ b/tests/queries/0_stateless/01631_date_overflow_as_partition_key.reference @@ -1,2 +1,2 @@ -1970-01-01 1 -1970-01-01 1 +2106-11-11 1 +2106-11-12 1 diff --git a/tests/queries/0_stateless/01631_date_overflow_as_partition_key.sql b/tests/queries/0_stateless/01631_date_overflow_as_partition_key.sql index f252e10806a..9a8d37084fb 100644 --- a/tests/queries/0_stateless/01631_date_overflow_as_partition_key.sql +++ b/tests/queries/0_stateless/01631_date_overflow_as_partition_key.sql @@ -6,6 +6,6 @@ insert into dt_overflow values('2106-11-11', 1); insert into dt_overflow values('2106-11-12', 1); -select * from dt_overflow; +select * from dt_overflow ORDER BY d; drop table if exists dt_overflow; diff --git a/tests/queries/0_stateless/01691_DateTime64_clamp.reference b/tests/queries/0_stateless/01691_DateTime64_clamp.reference index 3adc9a17e5c..da80de59e50 100644 --- a/tests/queries/0_stateless/01691_DateTime64_clamp.reference +++ b/tests/queries/0_stateless/01691_DateTime64_clamp.reference @@ -1,4 +1,5 @@ -- { echo } +<<<<<<< HEAD SELECT toTimeZone(toDateTime(-2, 2), 'Europe/Moscow'); 1970-01-01 03:00:00.00 SELECT toDateTime64(-2, 2, 'Europe/Moscow'); @@ -15,3 +16,25 @@ SELECT toDateTime64(-2., 2, 'Europe/Moscow'); SELECT toDateTime64(toFloat32(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow'); 2106-02-07 09:00:00.00 SELECT toDateTime64(toFloat64(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow') FORMAT Null; +======= +-- These values are within the extended range of DateTime64 [1925-01-01, 2284-01-01) +SELECT toDateTime(-2, 2); +1970-01-01 02:59:58.00 +SELECT toDateTime64(-2, 2); +1970-01-01 02:59:58.00 +SELECT CAST(-1 AS DateTime64); +1970-01-01 02:59:59.000 +SELECT CAST('2020-01-01 00:00:00.3' AS DateTime64); +2020-01-01 00:00:00.300 +SELECT toDateTime64(bitShiftLeft(toUInt64(1),33), 2); +2242-03-16 15:56:32.00 +-- These are outsize of extended range and hence clamped +SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1),35), 2); +1925-01-01 02:00:00.00 +SELECT CAST(-1 * bitShiftLeft(toUInt64(1),35) AS DateTime64); +1925-01-01 02:00:00.000 +SELECT CAST(bitShiftLeft(toUInt64(1),35) AS DateTime64); +2282-12-31 03:00:00.000 +SELECT toDateTime64(bitShiftLeft(toUInt64(1),35), 2); +2282-12-31 03:00:00.00 +>>>>>>> af31042451... Extended range of DateTime64 to years 1925 - 2238 diff --git a/tests/queries/0_stateless/01691_DateTime64_clamp.sql b/tests/queries/0_stateless/01691_DateTime64_clamp.sql index 92d5a33328f..958de4edada 100644 --- a/tests/queries/0_stateless/01691_DateTime64_clamp.sql +++ b/tests/queries/0_stateless/01691_DateTime64_clamp.sql @@ -1,4 +1,5 @@ -- { echo } +-- These values are within the extended range of DateTime64 [1925-01-01, 2284-01-01) SELECT toTimeZone(toDateTime(-2, 2), 'Europe/Moscow'); SELECT toDateTime64(-2, 2, 'Europe/Moscow'); SELECT CAST(-1 AS DateTime64(0, 'Europe/Moscow')); @@ -8,3 +9,9 @@ SELECT toTimeZone(toDateTime(-2., 2), 'Europe/Moscow'); SELECT toDateTime64(-2., 2, 'Europe/Moscow'); SELECT toDateTime64(toFloat32(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow'); SELECT toDateTime64(toFloat64(bitShiftLeft(toUInt64(1),33)), 2, 'Europe/Moscow') FORMAT Null; + +-- These are outsize of extended range and hence clamped +SELECT toDateTime64(-1 * bitShiftLeft(toUInt64(1),35), 2); +SELECT CAST(-1 * bitShiftLeft(toUInt64(1),35) AS DateTime64); +SELECT CAST(bitShiftLeft(toUInt64(1),35) AS DateTime64); +SELECT toDateTime64(bitShiftLeft(toUInt64(1),35), 2); diff --git a/utils/convert-month-partitioned-parts/main.cpp b/utils/convert-month-partitioned-parts/main.cpp index 0a697937eb6..a6829d79726 100644 --- a/utils/convert-month-partitioned-parts/main.cpp +++ b/utils/convert-month-partitioned-parts/main.cpp @@ -47,8 +47,9 @@ void run(String part_path, String date_column, String dest_path) DayNum max_date; MergeTreePartInfo::parseMinMaxDatesFromPartName(old_part_name, min_date, max_date); - UInt32 yyyymm = DateLUT::instance().toNumYYYYMM(min_date); - if (yyyymm != DateLUT::instance().toNumYYYYMM(max_date)) + const auto & time_zone = DateLUT::instance(); + UInt32 yyyymm = time_zone.toNumYYYYMM(min_date); + if (yyyymm != time_zone.toNumYYYYMM(max_date)) throw Exception("Part " + old_part_name + " spans different months", ErrorCodes::BAD_DATA_PART_NAME);