Part2 DateTime functions

src/Parsers/Kusto/KQL_ReleaseNote.md

@@ -162,6 +162,15 @@
 
 - [unixtime_seconds_todatetime](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/unixtime-seconds-todatetimefunction)  
    `print unixtime_seconds_todatetime(1546300800)`  
+   
+- [unixtime_microseconds_todatetime]
+   `print unixtime_microseconds_todatetime(1546300800000000)`    
+
+- [unixtime_milliseconds_todatetime]
+   `print unixtime_milliseconds_todatetime(1546300800000)`
+
+- [unixtime_nanoseconds_todatetime]
+   `print unixtime_nanoseconds_todatetime(1546300800000000000)`
 
 - [dayofweek](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/dayofweekfunction)  
    `print dayofweek(datetime(2015-12-20))`  
@@ -176,6 +185,38 @@
    `print now(-2d) `  
    `print now(time(1d))`  
 
+- [ago]
+   `print ago(2h)`
+
+- [endofday]
+   `print endofday(datetime(2017-01-01 10:10:17), -1)`
+   `print endofday(datetime(2017-01-01 10:10:17), 1)`
+   `print endofday(datetime(2017-01-01 10:10:17))`
+
+- [endofmonth]
+   `print endofmonth(datetime(2017-01-01 10:10:17), -1)`
+   `print endofmonth(datetime(2017-01-01 10:10:17), 1)`
+   `print endofmonth(datetime(2017-01-01 10:10:17))`
+
+- [endofweek]
+   `print endofweek(datetime(2017-01-01 10:10:17), 1)`
+   `print endofweek(datetime(2017-01-01 10:10:17), -1)`
+   `print endofweek(datetime(2017-01-01 10:10:17))` 
+
+- [endofyear]
+   `print endofyear(datetime(2017-01-01 10:10:17), -1)`
+   `print endofyear(datetime(2017-01-01 10:10:17), 1)`
+   `print endofyear(datetime(2017-01-01 10:10:17))`
+
+- [make_datetime]
+   `print make_datetime(2017,10,01)`
+   `print make_datetime(2017,10,01,12,10)`
+   `print make_datetime(2017,10,01,12,11,0.1234567)`
+
+-  [datetime_diff]
+   `print datetime_diff('year',datetime(2017-01-01),datetime(2000-12-31))`
+   `print datetime_diff('quarter',datetime(2017-07-01),datetime(2017-03-30))`
+   `print datetime_diff('minute',datetime(2017-10-30 23:05:01),datetime(2017-10-30 23:00:59))` 
 
 ## Binary functions
 - [binary_and](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/binary-andfunction)  

src/Parsers/Kusto/KustoFunctions/KQLDateTimeFunctions.cpp

@@ -35,16 +35,24 @@ bool DateTime::convertImpl(String & out, IParser::Pos & pos)
 
 bool Ago::convertImpl(String & out, IParser::Pos & pos)
 {
-    String res = String(pos->begin, pos->end);
-    out = res;
-    return false;
+   const String fn_name = getKQLFunctionName(pos);
+    if (fn_name.empty())
+        return false;
+
+    ++pos;
+    if (pos->type != TokenType::ClosingRoundBracket)
+    {
+        const auto offset = getConvertedArgument(fn_name, pos);
+        out = std::format("now64(9,'UTC') - {}", offset);
+    }
+    else
+        out = "now64(9,'UTC')";
+    return true;
 }
 
 bool DatetimeAdd::convertImpl(String & out, IParser::Pos & pos)
 {
-    String res = String(pos->begin, pos->end);
-    out = res;
-    return false;
+    return directMapping(out, pos, "date_add");
 };
 
 bool DatetimePart::convertImpl(String & out, IParser::Pos & pos)
@@ -56,9 +64,21 @@ bool DatetimePart::convertImpl(String & out, IParser::Pos & pos)
 
 bool DatetimeDiff::convertImpl(String & out, IParser::Pos & pos)
 {
-    String res = String(pos->begin, pos->end);
-    out = res;
-    return false;
+    const String fn_name = getKQLFunctionName(pos);
+    if (fn_name.empty())
+        return false;
+    ++pos;
+    String arguments;
+    
+    arguments = arguments + getConvertedArgument(fn_name, pos) + ",";
+    ++pos;
+    arguments = arguments + getConvertedArgument(fn_name, pos) + ",";
+    ++pos;
+    arguments = arguments + getConvertedArgument(fn_name, pos);
+
+    out = std::format("ABS(DateDiff({}))",arguments);
+    return true;
+
 }
 
 bool DayOfMonth::convertImpl(String & out, IParser::Pos & pos)
@@ -84,25 +104,93 @@ bool DayOfYear::convertImpl(String & out, IParser::Pos & pos)
     return directMapping(out, pos, "toDayOfYear");
 }
 
+bool EndOfMonth::convertImpl(String & out, IParser::Pos & pos)
+{
+
+    const String fn_name = getKQLFunctionName(pos);
+    if (fn_name.empty())
+        return false;
+
+    ++pos;
+    const String datetime_str = getConvertedArgument(fn_name, pos);
+    String offset = "0";
+
+    if (pos->type == TokenType::Comma)
+    {
+         ++pos;
+         offset = getConvertedArgument(fn_name, pos);
+    }
+    
+    out = std::format("toDateTime(toStartOfDay({}),9,'UTC') + (INTERVAL {} +1 MONTH) - (INTERVAL 1 microsecond)", datetime_str, toString(offset));
+
+    return true;
+   
+}
+
 bool EndOfDay::convertImpl(String & out, IParser::Pos & pos)
 {
-    String res = String(pos->begin, pos->end);
-    out = res;
-    return false;
+
+    const String fn_name = getKQLFunctionName(pos);
+    if (fn_name.empty())
+        return false;
+
+    ++pos;
+    const String datetime_str = getConvertedArgument(fn_name, pos);
+    String offset = "0";
+
+    if (pos->type == TokenType::Comma)
+    {
+         ++pos;
+         offset = getConvertedArgument(fn_name, pos);
+    }
+    out = std::format("toDateTime(toStartOfDay({}),9,'UTC') + (INTERVAL {} +1 DAY) - (INTERVAL 1 microsecond)", datetime_str, toString(offset));
+
+    return true;
+   
 }
 
 bool EndOfWeek::convertImpl(String & out, IParser::Pos & pos)
 {
-    String res = String(pos->begin, pos->end);
-    out = res;
-    return false;
+    
+    const String fn_name = getKQLFunctionName(pos);
+    if (fn_name.empty())
+        return false;
+
+    ++pos;
+    const String datetime_str = getConvertedArgument(fn_name, pos);
+    String offset = "0";
+
+    if (pos->type == TokenType::Comma)
+    {
+         ++pos;
+         offset = getConvertedArgument(fn_name, pos);
+    }
+        out = std::format("toDateTime(toStartOfDay({}),9,'UTC') + (INTERVAL {} +1 WEEK) - (INTERVAL 1 microsecond)", datetime_str, toString(offset));
+
+    return true;
+   
 }
 
 bool EndOfYear::convertImpl(String & out, IParser::Pos & pos)
 {
-    String res = String(pos->begin, pos->end);
-    out = res;
-    return false;
+    
+    const String fn_name = getKQLFunctionName(pos);
+    if (fn_name.empty())
+        return false;
+
+    ++pos;
+    const String datetime_str = getConvertedArgument(fn_name, pos);
+    String offset = "0";
+
+    if (pos->type == TokenType::Comma)
+    {
+         ++pos;
+         offset = getConvertedArgument(fn_name, pos);
+    }
+        out = std::format("toDateTime(toStartOfDay({}),9,'UTC') + (INTERVAL {} +1 YEAR) - (INTERVAL 1 microsecond)", datetime_str, toString(offset));
+
+    return true;
+   
 }
 
 bool FormatDateTime::convertImpl(String & out, IParser::Pos & pos)
@@ -143,9 +231,44 @@ bool MakeTimeSpan::convertImpl(String & out, IParser::Pos & pos)
 
 bool MakeDateTime::convertImpl(String & out, IParser::Pos & pos)
 {
-    String res = String(pos->begin, pos->end);
-    out = res;
-    return false;
+    const String fn_name = getKQLFunctionName(pos);
+
+    if (fn_name.empty())
+        return false;
+
+    ++pos;
+    String arguments;
+    int number_of_arguments=1;
+    String argument;
+    while (pos->type != TokenType::ClosingRoundBracket)
+    {
+        argument = String(pos->begin,pos->end);
+        auto dot_pos = argument.find('.');
+
+        if (dot_pos == String::npos)
+            arguments = arguments +  String(pos->begin,pos->end);
+        else
+        {
+            arguments = arguments + argument.substr(dot_pos-1, dot_pos) + "," + argument.substr(dot_pos+1,argument.length());
+            number_of_arguments++;
+        }
+            
+        ++pos;
+        if(pos->type == TokenType::Comma)
+            number_of_arguments++;
+    }
+    
+    while(number_of_arguments < 7)
+    {
+        arguments = arguments+ ",";
+        arguments = arguments+ "0";
+        number_of_arguments++;
+    }
+    arguments = arguments + ",7,'UTC'";
+
+    out = std::format("makeDateTime64({})",arguments);
+    
+    return true;
 }
 
 bool Now::convertImpl(String & out, IParser::Pos & pos)
@@ -174,7 +297,7 @@ bool StartOfDay::convertImpl(String & out, IParser::Pos & pos)
 
     ++pos;
     const String datetime_str = getConvertedArgument(fn_name, pos);
-    String offset;
+    String offset = "0";
 
     if (pos->type == TokenType::Comma)
     {
@@ -194,7 +317,7 @@ bool StartOfMonth::convertImpl(String & out, IParser::Pos & pos)
 
     ++pos;
     const String datetime_str = getConvertedArgument(fn_name, pos);
-    String offset;
+    String offset = "0";
 
     if (pos->type == TokenType::Comma)
     {
@@ -214,7 +337,7 @@ bool StartOfWeek::convertImpl(String & out, IParser::Pos & pos)
 
     ++pos;
     const String datetime_str = getConvertedArgument(fn_name, pos);
-    String offset;
+    String offset = "0";
 
     if (pos->type == TokenType::Comma)
     {
@@ -234,7 +357,7 @@ bool StartOfYear::convertImpl(String & out, IParser::Pos & pos)
 
     ++pos;
     const String datetime_str = getConvertedArgument(fn_name, pos);
-    String offset ;
+    String offset = "0";
 
     if (pos->type == TokenType::Comma)
     {
@@ -247,23 +370,39 @@ bool StartOfYear::convertImpl(String & out, IParser::Pos & pos)
 
 bool UnixTimeMicrosecondsToDateTime::convertImpl(String & out, IParser::Pos & pos)
 {
-    String res = String(pos->begin, pos->end);
-    out = res;
-    return false;
+    const String fn_name = getKQLFunctionName(pos);
+    if (fn_name.empty())
+        return false;
+
+    ++pos;
+    const String value = getConvertedArgument(fn_name, pos);
+    out = std::format("fromUnixTimestamp64Micro({},'UTC')", value);
+    return true;
 }
 
 bool UnixTimeMillisecondsToDateTime::convertImpl(String & out, IParser::Pos & pos)
 {
-    String res = String(pos->begin, pos->end);
-    out = res;
-    return false;
+    const String fn_name = getKQLFunctionName(pos);
+    if (fn_name.empty())
+        return false;
+
+    ++pos;
+    const String value = getConvertedArgument(fn_name, pos);
+    out = std::format("fromUnixTimestamp64Milli({},'UTC')", value);
+    return true;
+
 }
 
 bool UnixTimeNanosecondsToDateTime::convertImpl(String & out, IParser::Pos & pos)
 {
-    String res = String(pos->begin, pos->end);
-    out = res;
-    return false;
+    const String fn_name = getKQLFunctionName(pos);
+    if (fn_name.empty())
+        return false;
+
+    ++pos;
+    const String value = getConvertedArgument(fn_name, pos);
+    out = std::format("fromUnixTimestamp64Nano({},'UTC')", value);
+    return true;
 }
 
 bool UnixTimeSecondsToDateTime::convertImpl(String & out, IParser::Pos & pos)

src/Parsers/Kusto/KustoFunctions/KQLDateTimeFunctions.h

@@ -19,6 +19,7 @@ protected:
     bool convertImpl(String &out,IParser::Pos &pos) override;
 };*/
 
+
 class Ago : public IParserKQLFunction
 {
 protected:
@@ -75,6 +76,13 @@ protected:
     bool convertImpl(String &out,IParser::Pos &pos) override;
 };
 
+class EndOfMonth : public IParserKQLFunction
+{
+protected:
+    const char * getName() const override { return "endofmonth()"; }
+    bool convertImpl(String &out,IParser::Pos &pos) override;
+};
+
 class EndOfWeek : public IParserKQLFunction
 {
 protected:

src/Parsers/Kusto/KustoFunctions/KQLFunctionFactory.cpp

@@ -31,6 +31,8 @@ namespace DB
         {"endofday", KQLFunctionValue::endofday},
         {"endofweek", KQLFunctionValue::endofweek},
         {"endofyear", KQLFunctionValue::endofyear},
+        {"endofmonth", KQLFunctionValue::endofmonth},
+
         {"format_datetime", KQLFunctionValue::format_datetime},
         {"format_timespan", KQLFunctionValue::format_timespan},
         {"getmonth", KQLFunctionValue::getmonth},
@@ -273,6 +275,9 @@ std::unique_ptr<IParserKQLFunction> KQLFunctionFactory::get(String &kql_function
         case KQLFunctionValue::endofyear:
             return std::make_unique<EndOfYear>();
         
+        case KQLFunctionValue::endofmonth:
+            return std::make_unique<EndOfMonth>();
+        
         case KQLFunctionValue::monthofyear:
             return std::make_unique<MonthOfYear>();
 

src/Parsers/Kusto/KustoFunctions/KQLFunctionFactory.h

@@ -19,6 +19,7 @@ namespace DB
         endofday,
         endofweek,
         endofyear,
+        endofmonth,
         monthofyear,
         format_datetime,
         format_timespan,

src/Parsers/tests/KQL/gtest_KQL_dateTimeFunctions.cpp

@@ -86,6 +86,10 @@ INSTANTIATE_TEST_SUITE_P(ParserKQLQuery, ParserDateTimeFuncTest,
             "SELECT toDateTime64(toStartOfDay(toDateTime64('2017-01-01 10:10:17', 9, 'UTC')), 9, 'UTC') + toIntervalDay(-1)"
 
         },
+        {
+            "print startofyear(datetime(2017-01-01 10:10:17), -1)",
+            "SELECT toDateTime64(toStartOfYear(toDateTime64('2017-01-01 10:10:17', 9, 'UTC'), 'UTC'), 9, 'UTC') + toIntervalYear(-1)"
+        },
         {
             "print monthofyear(datetime(2015-12-14))",
             "SELECT toMonth(toDateTime64('2015-12-14', 9, 'UTC'))"
@@ -125,6 +129,75 @@ INSTANTIATE_TEST_SUITE_P(ParserKQLQuery, ParserDateTimeFuncTest,
         {
             "print now(1d)",
             "SELECT now64(9, 'UTC') + 86400."
+        },
+        {
+            "print ago(2d)",
+            "SELECT now64(9, 'UTC') - 172800."
+        },  
+        {
+            "print endofday(datetime(2017-01-01 10:10:17), -1)",
+            "SELECT (toDateTime(toStartOfDay(toDateTime64('2017-01-01 10:10:17', 9, 'UTC')), 9, 'UTC') + toIntervalDay(-1 + 1)) - toIntervalMicrosecond(1)"
+        },
+        {
+            "print endofday(datetime(2017-01-01 10:10:17), 1)",
+            "SELECT (toDateTime(toStartOfDay(toDateTime64('2017-01-01 10:10:17', 9, 'UTC')), 9, 'UTC') + toIntervalDay(1 + 1)) - toIntervalMicrosecond(1)"
+
+        },
+        {
+            "print endofmonth(datetime(2017-01-01 10:10:17), -1)",
+            "SELECT (toDateTime(toStartOfDay(toDateTime64('2017-01-01 10:10:17', 9, 'UTC')), 9, 'UTC') + toIntervalMonth(-1 + 1)) - toIntervalMicrosecond(1)"
+        },
+        {
+            "print endofmonth(datetime(2017-01-01 10:10:17), 1)",
+            "SELECT (toDateTime(toStartOfDay(toDateTime64('2017-01-01 10:10:17', 9, 'UTC')), 9, 'UTC') + toIntervalMonth(1 + 1)) - toIntervalMicrosecond(1)"
+        },
+        {
+            "print endofweek(datetime(2017-01-01 10:10:17), -1)",
+            "SELECT (toDateTime(toStartOfDay(toDateTime64('2017-01-01 10:10:17', 9, 'UTC')), 9, 'UTC') + toIntervalWeek(-1 + 1)) - toIntervalMicrosecond(1)"
+        },
+        {
+            "print endofweek(datetime(2017-01-01 10:10:17), 1)",
+            "SELECT (toDateTime(toStartOfDay(toDateTime64('2017-01-01 10:10:17', 9, 'UTC')), 9, 'UTC') + toIntervalWeek(1 + 1)) - toIntervalMicrosecond(1)"
+        },
+        {
+            "print endofyear(datetime(2017-01-01 10:10:17), -1) ",
+            "SELECT (toDateTime(toStartOfDay(toDateTime64('2017-01-01 10:10:17', 9, 'UTC')), 9, 'UTC') + toIntervalYear(-1 + 1)) - toIntervalMicrosecond(1)"
+        },
+        {
+           "print endofyear(datetime(2017-01-01 10:10:17), 1)" ,
+           "SELECT (toDateTime(toStartOfDay(toDateTime64('2017-01-01 10:10:17', 9, 'UTC')), 9, 'UTC') + toIntervalYear(1 + 1)) - toIntervalMicrosecond(1)"
+        },
+        {
+            "print make_datetime(2017,10,01)",
+            "SELECT makeDateTime64(2017, 10, 1, 0, 0, 0, 0, 7, 'UTC')"
+        },
+        {
+            "print make_datetime(2017,10,01,12,10)",
+            "SELECT makeDateTime64(2017, 10, 1, 12, 10, 0, 0, 7, 'UTC')"
+        },
+        {
+            "print make_datetime(2017,10,01,12,11,0.1234567)",
+            "SELECT makeDateTime64(2017, 10, 1, 12, 11, 0, 1234567, 7, 'UTC')"
+        },
+        {
+            "print unixtime_microseconds_todatetime(1546300800000000)",
+            "SELECT fromUnixTimestamp64Micro(1546300800000000, 'UTC')"
+        },
+        {
+            "print unixtime_milliseconds_todatetime(1546300800000)",
+            "SELECT fromUnixTimestamp64Milli(1546300800000, 'UTC')"
+        },
+        {
+            "print unixtime_nanoseconds_todatetime(1546300800000000000)",
+            "SELECT fromUnixTimestamp64Nano(1546300800000000000, 'UTC')"
+        },
+        {
+            "print datetime_diff('year',datetime(2017-01-01),datetime(2000-12-31))",
+            "SELECT ABS(dateDiff('year', toDateTime64('2017-01-01', 9, 'UTC'), toDateTime64('2000-12-31', 9, 'UTC')))"
+        },
+        {
+            "print datetime_diff('minute',datetime(2017-10-30 23:05:01),datetime(2017-10-30 23:00:59))",
+            "SELECT ABS(dateDiff('minute', toDateTime64('2017-10-30 23:05:01', 9, 'UTC'), toDateTime64('2017-10-30 23:00:59', 9, 'UTC')))"
         }
 
 })));