From 9806fce2c02784d9b4955cf120930c1c9167c380 Mon Sep 17 00:00:00 2001 From: ltrk2 <107155950+ltrk2@users.noreply.github.com> Date: Fri, 16 Sep 2022 12:43:46 -0700 Subject: [PATCH] Implement KQL functional tests and fix bugs that came up --- .../KustoFunctions/KQLDynamicFunctions.cpp | 8 +- .../KQLMathematicalFunctions.cpp | 11 +- .../0_stateless/02366_kql_datatype.reference | 7 +- .../0_stateless/02366_kql_datatype.sql | 10 +- .../02366_kql_func_dynamic.reference | 101 +++++++++++++++++ .../0_stateless/02366_kql_func_dynamic.sql | 104 +++++++++++++++++- .../0_stateless/02366_kql_math.reference | 4 + tests/queries/0_stateless/02366_kql_math.sql | 7 ++ 8 files changed, 241 insertions(+), 11 deletions(-) create mode 100644 tests/queries/0_stateless/02366_kql_math.reference create mode 100644 tests/queries/0_stateless/02366_kql_math.sql diff --git a/src/Parsers/Kusto/KustoFunctions/KQLDynamicFunctions.cpp b/src/Parsers/Kusto/KustoFunctions/KQLDynamicFunctions.cpp index e4ac4dd0ee6..f2e4f3b8d46 100644 --- a/src/Parsers/Kusto/KustoFunctions/KQLDynamicFunctions.cpp +++ b/src/Parsers/Kusto/KustoFunctions/KQLDynamicFunctions.cpp @@ -86,7 +86,10 @@ bool ArrayRotateLeft::convertImpl(String & out, IParser::Pos & pos) const auto array = getArgument(function_name, pos); const auto count = getArgument(function_name, pos); out = std::format( - "arrayMap(x -> {0}[(x + length({0}) + {1} % toInt64(length({0}))) % length({0}) + 1], range(0, length({0})))", array, count); + "arrayMap(x -> {0}[moduloOrZero(x + length({0}) + moduloOrZero({1}, toInt64(length({0}))), length({0})) + 1], " + "range(0, length({0})))", + array, + count); return true; } @@ -115,7 +118,8 @@ bool ArrayShiftLeft::convertImpl(String & out, IParser::Pos & pos) const auto fill = getOptionalArgument(function_name, pos); out = std::format( "arrayResize(if({1} > 0, arraySlice({0}, {1} + 1), arrayConcat(arrayWithConstant(abs({1}), fill_value_{3}), {0})), " - "length({0}), ifNull({2}, if(toTypeName({0}) = 'Array(String)', defaultValueOfArgumentType({0}[1]), null)) as fill_value_{3})", + "length({0}), if(isNull({2}) and (extract(toTypeName({0}), 'Array\\((.*)\\)') as element_type_{3}) = 'String', " + "defaultValueOfTypeName(if(element_type_{3} = 'Nothing', 'Nullable(Nothing)', element_type_{3})), {2}) as fill_value_{3})", array, count, fill ? *fill : "null", diff --git a/src/Parsers/Kusto/KustoFunctions/KQLMathematicalFunctions.cpp b/src/Parsers/Kusto/KustoFunctions/KQLMathematicalFunctions.cpp index 77aacd169e5..9e4df709f8d 100644 --- a/src/Parsers/Kusto/KustoFunctions/KQLMathematicalFunctions.cpp +++ b/src/Parsers/Kusto/KustoFunctions/KQLMathematicalFunctions.cpp @@ -1,9 +1,18 @@ #include "KQLMathematicalFunctions.h" +#include + namespace DB { bool IsNan::convertImpl(String & out, IParser::Pos & pos) { - return directMapping(out, pos, "isNaN"); + const auto function_name = getKQLFunctionName(pos); + if (function_name.empty()) + return false; + + const auto argument = getArgument(function_name, pos); + out = std::format("if(toTypeName({0}) = 'Float64', isNaN({0}), throwIf(true, 'Expected argument of data type real'))", argument); + + return true; } } diff --git a/tests/queries/0_stateless/02366_kql_datatype.reference b/tests/queries/0_stateless/02366_kql_datatype.reference index b0b146bdf50..5aa4b509d53 100644 --- a/tests/queries/0_stateless/02366_kql_datatype.reference +++ b/tests/queries/0_stateless/02366_kql_datatype.reference @@ -47,9 +47,12 @@ inf \N 123.345 -- dynamic -[1,2,3] -['a','b','c'] \N +1 +86400 +[1,2,3] +[[1],[2],[3]] +['a','b','c'] -- cast functions true 1 diff --git a/tests/queries/0_stateless/02366_kql_datatype.sql b/tests/queries/0_stateless/02366_kql_datatype.sql index 602dc619109..fdd48fb3688 100644 --- a/tests/queries/0_stateless/02366_kql_datatype.sql +++ b/tests/queries/0_stateless/02366_kql_datatype.sql @@ -53,11 +53,13 @@ print bool(null), int(null), long(null), real(null), double(null); print '-- decimal'; print decimal(null); print decimal(123.345); -print '-- dynamic'; -- only support 1D array at the moment -print dynamic([1,2,3]); -print dynamic(['a', 'b', 'c']); +print '-- dynamic'; -- no support for mixed types and bags for now print dynamic(null); - +print dynamic(1); +print dynamic(timespan(1d)); +print dynamic([1,2,3]); +print dynamic([[1], [2], [3]]); +print dynamic(['a', "b", 'c']); print '-- cast functions' print '--tobool("true")'; -- == true print tobool('true'); -- == true diff --git a/tests/queries/0_stateless/02366_kql_func_dynamic.reference b/tests/queries/0_stateless/02366_kql_func_dynamic.reference index 5de387c3a78..cb160ab9d19 100644 --- a/tests/queries/0_stateless/02366_kql_func_dynamic.reference +++ b/tests/queries/0_stateless/02366_kql_func_dynamic.reference @@ -14,9 +14,51 @@ [NULL,NULL,NULL] -- array_concat() [1,2,3,4,5,6] +-- array_reverse() +[] +[1] +[4,3,2,1] +['example','an','is','this'] +-- array_rotate_left() +[] +[] +[] +[3,4,5,1,2] +[1,2,3,4,5] +[3,4,5,1,2] +[4,5,1,2,3] +[1,2,3,4,5] +[4,5,1,2,3] +-- array_rotate_right() +[] +[] +[] +[4,5,1,2,3] +[1,2,3,4,5] +[4,5,1,2,3] +[3,4,5,1,2] +[1,2,3,4,5] +[3,4,5,1,2] +-- array_shift_left() +[] +[] +[] +[3,4,5,NULL,NULL] +[NULL,NULL,1,2,3] +[3,4,5,-1,-1] +['c','',''] +-- array_shift_right() +[] +[] +[] +[3,4,5,NULL,NULL] +[NULL,NULL,1,2,3] +[3,4,5,-1,-1] +['c','',''] -- array_slice() [2,3] [3,4] +-- array_split() [[1],[2,3],[4,5]] [[1,2],[3,4,5]] [[1],[2,3],[4,5]] @@ -40,9 +82,68 @@ ['c','b','a'] [['r','q','p'],['world','clickhouse','hello']] [[NULL,'d','c','c','a']] +[[NULL,'d','c','c','a']] [[NULL,NULL,NULL]] [3,2,1,NULL,NULL] ['d','c','b','e','a'] [['Ringo','Paul','John','George']] [['yellow','green','blue',NULL,NULL]] [[NULL,NULL,'yellow','green','blue']] +-- jaccard_index() +0.75 +0 +0 +nan +0 +0.75 +0.25 +-- pack_array() +1 2 4 [1,2,4] +['ab','0.0.0.42','4.2'] +-- repeat() +[] +[1,1,1] +['asd','asd','asd'] +[86400,86400,86400] +[true,true,true] +-- set_difference() +[] +[] +[] +[] +[4,5,6] +[4] +[1,3] +[1,2,3] +['d','s'] +['Chewbacca','Han Solo'] +-- set_has_element() +0 +1 +0 +1 +0 +-- set_intersect() +[] +[1,2,3] +[1,2,3] +[] +[5] +[] +['a'] +['Darth Vader'] +-- set_union() +[] +[1,2,3] +[1,2,3,4,5,6] +[1,2,3,4] +[1,2,3,4,5] +[1,2,3] +['a','d','f','s'] +['Chewbacca','Darth Sidious','Darth Vader','Han Solo'] +-- zip() +[] +[[1,2],[3,4],[5,6]] +[['Darth','Vader','has a suit'],['Master','Yoda','doesn\'t have a suit']] +[[1,10],[2,20],[3,NULL]] +[[NULL,1],[NULL,2],[NULL,3]] diff --git a/tests/queries/0_stateless/02366_kql_func_dynamic.sql b/tests/queries/0_stateless/02366_kql_func_dynamic.sql index 54dea08b52a..333e182a6a6 100644 --- a/tests/queries/0_stateless/02366_kql_func_dynamic.sql +++ b/tests/queries/0_stateless/02366_kql_func_dynamic.sql @@ -15,9 +15,50 @@ print array_iif(dynamic([true,false,true]), dynamic([1,2]), dynamic([4,5,6])); print array_iif(dynamic(['a','b','c']), dynamic([1,2,3]), dynamic([4,5,6])); print '-- array_concat()'; print array_concat(dynamic([1,2,3]),dynamic([4,5,6])); +print '-- array_reverse()'; +print array_reverse(dynamic([])); +print array_reverse(dynamic([1])); +print array_reverse(dynamic([1,2,3,4])); +print array_reverse(dynamic(["this", "is", "an", "example"])); +print '-- array_rotate_left()'; +print array_rotate_left(dynamic([]), 0); +print array_rotate_left(dynamic([]), 500); +print array_rotate_left(dynamic([]), -500); +print array_rotate_left(dynamic([1,2,3,4,5]), 2); +print array_rotate_left(dynamic([1,2,3,4,5]), 5); +print array_rotate_left(dynamic([1,2,3,4,5]), 7); +print array_rotate_left(dynamic([1,2,3,4,5]), -2); +print array_rotate_left(dynamic([1,2,3,4,5]), -5); +print array_rotate_left(dynamic([1,2,3,4,5]), -7); +print '-- array_rotate_right()'; +print array_rotate_right(dynamic([]), 0); +print array_rotate_right(dynamic([]), 500); +print array_rotate_right(dynamic([]), -500); +print array_rotate_right(dynamic([1,2,3,4,5]), 2); +print array_rotate_right(dynamic([1,2,3,4,5]), 5); +print array_rotate_right(dynamic([1,2,3,4,5]), 7); +print array_rotate_right(dynamic([1,2,3,4,5]), -2); +print array_rotate_right(dynamic([1,2,3,4,5]), -5); +print array_rotate_right(dynamic([1,2,3,4,5]), -7); +print '-- array_shift_left()'; +print array_shift_left(dynamic([]), 0); +print array_shift_left(dynamic([]), 555); +print array_shift_left(dynamic([]), -555); +print array_shift_left(dynamic([1,2,3,4,5]), 2); +print array_shift_left(dynamic([1,2,3,4,5]), -2); +print array_shift_left(dynamic([1,2,3,4,5]), 2, -1); +print array_shift_left(dynamic(['a', 'b', 'c']), 2); +print '-- array_shift_right()'; +print array_shift_left(dynamic([]), 0); +print array_shift_left(dynamic([]), 555); +print array_shift_left(dynamic([]), -555); +print array_shift_right(dynamic([1,2,3,4,5]), -2); +print array_shift_right(dynamic([1,2,3,4,5]), 2); +print array_shift_right(dynamic([1,2,3,4,5]), -2, -1); +print array_shift_right(dynamic(['a', 'b', 'c']), -2); print '-- array_slice()'; print array_slice(dynamic([1,2,3]), 1, 2); -print array_slice(dynamic([1,2,3,4,5]), -3, -2) +print array_slice(dynamic([1,2,3,4,5]), -3, -2); print '-- array_split()'; print array_split(dynamic([1,2,3,4,5]), dynamic([1,-2])); print array_split(dynamic([1,2,3,4,5]), 2); @@ -40,7 +81,7 @@ print '-- array_sort_desc()'; print t = array_sort_desc(dynamic([null, 'd', 'a', 'c', 'c'])); print t = array_sort_desc(dynamic([4, 1, 3, 2])); print t = array_sort_desc(dynamic(['b', 'a', 'c']), dynamic(['q', 'p', 'r']))[1]; -print array_sort_desc(dynamic(['q', 'p', 'r']), dynamic(['clickhouse','hello', 'world'])) +print array_sort_desc(dynamic(['q', 'p', 'r']), dynamic(['clickhouse','hello', 'world'])); print t = array_sort_desc( dynamic(['d', null, 'a', 'c', 'c']) , false); print t = array_sort_desc( dynamic(['d', null, 'a', 'c', 'c']) , 1 > 2); print t = array_sort_desc( dynamic([null, null, null]) , false); @@ -49,3 +90,62 @@ print array_sort_desc(dynamic(['1','3','4','5','2']),dynamic(["a","b","c","d","e print array_sort_desc(split("John,Paul,George,Ringo", ",")); print array_sort_desc(dynamic([null,"blue","yellow","green",null])); print v=array_sort_desc(dynamic([null,"blue","yellow","green",null]), false); +print '-- jaccard_index()'; +print jaccard_index(dynamic([1, 1, 2, 2, 3, 3]), dynamic([1, 2, 3, 4, 4, 4])); +print jaccard_index(dynamic([1, 2, 3]), dynamic([])); +print jaccard_index(dynamic([]), dynamic([1, 2, 3, 4])); +print jaccard_index(dynamic([]), dynamic([])); +print jaccard_index(dynamic([1, 2, 3]), dynamic([4, 5, 6, 7])); +print jaccard_index(dynamic(['a', 's', 'd']), dynamic(['f', 'd', 's', 'a'])); +print jaccard_index(dynamic(['Chewbacca', 'Darth Vader', 'Han Solo']), dynamic(['Darth Sidious', 'Darth Vader'])); +print '-- pack_array()'; +print pack_array(); -- { serverError } +print x = 1, y = x * 2, z = y * 2, pack_array(x,y,z); +print pack_array(strcat('a', 'b'), format_ipv4(42), tostring(4.2)); +print '-- repeat()'; +print repeat(1, 0); +print repeat(1, 3); +print repeat("asd", 3); +print repeat(timespan(1d), 3); +print repeat(true, 3); +print '-- set_difference()'; +print set_difference(dynamic([]), dynamic([])); +print set_difference(dynamic([]), dynamic([9])); +print set_difference(dynamic([]), dynamic(["asd"])); +print set_difference(dynamic([1, 1, 2, 2, 3, 3]), dynamic([1, 2, 3])); +print array_sort_asc(set_difference(dynamic([1, 4, 2, 3, 5, 4, 6]), dynamic([1, 2, 3])))[1]; +print set_difference(dynamic([4]), dynamic([1, 2, 3])); +print array_sort_asc(set_difference(dynamic([1, 2, 3, 4, 5]), dynamic([5]), dynamic([2, 4])))[1]; +print array_sort_asc(set_difference(dynamic([1, 2, 3]), dynamic([])))[1]; +print array_sort_asc(set_difference(dynamic(['a', 's', 'd']), dynamic(['a', 'f'])))[1]; +print array_sort_asc(set_difference(dynamic(['Chewbacca', 'Darth Vader', 'Han Solo']), dynamic(['Darth Sidious', 'Darth Vader'])))[1]; +print '-- set_has_element()'; +print set_has_element(dynamic([]), 9); +print set_has_element(dynamic(["this", "is", "an", "example"]), "example"); +print set_has_element(dynamic(["this", "is", "an", "example"]), "examplee"); +print set_has_element(dynamic([1, 2, 3]), 2); +print set_has_element(dynamic([1, 2, 3, 4.2]), 4); +print '-- set_intersect()'; +print set_intersect(dynamic([]), dynamic([])); +print array_sort_asc(set_intersect(dynamic([1, 1, 2, 2, 3, 3]), dynamic([1, 2, 3])))[1]; +print array_sort_asc(set_intersect(dynamic([1, 4, 2, 3, 5, 4, 6]), dynamic([1, 2, 3])))[1]; +print set_intersect(dynamic([4]), dynamic([1, 2, 3])); +print set_intersect(dynamic([1, 2, 3, 4, 5]), dynamic([1, 3, 5]), dynamic([2, 5])); +print set_intersect(dynamic([1, 2, 3]), dynamic([])); +print set_intersect(dynamic(['a', 's', 'd']), dynamic(['a', 'f'])); +print set_intersect(dynamic(['Chewbacca', 'Darth Vader', 'Han Solo']), dynamic(['Darth Sidious', 'Darth Vader'])); +print '-- set_union()'; +print set_union(dynamic([]), dynamic([])); +print array_sort_asc(set_union(dynamic([1, 1, 2, 2, 3, 3]), dynamic([1, 2, 3])))[1]; +print array_sort_asc(set_union(dynamic([1, 4, 2, 3, 5, 4, 6]), dynamic([1, 2, 3])))[1]; +print array_sort_asc(set_union(dynamic([4]), dynamic([1, 2, 3])))[1]; +print array_sort_asc(set_union(dynamic([1, 3, 4]), dynamic([5]), dynamic([2, 4])))[1]; +print array_sort_asc(set_union(dynamic([1, 2, 3]), dynamic([])))[1]; +print array_sort_asc(set_union(dynamic(['a', 's', 'd']), dynamic(['a', 'f'])))[1]; +print array_sort_asc(set_union(dynamic(['Chewbacca', 'Darth Vader', 'Han Solo']), dynamic(['Darth Sidious', 'Darth Vader'])))[1]; +print '-- zip()'; +print zip(dynamic([]), dynamic([])); +print zip(dynamic([1,3,5]), dynamic([2,4,6])); +print zip(dynamic(['Darth','Master']), dynamic(['Vader','Yoda']), dynamic(['has a suit','doesn\'t have a suit'])); +print zip(dynamic([1,2,3]), dynamic([10,20])); +print zip(dynamic([]), dynamic([1,2,3])); diff --git a/tests/queries/0_stateless/02366_kql_math.reference b/tests/queries/0_stateless/02366_kql_math.reference new file mode 100644 index 00000000000..92f283abcb6 --- /dev/null +++ b/tests/queries/0_stateless/02366_kql_math.reference @@ -0,0 +1,4 @@ +-- isnan -- +1 +0 +0 diff --git a/tests/queries/0_stateless/02366_kql_math.sql b/tests/queries/0_stateless/02366_kql_math.sql new file mode 100644 index 00000000000..4e83622eb6b --- /dev/null +++ b/tests/queries/0_stateless/02366_kql_math.sql @@ -0,0 +1,7 @@ +set dialect = 'kusto'; +print '-- isnan --'; +print isnan(double(nan)); +print isnan(4.2); +print isnan(4); -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO } +print isnan(real(+inf)); +print isnan(dynamic(null)); -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }