thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-30 03:22:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added expression support in sorting functions

Browse Source
This commit is contained in:
root 2022-08-24 11:21:58 -07:00 committed by Yong Wang
parent 422189e899
commit 79ee761e7b
4 changed files with 37 additions and 133 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/Parsers/Kusto/KQL_ReleaseNote.md
View File

@ -8,8 +8,8 @@
`print t = array_sort_asc(dynamic([4, 1, 3, 2]))`
`print array_sort_asc(dynamic(['b', 'a', 'c']), dynamic([20, 10, 30]))`
`print array_sort_asc(dynamic([2, 1, 3]), dynamic(['clickhouse','hello', 'world']))`
***For below queries, do not support expression in place of boolean yet***
`print t = array_sort_asc( dynamic(['d', null, 'a', 'c', 'c']) , false)`
`print t = array_sort_asc( dynamic(['d', null, 'a', 'c', 'c']) , 1 > 2)`
`print t = array_sort_asc( dynamic([null, 'd', null, null, 'a', 'c', 'c', null, null, null]) , false)`
`print t = array_sort_asc( dynamic([null, null, null]) , false)`
- [array_sort_desc](https://docs.microsoft.com/en-us/azure/data-explorer/kusto/query/arraysortdescfunction)
@ -17,7 +17,7 @@
`print t = array_sort_desc(dynamic([4, 1, 3, 2]))`
`print array_sort_desc(dynamic(['b', 'a', 'c']), dynamic([20, 10, 30]))`
`print array_sort_desc(dynamic([2, 1, 3]), dynamic(['clickhouse','hello', 'world']))`
***For below queries, do not support expression in place of boolean yet***
`print t = array_sort_desc( dynamic(['d', null, 'a', 'c', 'c']) , 1 < 2)`
`print t = array_sort_desc( dynamic(['d', null, 'a', 'c', 'c']) , false)`
`print t = array_sort_desc( dynamic([null, 'd', null, null, 'a', 'c', 'c', null, null, null]) , false)`
`print t = array_sort_desc( dynamic([null, null, null]) , false)`

81
src/Parsers/Kusto/KustoFunctions/IParserKQLFunction.cpp
View File

@ -228,65 +228,34 @@ String IParserKQLFunction::ArraySortHelper(String & out,IParser::Pos & pos, bool
if(!ascending)
reverse = "Reverse";
++pos;
std::vector<String> argument_list;
String first_arg = getConvertedArgument(fn_name, pos);
if(pos->type == TokenType::Comma)
++pos;
String second_arg;
if(pos->type != TokenType::ClosingRoundBracket && String(pos->begin, pos->end) != "dynamic")
{
second_arg = getConvertedArgument(fn_name, pos);
out = "if (" + second_arg + ", array" + reverse + "Sort(" + first_arg + "), concat( arraySlice(array" + reverse + "Sort(" + first_arg + ") as as1, indexOf(as1, NULL) as len1 ), arraySlice( as1, 1, len1-1)))";
return out;
}
--pos;
std::vector<String> argument_list;
if(pos->type != TokenType::ClosingRoundBracket)
{
while(pos->type != TokenType::ClosingRoundBracket)
{
++pos;
String second_arg = getConvertedArgument(fn_name, pos);
trim(second_arg);
if(second_arg == "true" || second_arg == "false")
{
if(second_arg == "true")
out = "array" + reverse + "Sort(" + first_arg + ")";
else
{
int nulls_total = 0;
trim(first_arg);
first_arg = removeNULLs(first_arg, "null", nulls_total);
first_arg = removeNULLs(first_arg, "NULL", nulls_total);
int index = first_arg.size() - 1;
while(index > 0)
{
if(first_arg[index] == '\'' || first_arg[index] == '\"')
break;
if(first_arg[index] == ',')
{
first_arg[index] = ' ';
break;
}
index -= 1;
}
String null_array = "[";
if(nulls_total > 0)
{
while(nulls_total > 0)
{
null_array += "null";
if(nulls_total > 1)
null_array += ", ";
nulls_total -= 1;
}
null_array += "]";
out = "arrayConcat( " + null_array + " , array"+ reverse +"Sort( " + first_arg + " ) )";
}
else
out = "array" + reverse + "Sort(" + first_arg + ")";
}
}
else
{
argument_list.push_back("array"+ reverse +"Sort((x, y) -> y, " + second_arg + "," + first_arg + ")");
}
second_arg = getConvertedArgument(fn_name, pos);
argument_list.push_back("array"+ reverse +"Sort((x, y) -> y, " + second_arg + "," + first_arg + ")");
}
}
else
{
++pos;
out = "array"+ reverse +"Sort(" + first_arg + ")";
}
if(argument_list.size() > 0)
{
out = "array"+ reverse +"Sort(" + first_arg + ") AS array0_sorted, ";
@ -301,18 +270,4 @@ String IParserKQLFunction::ArraySortHelper(String & out,IParser::Pos & pos, bool
}
return out;
}
String IParserKQLFunction::removeNULLs(String arg, String nullString, int & nullsTotal)
{
size_t position = std::string::npos;
while (arg.find(nullString) != std::string::npos)
{
position = arg.find(nullString);
arg.erase(position, 4);
if(arg[position] == ',')
arg[position] = ' ';
nullsTotal += 1;
}
return arg;
}
}

1
src/Parsers/Kusto/KustoFunctions/IParserKQLFunction.h
View File

@ -52,6 +52,5 @@ protected:
static void validateEndOfFunction(const String & fn_name, IParser::Pos & pos);
static String getKQLFunctionName(IParser::Pos & pos);
static String ArraySortHelper(String & out, IParser::Pos & pos, bool ascending);
static String removeNULLs(String arg, String nullString, int & nullsTotal);
};
}

84
src/Parsers/tests/KQL/gtest_KQL_dynamicFunctions.cpp
View File

@ -1,72 +1,7 @@
#include <Parsers/tests/gtest_common.h>
#include <IO/WriteBufferFromOStream.h>
#include <Interpreters/applyTableOverride.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/Access/ASTCreateUserQuery.h>
#include <Parsers/Access/ParserCreateUserQuery.h>
#include <Parsers/ParserAlterQuery.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/ParserOptimizeQuery.h>
#include <Parsers/ParserQueryWithOutput.h>
#include <Parsers/ParserAttachAccessEntity.h>
#include <Parsers/formatAST.h>
#include <Parsers/parseQuery.h>
#include <Parsers/Kusto/ParserKQLQuery.h>
#include <string_view>
#include <regex>
#include <gtest/gtest.h>
namespace
{
using namespace DB;
using namespace std::literals;
}
class ParserDynamicFuncTest : public ::testing::TestWithParam<std::tuple<std::shared_ptr<DB::IParser>, ParserTestCase>>
{};
TEST_P(ParserDynamicFuncTest, ParseQuery)
{ const auto & parser = std::get<0>(GetParam());
const auto & [input_text, expected_ast] = std::get<1>(GetParam());
ASSERT_NE(nullptr, parser);
if (expected_ast)
{
if (std::string(expected_ast).starts_with("throws"))
{
EXPECT_THROW(parseQuery(*parser, input_text.begin(), input_text.end(), 0, 0), DB::Exception);
}
else
{
ASTPtr ast;
ASSERT_NO_THROW(ast = parseQuery(*parser, input_text.begin(), input_text.end(), 0, 0));
if (std::string("CREATE USER or ALTER USER query") != parser->getName()
&& std::string("ATTACH access entity query") != parser->getName())
{
EXPECT_EQ(expected_ast, serializeAST(*ast->clone(), false));
}
else
{
if (input_text.starts_with("ATTACH"))
{
auto salt = (dynamic_cast<const ASTCreateUserQuery *>(ast.get())->auth_data)->getSalt();
EXPECT_TRUE(std::regex_match(salt, std::regex(expected_ast)));
}
else
{
EXPECT_TRUE(std::regex_match(serializeAST(*ast->clone(), false), std::regex(expected_ast)));
}
}
}
}
else
{
ASSERT_THROW(parseQuery(*parser, input_text.begin(), input_text.end(), 0, 0), DB::Exception);
}
}
INSTANTIATE_TEST_SUITE_P(ParserKQLQuery, ParserDynamicFuncTest,
INSTANTIATE_TEST_SUITE_P(ParserKQLQuery, ParserTest,
::testing::Combine(
::testing::Values(std::make_shared<DB::ParserKQLQuery>()),
::testing::ValuesIn(std::initializer_list<ParserTestCase>{
@ -98,6 +33,14 @@ class ParserDynamicFuncTest : public ::testing::TestWithParam<std::tuple<std::sh
"print t = array_sort_asc( dynamic([null, null, null]) , false)",
"SELECT arrayConcat([NULL, NULL, NULL], arraySort([])) AS t"
},
{
"print t = array_sort_asc( dynamic(['d', null, 'a', 'c', 'c']) , 1 < 2)",
"SELECT if(1 < 2, arraySort(['d', NULL, 'a', 'c', 'c']), concat(arraySlice(arraySort(['d', NULL, 'a', 'c', 'c']) AS as1, indexOf(as1, NULL) AS len1), arraySlice(as1, 1, len1 - 1))) AS t"
},
{
"print t = array_sort_asc( dynamic(['d', null, 'a', 'c', 'c']) , 1 > 2)",
"SELECT if(1 > 2, arraySort(['d', NULL, 'a', 'c', 'c']), concat(arraySlice(arraySort(['d', NULL, 'a', 'c', 'c']) AS as1, indexOf(as1, NULL) AS len1), arraySlice(as1, 1, len1 - 1))) AS t"
},
{
"print t = array_sort_desc(dynamic([null, 'd', 'a', 'c', 'c']))",
"SELECT arrayReverseSort([NULL, 'd', 'a', 'c', 'c']) AS t"
@ -125,6 +68,13 @@ class ParserDynamicFuncTest : public ::testing::TestWithParam<std::tuple<std::sh
{
"print t = array_sort_desc( dynamic([null, null, null]) , false)",
"SELECT arrayConcat([NULL, NULL, NULL], arrayReverseSort([])) AS t"
},
{
"print t = array_sort_desc( dynamic(['d', null, 'a', 'c', 'c']) , 1 < 2)",
"SELECT if(1 < 2, arrayReverseSort(['d', NULL, 'a', 'c', 'c']), concat(arraySlice(arrayReverseSort(['d', NULL, 'a', 'c', 'c']) AS as1, indexOf(as1, NULL) AS len1), arraySlice(as1, 1, len1 - 1))) AS t"
},
{
"print t = array_sort_desc( dynamic(['d', null, 'a', 'c', 'c']) , 1 > 2)",
"SELECT if(1 > 2, arrayReverseSort(['d', NULL, 'a', 'c', 'c']), concat(arraySlice(arrayReverseSort(['d', NULL, 'a', 'c', 'c']) AS as1, indexOf(as1, NULL) AS len1), arraySlice(as1, 1, len1 - 1))) AS t"
}
})));