From 760fd6759e9f9772486e9b2258132516cbcedec1 Mon Sep 17 00:00:00 2001 From: Yong Wang Date: Thu, 23 Jun 2022 14:26:37 -0700 Subject: [PATCH] Kusto-phase2: add kusto_auto dialect --- src/Client/ClientBase.cpp | 18 ++++- src/Interpreters/executeQuery.cpp | 17 +++- .../KustoFunctions/IParserKQLFunction.cpp | 78 ++++++++++--------- src/Parsers/Kusto/ParserKQLOperators.cpp | 1 + 4 files changed, 74 insertions(+), 40 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 495e3bdfd4e..2040c3f1440 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -328,12 +328,15 @@ void ClientBase::setupSignalHandler() ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, bool allow_multi_statements) const { - std::unique_ptr parser; + std::shared_ptr parser; + ParserKQLStatement kql_parser(end, global_context->getSettings().allow_settings_after_format_in_insert); ASTPtr res; const auto & settings = global_context->getSettingsRef(); size_t max_length = 0; + auto begin = pos; + if (!allow_multi_statements) max_length = settings.max_query_size; @@ -353,13 +356,22 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, bool allow_mu if (!res) { - std::cerr << std::endl << message << std::endl << std::endl; - return nullptr; + if (sql_dialect != "kusto") + res = tryParseQuery(kql_parser, begin, end, message, true, "", allow_multi_statements, max_length, settings.max_parser_depth); + + if (!res) + { + std::cerr << std::endl << message << std::endl << std::endl; + return nullptr; + } } } else { res = parseQueryAndMovePosition(*parser, pos, end, "", allow_multi_statements, max_length, settings.max_parser_depth); + + if (!res && sql_dialect != "kusto") + res = parseQueryAndMovePosition(kql_parser, begin, end, "", allow_multi_statements, max_length, settings.max_parser_depth); } if (is_interactive) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index bac6807b682..0cba3714855 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -701,12 +701,27 @@ static std::tuple executeQueryImpl( /// Parse the query from string. try { - if (settings.dialect == Dialect::kusto && !internal) + const String & sql_dialect = settings.sql_dialect; + assert(sql_dialect == "clickhouse" || sql_dialect == "kusto" || sql_dialect == "kusto_auto"); + + if (sql_dialect == "kusto" && !internal) { ParserKQLStatement parser(end, settings.allow_settings_after_format_in_insert); + ast = parseQuery(parser, begin, end, "", max_query_size, settings.max_parser_depth); + } + else if (sql_dialect == "kusto_auto" && !internal) + { + try { + ParserQuery parser(end, settings.allow_settings_after_format_in_insert); /// TODO: parser should fail early when max_query_size limit is reached. ast = parseQuery(parser, begin, end, "", max_query_size, settings.max_parser_depth); + } + catch(...) + { + ParserKQLStatement parser(end, settings.allow_settings_after_format_in_insert); + ast = parseQuery(parser, begin, end, "", max_query_size, settings.max_parser_depth); + } } else if (settings.dialect == Dialect::prql && !internal) { diff --git a/src/Parsers/Kusto/KustoFunctions/IParserKQLFunction.cpp b/src/Parsers/Kusto/KustoFunctions/IParserKQLFunction.cpp index ed90c865f51..73472a42010 100644 --- a/src/Parsers/Kusto/KustoFunctions/IParserKQLFunction.cpp +++ b/src/Parsers/Kusto/KustoFunctions/IParserKQLFunction.cpp @@ -15,6 +15,7 @@ #include #include #include +#include namespace DB { @@ -38,41 +39,36 @@ bool IParserKQLFunction::convert(String &out,IParser::Pos &pos) bool IParserKQLFunction::directMapping(String &out,IParser::Pos &pos,const String &ch_fn) { - std::unique_ptr fun; - std::vector args; + std::vector arguments; - String res =ch_fn + "("; - out = res; - auto begin = pos; + String fn_name = getKQLFunctionName(pos); - ++pos; - if (pos->type != TokenType::OpeningRoundBracket) - { - pos = begin; + if (fn_name.empty()) return false; - } + String res; + auto begin = pos; + ++pos; while (!pos->isEnd() && pos->type != TokenType::PipeMark && pos->type != TokenType::Semicolon) { - ++pos; - String tmp_arg = String(pos->begin,pos->end); - if (pos->type == TokenType::BareWord ) - { - String new_arg; - fun = KQLFunctionFactory::get(tmp_arg); - if (fun && fun->convert(new_arg,pos)) - tmp_arg = new_arg; - } - else if (pos->type == TokenType::ClosingRoundBracket) - { - for (auto arg : args) - res+=arg; + String argument = getConvertedArgument(fn_name,pos); + arguments.push_back(argument); + if (pos->type == TokenType::ClosingRoundBracket) + { + for (auto arg : arguments) + { + if (res.empty()) + res = ch_fn + "(" + arg; + else + res = res + ", "+ arg; + } res += ")"; + out = res; return true; } - args.push_back(tmp_arg); + ++pos; } pos = begin; @@ -82,6 +78,7 @@ bool IParserKQLFunction::directMapping(String &out,IParser::Pos &pos,const Strin String IParserKQLFunction::getConvertedArgument(const String &fn_name, IParser::Pos &pos) { String converted_arg; + std::vector tokens; std::unique_ptr fun; if (pos->type == TokenType::ClosingRoundBracket) @@ -93,23 +90,32 @@ String IParserKQLFunction::getConvertedArgument(const String &fn_name, IParser:: while (!pos->isEnd() && pos->type != TokenType::PipeMark && pos->type != TokenType::Semicolon) { String token = String(pos->begin,pos->end); - if (pos->type == TokenType::BareWord ) + String new_token; + if (!KQLOperators().convert(tokens,pos)) { - String converted; - fun = KQLFunctionFactory::get(token); - if ( fun && fun->convert(converted,pos)) - converted_arg += converted; + if (pos->type == TokenType::BareWord ) + { + String converted; + fun = KQLFunctionFactory::get(token); + if ( fun && fun->convert(converted,pos)) + tokens.push_back(converted); + else + tokens.push_back(token); + } + else if (pos->type == TokenType::Comma || pos->type == TokenType::ClosingRoundBracket) + { + break; + } else - converted_arg += token; + tokens.push_back(token); } - else if (pos->type == TokenType::Comma ||pos->type == TokenType::ClosingRoundBracket) - { - break; - } - else - converted_arg += token; ++pos; + if (pos->type == TokenType::Comma || pos->type == TokenType::ClosingRoundBracket) + break; } + for (auto token : tokens) + converted_arg = converted_arg + token +" "; + return converted_arg; } diff --git a/src/Parsers/Kusto/ParserKQLOperators.cpp b/src/Parsers/Kusto/ParserKQLOperators.cpp index 50550376667..33e8813286a 100644 --- a/src/Parsers/Kusto/ParserKQLOperators.cpp +++ b/src/Parsers/Kusto/ParserKQLOperators.cpp @@ -297,6 +297,7 @@ bool KQLOperators::convert(std::vector &tokens,IParser::Pos &pos) } return true; } + pos = begin; return false; }